libglusterfs: Adding missing event strings for events.

Change-Id: Idf55abfa77712d32b27798fedcfddb65599b562b Reviewed-on: http://review.gluster.com/493 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vijay@gluster.com>
author: Krishnan Parthasarathi <kp@gluster.com> 2011-09-23 11:49:29 +0530
committer: Vijay Bellur <vijay@gluster.com> 2011-09-23 01:19:38 -0700
commit: 2b21ea3c625fab19e88c21f4c104dca321eef262 (patch)
tree: afc74ccaa9a1c6ed412aace2f9403933aa4bfdf5 /cli
parent: 4765dd1a1c51c67ab86687fbd871c89156680c34 (diff)
0 files changed, 0 insertions, 0 deletions
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index c5371b26436..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,31 +3,123 @@ autom4te.cache
 build
 config.*
 configure
+cscope.*
+tags
 depcomp
+INSTALL
 install-sh
 ltmain.sh
+Makefile
 Makefile.in
 missing
+stamp-h1
+stamp-h2
+test-driver
+*compile
+*.gcda
+*.gcno
 *.sw?
 *~
-*lo
-*la
-*o
+*.lo
+*.la
+*.o
+*.tar.gz
+*.rpm
+*.diff
+*.patch
 .libs
-Makefile
-stamp-h1
+.deps
+.dirstamp
+
+# Softlinks to test and log
+log
+*.vol
+.clang-format
+
+# cmocka unit tests
+*.log
+*.trs
+*_unittest
 
 # Generated files
-extras/init.d/glusterfs-server.plist
-extras/init.d/glusterfsd-Debian
-extras/init.d/glusterfsd-Redhat
-extras/init.d/glusterfsd-SuSE
+site.h
+*.py[co]
+api/examples/__init__.py
+api/examples/setup.py
+api/src/gfapi.map
+cli/src/gluster
+contrib/fuse-util/fusermount-glusterfs
+extras/geo-rep/gsync-sync-gfid
+extras/geo-rep/schedule_georep.py
+extras/snap_scheduler/conf.py
+extras/init.d/glusterd-Debian
+extras/init.d/glusterd-FreeBSD
+extras/init.d/glusterd-Redhat
+extras/init.d/glusterd-SuSE
+extras/init.d/glusterd.plist
+extras/ocf/glusterd
+extras/ocf/volume
+extras/run-gluster.tmpfiles
+extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
+extras/who-wrote-glusterfs/gitdm
+geo-replication/.tox
+geo-replication/gsyncd.conf
+geo-replication/src/gsyncd
+geo-replication/src/peer_gsec_create
+geo-replication/src/peer_mountbroker
+geo-replication/src/peer_mountbroker.py
+geo-replication/src/set_geo_rep_pem_keys.sh
+geo-replication/src/peer_georep-sshkey.py
+geo-replication/syncdaemon.egg-info
+geo-replication/syncdaemon/conf.py
+geo-replication/tests/unit/.coverage
+geo-replication/tests/unit/cover
+geo-replication/tests/unit/coverage.xml
+geo-replication/tests/unit/nosetests.xml
+geo-replication/tests/unit/results.html
+glusterfs-api.pc
 glusterfs.spec
-libtool
-xlators/mount/fuse/utils/mount.glusterfs
-xlators/mount/fuse/utils/mount_glusterfs
-argp-standalone/libargp.a
+glusterfsd/src/glusterd
+glusterfsd/src/glusterfs
 glusterfsd/src/glusterfsd
-libglusterfs/src/spec.lex.c
+glusterfsd/src/gf_attach
+heal/src/glfsheal
+libgfchangelog.pc
+libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libglusterfs/src/defaults.c
+libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
+libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
+run-tests.sh
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
+tests/env.rc
+tests/utils/arequal-checksum
+xlators/features/glupy/src/__init__.py
+xlators/features/glupy/src/setup.py
+xlators/mount/fuse/utils/mount.glusterfs
+xlators/mount/fuse/utils/mount_glusterfs
+extras/peer_add_secret_pub
+tools/gfind_missing_files/gcrawler
+tools/glusterfind/glusterfind
+tools/glusterfind/src/tool.conf
+# Eventing
+events/src/eventsapiconf.py
+extras/systemd/glustereventsd.service
+events/src/eventtypes.py
+libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 00000000000..141a1667ffa
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,43 @@
+# .mailmap, see 'git short-log --help' for details
+#
+# This file needs to match extras/who-wrote-glusterfs/gitdm.aliases.
+#
+# Listing of contributors that filed patches with different email addresses.
+# Format: <name> <main-email> <alias> [<alias> ...]
+#
+
+Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
+Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
+Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
+Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
+Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
+Justin Clift <justin@gluster.org> <jclift@redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
+Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
+Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
+Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
+Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
+Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
+M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
+Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
+Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Prashanth Pai <ppai@redhat.com> <nullpai@gmail.com>
+Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
+Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
+Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
+Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
+Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
+Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
+Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
+Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
+Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
+shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/COPYING b/COPYING
deleted file mode 100644
index dba13ed2ddf..00000000000
--- a/COPYING
+++ /dev/null
@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.
diff --git a/COPYING-GPLV2 b/COPYING-GPLV2
new file mode 100644
index 00000000000..d159169d105
--- /dev/null
+++ b/COPYING-GPLV2
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/COPYING-LGPLV3 b/COPYING-LGPLV3
new file mode 100644
index 00000000000..65c5ca88a67
--- /dev/null
+++ b/COPYING-LGPLV3
@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/INSTALL b/INSTALL
index 88e28999df9..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,32 +1,47 @@
 Installation Instructions
 *************************
 
-Run ./configure after untaring the package.
+0. If you have cloned from git, run ./autogen.sh.
+
+1. Run ./configure.
 
  bash# ./configure
  GlusterFS configure summary
  ===========================
- FUSE client        : yes
- Infiniband verbs   : yes
- epoll IO multiplex : yes
- Berkeley-DB        : yes
- libglusterfsclient : yes
- mod_glusterfs      : yes
- argp-standalone    : no
+ GlusterFS configure summary
+  ===========================
+   FUSE client          : yes
+   Infiniband verbs     : yes
+   epoll IO multiplex   : yes
+   argp-standalone      : no
+   fusermount           : yes
+   readline             : yes
+   georeplication       : yes
+   Linux-AIO            : yes
+   Enable Debug         : no
+   Block Device xlator  : yes
+   glupy                : yes
+   Use syslog           : yes
+   XML output           : yes
+   QEMU Block formats   : yes
+   Encryption xlator    : yes
+
  
 The configure summary will tell you what all components will be built with 
 GlusterFS. Other than 'argp-standalone' if something else says 'no', that 
 feature in GlusterFS will not be built. 'argp-standalone' package will only 
 be used if the system doesn't have a proper argp package installed.
 
-Now just run 'make' and later run 'make install' to install the package.
+2. Now just run 'make' and later run 'make install' to install the package.
 
  bash# make 
  bash# make install
 
-Installation complete :-)
+Installation completed :-)
 
  bash# glusterfs --version
 
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
+
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
new file mode 100644
index 00000000000..953e8755fd9
--- /dev/null
+++ b/MAINTAINERS
@@ -0,0 +1,510 @@
+GlusterFS Maintainers
+=====================
+
+The intention of this file is not to establish who owns what portions of the
+code base, but to provide a set of names that developers can consult when they
+have a question about a particular subset and also to provide a set of names
+to be CC'd when submitting a patch to obtain appropriate review.
+
+In general, if you have a question about inclusion of a patch, you should
+consult gluster-devel@gluster.org and not any specific individual privately.
+
+Descriptions of section entries:
+
+	M: Main contact that knows and takes care of this area
+	L: Mailing list that is relevant to this area
+	W: Web-page with status/info
+	Q: Patchwork web based patch tracking system site
+	T: SCM tree type and location.  Type is one of: git, hg, quilt, stgit.
+	S: Status, one of the following:
+	   Supported:	Someone is actually paid to look after this.
+	   Maintained:	Someone actually looks after it.
+	   Odd Fixes:	It has a maintainer but they don't have time to do
+			much other than throw the odd patch in. See below.
+	   Orphan:	No current maintainer [but maybe you could take the
+			role as you write your new code].
+	   Obsolete:	Old code. Something tagged obsolete generally means
+			it has been replaced by a better system and you
+			should be using that.
+	F: Files and directories with wildcard patterns.
+	   A trailing slash includes all files and subdirectory files.
+	   F:	drivers/net/	all files in and below drivers/net
+	   F:	drivers/net/*	all files in drivers/net, but not below
+	   F:	*/net/*		all files in "any top level directory"/net
+	   One pattern per line.  Multiple F: lines acceptable.
+	X: Files and directories that are NOT maintained, same rules as F:
+	   Files exclusions are tested before file matches.
+	   Can be useful for excluding a specific subdirectory, for instance:
+	   F:	net/
+	   X:	net/ipv6/
+	   matches all files in and below net excluding net/ipv6/
+	K: Keyword perl extended regex pattern to match content in a
+	   patch or file.  For instance:
+	   K: of_get_profile
+	      matches patches or files that contain "of_get_profile"
+	   K: \b(printk|pr_(info|err))\b
+	      matches patches or files that contain one or more of the words
+	      printk, pr_info or pr_err
+	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
+
+
+General Project Architects
+--------------------------
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+
+xlators:
+--------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
+Automatic File Replication (AFR)
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/cluster/afr/
+
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
+
+BitRot
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
+S: Maintained
+F: xlators/features/bit-rot/
+
+Changelog
+M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/changelog/
+
+Distributed Hashing Table (DHT)
+P: Susant Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/cluster/dht/
+
+Erasure Coding
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
+S: Maintained
+F: xlators/cluster/ec/
+
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
+FUSE Bridge
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/mount/
+
+Index
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/features/index/
+
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
+IO threads
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
+S: Maintained
+F: xlators/performance/io-threads/
+
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
+Locks
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/locks/
+
+Marker
+M: Kotresh HR <khiremat@redhat.com>
+S: Maintained
+F: xlators/features/marker/
+
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/features/meta/
+
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/performance/md-cache/
+
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/performance/nl-cache/
+
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
+F: xlators/nfs/server/
+
+Open-behind
+S: Maintained
+F: xlators/performance/open-behind/
+
+Posix:
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
+S: Maintained
+F: xlators/storage/posix/
+
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
+Quota
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
+S: Maintained
+F: xlators/features/quota/
+
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
+S: Maintained
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
+
+Upcall
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+F: xlators/features/upcall/
+
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
+
+Other bits of code:
+-------------------
+
+Doc
+M: Humble Chirammal <hchiramm@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: doc/
+
+Geo Replication
+M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: geo-replication/
+
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: tools/glusterfind/
+
+libgfapi
+M: Niels de Vos <ndevos@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: api/
+
+libglusterfs
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+S: Maintained
+F: libglusterfs/
+
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
+M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
+S: Maintained
+F: cli/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
+
+Remote Procedure Call subsystem
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-lib/
+F: rpc/xdr/
+
+Snapshot
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
+S: Maintained
+F: xlators/mgmt/glusterd/src/glusterd-snap*
+F: extras/snap-scheduler.py
+
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
+S: Maintained
+F: rpc/rpc-transport/socket/
+
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: tests/
+
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
+Events APIs
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: events/
+F: libglusterfs/src/events*
+F: libglusterfs/src/eventtypes*
+F: extras/systemd/glustereventsd*
+
+Distribution Specific:
+----------------------
+Build:
+M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
+S: Maintained
+
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+OpenSuSE
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
+
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
+
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
+S: Maintained
+W: https://launchpad.net/~gluster
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
+T: https://github.com/gluster/glusterfs-debian.git
+
+Related projects
+----------------
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
+S: Maintained
+T: https://github.com/gluster/gluster-block.git
+
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusterfs-coreutils.git
+
+NFS-Ganesha FSAL plugin
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+T: git://github.com/nfs-ganesha/nfs-ganesha.git
+F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
+
+QEMU integration
+M: Niels de Vos <ndevos@redhat.com>
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+S: Maintained
+T: git://git.qemu.org/qemu.git
+F: block/gluster.c
+
+Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
+M: Raghavendra Talur <rtalur@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
+S: Maintained
+T: git://git.samba.org/samba.git
+F: source3/modules/vfs_glusterfs.c
+
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
+Wireshark dissectors
+M: Niels de Vos <ndevos@redhat.com>
+S: Maintained
+W: https://forge.gluster.org/wireshark
+T: http://code.wireshark.org/git/wireshark
+F: epan/dissectors/packet-gluster*
+
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
+Special Thanks
+--------------
+
+GlusterFS would not be possible without the contributions of:
+
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
+M: M. Mohan Kumar <mohan@in.ibm.com>
+M: Shishir Gowda <gowda.shishir@gmail.com>
+M: Brian Foster <bfoster@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
+M: Dennis Schafroth <dennis@schafroth.com>
+M: Harshavardhana <harsha@harshavardhana.net>
+M: Krishnan Parthasarathi
+M: Justin Clift <justin@gluster.org>
+M: Venky Shankar <vshankar@redhat.com>
+M: Shravan Chandrashekar <shravantc99@gmail.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
+M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index 85e0a13f1b6..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,14 +1,55 @@
-EXTRA_DIST = autogen.sh COPYING INSTALL README AUTHORS THANKS NEWS glusterfs.spec
+SOURCES = site.h
 
-SUBDIRS = argp-standalone libglusterfs rpc xlators glusterfsd $(FUSERMOUNT_SUBDIR) doc extras cli
+EXTRA_DIST = autogen.sh \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
+	INSTALL README.md AUTHORS THANKS NEWS \
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
+	run-tests.sh \
+	build-aux/pkg-version \
+	contrib/umountd \
+	$(shell find $(top_srcdir)/tests -type f -print)
 
-CLEANFILES = 
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
+
+pkgconfigdir = @pkgconfigdir@
+pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
+
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
-	find . -name Makefile -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	rm -fr argp-standalone/autom4te.cache
-	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in argp-standalone/configure argp-standalone/depcomp argp-standalone/install-sh argp-standalone/missing
+
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
+dist-hook: gen-VERSION gen-ChangeLog
+	-rm -fr $(distdir)/contrib/umountd/.deps
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
+	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
+	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
+
+.PHONY: gen-VERSION gen-ChangeLog clang-check
+
+clang-check:
+	@$(top_srcdir)/extras/clang-checker.sh
+
+gen-ChangeLog:
+	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
+
+.PHONY : gen-VERSION
+gen-VERSION:
+	if test -d $(top_srcdir)/.git; then             \
+	  cd $(top_srcdir);                             \
+	  ./build-aux/pkg-version --full                \
+	    > $(abs_top_builddir)/$(distdir)/VERSION;   \
+	fi
diff --git a/NEWS b/NEWS
index e69de29bb2d..ad00208d351 100644
--- a/NEWS
+++ b/NEWS
@@ -0,0 +1 @@
+Gluster moves to Gerrit.
diff --git a/README b/README
deleted file mode 100644
index 07e76c94539..00000000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-For more info visit http://www.gluster.org
diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..9d68e033782
--- /dev/null
+++ b/README.md
@@ -0,0 +1,46 @@
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
+
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
+
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
+
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
+
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/THANKS b/THANKS
index ea981666273..d056d00bb72 100644
--- a/THANKS
+++ b/THANKS
@@ -1,3 +1 @@
-
-For all of you, who use the product and help us making it more robust, useful, popular.
-
+For all of you, who use the product and help us make it more robust, useful, and popular.
diff --git a/api/Makefile.am b/api/Makefile.am
new file mode 100644
index 00000000000..f0ad1ee971c
--- /dev/null
+++ b/api/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src examples
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
new file mode 100644
index 00000000000..7112c81d6a7
--- /dev/null
+++ b/api/examples/Makefile.am
@@ -0,0 +1,6 @@
+# The bits needed for glfsxmp
+EXTRA_PROGRAMS = glfsxmp
+glfsxmp_SOURCES = glfsxmp.c
+glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
+glfsxmp_LDADD = $(GLFS_LIBS) -lrt
+
diff --git a/api/examples/README b/api/examples/README
new file mode 100644
index 00000000000..4d2b521f779
--- /dev/null
+++ b/api/examples/README
@@ -0,0 +1,36 @@
+This is an example application which uses libgfapi. It is
+a complete autotools based build system which demonstrates the
+required changes in configure.ac, Makefile.am etc to successfuly
+detect for and build an application against libgfapi.
+
+There are two approaches to building a libgfapi based application:
+
+1. In the presence of pkg-config in your build system.
+This is the recommended approach which is also used in this example.
+For this approach to work, you need to build glusterfs by passing
+--pkgconfigdir=/usr/lib64/pkgconfig (or the appropriate directory)
+in your distro. This already happens if you build RPMs with the
+glusterfs.spec provided in glusterfs.git. You will also need to
+install glusterfs-api RPM.
+
+2. In the absence of pkg-config in your build system.
+Make sure your LDFLAGS includes -L/path/to/lib where libgfapi.so is
+installed and -I/path/to/include/glusterfs where the 'api' directory
+containing the headers are available.
+
+glfsxmp.c
+=========
+
+glfsxmp.c is an example application which uses libgfapi
+
+Compilation Steps For glfsxmp.c
+===============================
+
+1. $./autogen.sh
+2. $./configure
+
+Note: Before running ./configure , as mentioned above, you need to
+      take care of #1 or #2 i.e. pkg-config path or LDFLAGS and
+      -I/<path> with correct values.
+
+3. $make glfsxmp
diff --git a/api/examples/autogen.sh b/api/examples/autogen.sh
new file mode 100755
index 00000000000..1fee6be11bb
--- /dev/null
+++ b/api/examples/autogen.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+aclocal
+autoconf
+automake --foreign
diff --git a/api/examples/configure.ac b/api/examples/configure.ac
new file mode 100644
index 00000000000..b80177a4e87
--- /dev/null
+++ b/api/examples/configure.ac
@@ -0,0 +1,12 @@
+
+AC_INIT([glfs-test],[0.1],[gluster-devel@nongu.org])
+
+AM_INIT_AUTOMAKE
+
+AC_CONFIG_FILES([Makefile])
+
+AC_PROG_CC
+
+PKG_CHECK_MODULES([GLFS], [glusterfs-api >= 3])
+
+AC_OUTPUT
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
new file mode 100755
index 00000000000..3b2c8ab5a15
--- /dev/null
+++ b/api/examples/getvolfile.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import ctypes
+import ctypes.util
+
+api = ctypes.CDLL("libgfapi.so")
+api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
+                                 ctypes.c_void_p,
+                                 ctypes.c_ulong]
+api.glfs_get_volfile.restype = ctypes.c_long
+
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
+
+if __name__ == "__main__":
+    import sys
+
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
+
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
new file mode 100644
index 00000000000..a55616ef739
--- /dev/null
+++ b/api/examples/glfsxmp.c
@@ -0,0 +1,1811 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
+#include <string.h>
+#include <time.h>
+
+#define TEST_STR_LEN 2048
+
+int
+test_dirops(glfs_t *fs)
+{
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
+}
+
+int
+test_xattr(glfs_t *fs)
+{
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
+}
+
+int
+test_chdir(glfs_t *fs)
+{
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
+}
+
+#ifdef DEBUG
+static void
+peek_stat(struct stat *sb)
+{
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
+}
+
+static void
+peek_handle(unsigned char *glid)
+{
+    int i;
+
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
+}
+#else  /* DEBUG */
+static void
+peek_stat(struct stat *sb)
+{
+    return;
+}
+
+static void
+peek_handle(unsigned char *id)
+{
+    return;
+}
+#endif /* DEBUG */
+
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
+
+void
+test_h_unlink(void)
+{
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
+
+out:
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
+}
+
+void
+test_h_getsetattrs(void)
+{
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
+out:
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
+}
+
+void
+test_h_truncate(void)
+{
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
+out:
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
+}
+
+void
+test_h_links(void)
+{
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
+
+out:
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
+}
+
+void
+test_h_rename(void)
+{
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
+
+out:
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
+}
+
+void
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
+{
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
+}
+
+#define MAX_FILES_CREATE 10
+#define MAXPATHNAME 512
+void
+test_h_performance(void)
+{
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
+        }
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+out:
+    return;
+}
+
+int
+test_handleops(int argc, char *argv[])
+{
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
+
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
+}
+
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
+
+int
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
+{
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
+
+    char *filename = "/filename2";
+    int ret;
+
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
+
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    glfs_lseek(fd2, 0, SEEK_SET);
+
+    ret = glfs_read(fd2, readbuf, 32, 0);
+
+    printf("read %d, %s", ret, readbuf);
+
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
+
+    ret = glfs_access(fs, filename, R_OK);
+
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
+
+    glfs_close(fd);
+    glfs_close(fd2);
+
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
+
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
+
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
+}
+int
+main(int argc, char *argv[])
+{
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
+
+    char *filename = "/filename2";
+
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
+
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
+
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
+
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
+
+    ret = glfs_init(fs);
+
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
+
+    if (ret)
+        goto out;
+
+    sleep(2);
+
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
+
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
+
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
+
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
+
+    ret = glfs_init(fs2);
+
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
+
+    test_metadata_ops(fs, fs2);
+
+    test_dirops(fs);
+
+    test_xattr(fs);
+
+    test_chdir(fs);
+
+    test_handleops(argc, argv);
+    // done
+
+    /* Test some extra apis */
+    test_write_apis(fs);
+
+    glfs_statvfs(fs, "/", &sfs);
+
+    glfs_fini(fs);
+    glfs_fini(fs2);
+
+    ret = 0;
+out:
+    return ret;
+}
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
new file mode 100644
index 00000000000..7f9a7d17b35
--- /dev/null
+++ b/api/src/Makefile.am
@@ -0,0 +1,43 @@
+lib_LTLIBRARIES = libgfapi.la
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h gfapi-messages.h
+libgfapi_HEADERS = glfs.h glfs-handles.h
+libgfapidir = $(includedir)/glusterfs/api
+
+EXTRA_DIST = gfapi.map gfapi.aliases
+
+libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
+	glfs-handleops.c
+libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
+
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
+
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/rpc-lib/src \
+        -I$(top_srcdir)/rpc/xdr/src \
+        -I$(top_builddir)/rpc/xdr/src \
+        -DDATADIR=\"$(localstatedir)\" \
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+xlator_LTLIBRARIES = api.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
+# workaround for broken parallel install support in automake with LTLIBRARIES
+# http://debbugs.gnu.org/cgi/bugreport.cgi?bug=7328
+install_xlatorLTLIBRARIES = install-xlatorLTLIBRARIES
+$(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
+
+api_la_SOURCES = glfs-master.c
+api_la_DEPENDENCIES = libgfapi.la
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
+api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
+api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/api/src/libgfapi.la
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
new file mode 100644
index 00000000000..b6ec95f9311
--- /dev/null
+++ b/api/src/README.Symbol_Versions
@@ -0,0 +1,3 @@
+
+See ../../doc/developer-guide/gfapi-symbol-versions.md
+
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
new file mode 100644
index 00000000000..b9223940416
--- /dev/null
+++ b/api/src/gfapi-messages.h
@@ -0,0 +1,147 @@
+/*
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ *   */
+
+#ifndef _GFAPI_MESSAGES_H__
+#define _GFAPI_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
+
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
+
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
+#endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
new file mode 100644
index 00000000000..bc639e6b99f
--- /dev/null
+++ b/api/src/gfapi.aliases
@@ -0,0 +1,201 @@
+
+_priv_glfs_loc_touchup _glfs_loc_touchup$GFAPI_PRIVATE_3.4.0
+_priv_glfs_active_subvol _glfs_active_subvol$GFAPI_PRIVATE_3.4.0
+_priv_glfs_subvol_done _glfs_subvol_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_init_done _glfs_init_done$GFAPI_PRIVATE_3.4.0
+_priv_glfs_resolve_at _glfs_resolve_at$GFAPI_PRIVATE_3.4.0
+
+_pub_glfs_new _glfs_new$GFAPI_3.4.0
+_pub_glfs_set_volfile _glfs_set_volfile$GFAPI_3.4.0
+_pub_glfs_set_volfile_server _glfs_set_volfile_server$GFAPI_3.4.0
+_pub_glfs_set_logging _glfs_set_logging$GFAPI_3.4.0
+_pub_glfs_init _glfs_init$GFAPI_3.4.0
+_pub_glfs_fini _glfs_fini$GFAPI_3.4.0
+_pub_glfs_open _glfs_open$GFAPI_3.4.0
+_pub_glfs_creat _glfs_creat$GFAPI_3.4.0
+_pub_glfs_close _glfs_close$GFAPI_3.4.0
+_pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
+_pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
+_pub_glfs_read _glfs_read$GFAPI_3.4.0
+_pub_glfs_write _glfs_write$GFAPI_3.4.0
+_pub_glfs_readv _glfs_readv$GFAPI_3.4.0
+_pub_glfs_writev _glfs_writev$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
+_pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
+_pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
+_pub_glfs_stat _glfs_stat$GFAPI_3.4.0
+_pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_access _glfs_access$GFAPI_3.4.0
+_pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
+_pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
+_pub_glfs_mknod _glfs_mknod$GFAPI_3.4.0
+_pub_glfs_mkdir _glfs_mkdir$GFAPI_3.4.0
+_pub_glfs_unlink _glfs_unlink$GFAPI_3.4.0
+_pub_glfs_rmdir _glfs_rmdir$GFAPI_3.4.0
+_pub_glfs_rename _glfs_rename$GFAPI_3.4.0
+_pub_glfs_link _glfs_link$GFAPI_3.4.0
+_pub_glfs_opendir _glfs_opendir$GFAPI_3.4.0
+_pub_glfs_readdir_r _glfs_readdir_r$GFAPI_3.4.0
+_pub_glfs_readdirplus_r _glfs_readdirplus_r$GFAPI_3.4.0
+_pub_glfs_telldir _glfs_telldir$GFAPI_3.4.0
+_pub_glfs_seekdir _glfs_seekdir$GFAPI_3.4.0
+_pub_glfs_closedir _glfs_closedir$GFAPI_3.4.0
+_pub_glfs_statvfs _glfs_statvfs$GFAPI_3.4.0
+_pub_glfs_chmod _glfs_chmod$GFAPI_3.4.0
+_pub_glfs_fchmod _glfs_fchmod$GFAPI_3.4.0
+_pub_glfs_chown _glfs_chown$GFAPI_3.4.0
+_pub_glfs_lchown _glfs_lchown$GFAPI_3.4.0
+_pub_glfs_fchown _glfs_fchown$GFAPI_3.4.0
+_pub_glfs_utimens _glfs_utimens$GFAPI_3.4.0
+_pub_glfs_lutimens _glfs_lutimens$GFAPI_3.4.0
+_pub_glfs_futimens _glfs_futimens$GFAPI_3.4.0
+_pub_glfs_getxattr _glfs_getxattr$GFAPI_3.4.0
+_pub_glfs_lgetxattr _glfs_lgetxattr$GFAPI_3.4.0
+_pub_glfs_fgetxattr _glfs_fgetxattr$GFAPI_3.4.0
+_pub_glfs_listxattr _glfs_listxattr$GFAPI_3.4.0
+_pub_glfs_llistxattr _glfs_llistxattr$GFAPI_3.4.0
+_pub_glfs_flistxattr _glfs_flistxattr$GFAPI_3.4.0
+_pub_glfs_setxattr _glfs_setxattr$GFAPI_3.4.0
+_pub_glfs_lsetxattr _glfs_lsetxattr$GFAPI_3.4.0
+_pub_glfs_fsetxattr _glfs_fsetxattr$GFAPI_3.4.0
+_pub_glfs_removexattr _glfs_removexattr$GFAPI_3.4.0
+_pub_glfs_lremovexattr _glfs_lremovexattr$GFAPI_3.4.0
+_pub_glfs_fremovexattr _glfs_fremovexattr$GFAPI_3.4.0
+_pub_glfs_getcwd _glfs_getcwd$GFAPI_3.4.0
+_pub_glfs_chdir _glfs_chdir$GFAPI_3.4.0
+_pub_glfs_fchdir _glfs_fchdir$GFAPI_3.4.0
+_pub_glfs_realpath34 _glfs_realpath$GFAPI_3.4.0
+_pub_glfs_posix_lock _glfs_posix_lock$GFAPI_3.4.0
+_pub_glfs_dup _glfs_dup$GFAPI_3.4.0
+
+_pub_glfs_setfsuid _glfs_setfsuid$GFAPI_3.4.2
+_pub_glfs_setfsgid _glfs_setfsgid$GFAPI_3.4.2
+_pub_glfs_setfsgroups _glfs_setfsgroups$GFAPI_3.4.2
+_pub_glfs_h_lookupat34 _glfs_h_lookupat$GFAPI_3.4.2
+_pub_glfs_h_creat _glfs_h_creat$GFAPI_3.4.2
+_pub_glfs_h_mkdir _glfs_h_mkdir$GFAPI_3.4.2
+_pub_glfs_h_mknod _glfs_h_mknod$GFAPI_3.4.2
+_pub_glfs_h_symlink _glfs_h_symlink$GFAPI_3.4.2
+_pub_glfs_h_unlink _glfs_h_unlink$GFAPI_3.4.2
+_pub_glfs_h_close _glfs_h_close$GFAPI_3.4.2
+_pub_glfs_h_truncate _glfs_h_truncate$GFAPI_3.4.2
+_pub_glfs_h_stat _glfs_h_stat$GFAPI_3.4.2
+_pub_glfs_h_getattrs _glfs_h_getattrs$GFAPI_3.4.2
+_pub_glfs_h_setattrs _glfs_h_setattrs$GFAPI_3.4.2
+_pub_glfs_h_readlink _glfs_h_readlink$GFAPI_3.4.2
+_pub_glfs_h_link _glfs_h_link$GFAPI_3.4.2
+_pub_glfs_h_rename _glfs_h_rename$GFAPI_3.4.2
+_pub_glfs_h_extract_handle _glfs_h_extract_handle$GFAPI_3.4.2
+_pub_glfs_h_create_from_handle _glfs_h_create_from_handle$GFAPI_3.4.2
+_pub_glfs_h_opendir _glfs_h_opendir$GFAPI_3.4.2
+_pub_glfs_h_open _glfs_h_open$GFAPI_3.4.2
+
+_pub_glfs_get_volumeid _glfs_get_volumeid$GFAPI_3.5.0
+_pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
+_pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
+_pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
+_pub_glfs_discard _glfs_discard$GFAPI_3.5.0
+_pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
+_pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
+
+_pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
+_pub_glfs_h_getxattrs _glfs_h_getxattrs$GFAPI_3.5.1
+_pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
+
+_pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
+_pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
+
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
+_pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
+_pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
+_pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
+_pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
+
+_priv_glfs_free_from_ctx _glfs_free_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_new_from_ctx _glfs_new_from_ctx$GFAPI_PRIVATE_3.7.0
+_priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
+_priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
+
+_pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
+
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
+_pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
+_pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_expire _glfs_upcall_inode_get_expire$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
+_pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
+
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
new file mode 100644
index 00000000000..228ac47c084
--- /dev/null
+++ b/api/src/gfapi.map
@@ -0,0 +1,283 @@
+
+GFAPI_PRIVATE_3.4.0 {
+	global:
+		glfs_loc_touchup;
+		glfs_active_subvol;
+		glfs_subvol_done;
+		glfs_init_done;
+		glfs_resolve_at;
+	local: *;
+};
+
+GFAPI_3.4.0 {
+	global:
+		glfs_new;
+		glfs_set_volfile;
+		glfs_set_volfile_server;
+		glfs_set_logging;
+		glfs_init;
+		glfs_fini;
+		glfs_open;
+		glfs_creat;
+		glfs_close;
+		glfs_from_glfd;
+		glfs_set_xlator_option;
+		glfs_read;
+		glfs_write;
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv;
+		glfs_writev;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv;
+		glfs_pwritev;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_lseek;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_lstat;
+		glfs_stat;
+		glfs_fstat;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_access;
+		glfs_symlink;
+		glfs_readlink;
+		glfs_mknod;
+		glfs_mkdir;
+		glfs_unlink;
+		glfs_rmdir;
+		glfs_rename;
+		glfs_link;
+		glfs_opendir;
+		glfs_readdir_r;
+		glfs_readdirplus_r;
+		glfs_telldir;
+		glfs_seekdir;
+		glfs_closedir;
+		glfs_statvfs;
+		glfs_chmod;
+		glfs_fchmod;
+		glfs_chown;
+		glfs_lchown;
+		glfs_fchown;
+		glfs_utimens;
+		glfs_lutimens;
+		glfs_futimens;
+		glfs_getxattr;
+		glfs_lgetxattr;
+		glfs_fgetxattr;
+		glfs_listxattr;
+		glfs_llistxattr;
+		glfs_flistxattr;
+		glfs_setxattr;
+		glfs_lsetxattr;
+		glfs_fsetxattr;
+		glfs_removexattr;
+		glfs_lremovexattr;
+		glfs_fremovexattr;
+		glfs_getcwd;
+		glfs_chdir;
+		glfs_fchdir;
+		glfs_realpath;
+		glfs_posix_lock;
+		glfs_dup;
+} GFAPI_PRIVATE_3.4.0;
+
+GFAPI_3.4.2 {
+	global:
+		glfs_setfsuid;
+		glfs_setfsgid;
+		glfs_setfsgroups;
+		glfs_h_creat;
+		glfs_h_mkdir;
+		glfs_h_mknod;
+		glfs_h_symlink;
+		glfs_h_unlink;
+		glfs_h_close;
+		glfs_h_truncate;
+		glfs_h_stat;
+		glfs_h_getattrs;
+		glfs_h_setattrs;
+		glfs_h_readlink;
+		glfs_h_link;
+		glfs_h_rename;
+		glfs_h_extract_handle;
+		glfs_h_create_from_handle;
+		glfs_h_opendir;
+		glfs_h_open;
+		glfs_h_lookupat;
+} GFAPI_3.4.0;
+
+GFAPI_3.5.0 {
+	global:
+		glfs_get_volumeid;
+		glfs_readdir;
+		glfs_readdirplus;
+		glfs_fallocate;
+		glfs_discard;
+		glfs_discard_async;
+		glfs_zerofill;
+		glfs_zerofill_async;
+		glfs_caller_specific_init;
+		glfs_h_setxattrs;
+} GFAPI_3.4.2;
+
+GFAPI_3.5.1 {
+	global:
+		glfs_unset_volfile_server;
+		glfs_h_getxattrs;
+		glfs_h_removexattrs;
+} GFAPI_3.5.0;
+
+GFAPI_3.6.0 {
+	global:
+		glfs_get_volfile;
+		glfs_h_access;
+} GFAPI_3.5.1;
+
+GFAPI_3.7.0 {
+	global:
+		glfs_h_poll_upcall;
+		glfs_h_acl_set;
+		glfs_h_acl_get;
+		glfs_h_statfs;
+		glfs_h_anonymous_read;
+		glfs_h_anonymous_write;
+} GFAPI_3.6.0;
+
+GFAPI_PRIVATE_3.7.0 {
+	global:
+		glfs_free_from_ctx;
+		glfs_new_from_ctx;
+		glfs_resolve;
+		glfs_process_upcall_event;
+} GFAPI_3.7.0;
+
+GFAPI_3.7.4 {
+	global:
+		glfs_h_lookupat;
+} GFAPI_PRIVATE_3.7.0;
+
+GFAPI_3.7.15 {
+	global:
+		glfs_truncate;
+} GFAPI_3.7.4;
+
+GFAPI_3.7.16 {
+	global:
+		glfs_free;
+		glfs_upcall_get_fs;
+		glfs_upcall_get_reason;
+		glfs_upcall_get_event;
+		glfs_upcall_inode_get_object;
+		glfs_upcall_inode_get_flags;
+		glfs_upcall_inode_get_stat;
+		glfs_upcall_inode_get_expire;
+		glfs_upcall_inode_get_pobject;
+		glfs_upcall_inode_get_pstat;
+		glfs_upcall_inode_get_oldpobject;
+		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
+} GFAPI_3.7.15;
+
+GFAPI_3.7.17 {
+	global:
+		glfs_realpath;
+} GFAPI_3.7.16;
+
+GFAPI_3.10.0 {
+	global:
+		glfs_sysrq;
+} GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
new file mode 100644
index 00000000000..6aa3c5602d1
--- /dev/null
+++ b/api/src/glfs-fops.c
@@ -0,0 +1,6445 @@
+
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* for SEEK_HOLE and SEEK_DATA */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <unistd.h>
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include <glusterfs/syncop.h>
+#include "glfs.h"
+#include "gfapi-messages.h"
+#include <glusterfs/compat-errno.h>
+#include <limits.h>
+#include "glusterfs3.h"
+#include <glusterfs/iatt.h>
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
+
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
+
+#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
+
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
+
+/*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
+int
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
+{
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
+
+    GF_REF_PUT(glfd);
+
+    return 0;
+}
+
+/* This function is useful for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
+{
+    gf_boolean_t ret = _gf_false;
+
+    LOCK(&glfd->lock);
+    {
+        if (glfd->state != GLFD_CLOSE)
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
+
+    return ret;
+}
+
+void
+glfd_set_state_bind(struct glfs_fd *glfd)
+{
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
+
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
+
+    return;
+}
+
+/*
+ * This routine is called when an upcall event of type
+ * 'GF_UPCALL_CACHE_INVALIDATION' is received.
+ * It makes a copy of the contents of the upcall cache-invalidation
+ * data received into an entry which is stored in the upcall list
+ * maintained by gfapi.
+ */
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
+{
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
+
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
+
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
+
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    to_up_data->data = ca_data;
+
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
+{
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
+
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
+
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
+
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    to_up_data->data = ca_data;
+
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
+
+    ret = 0;
+out:
+    return ret;
+}
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
+
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
+}
+
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
+{
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
+{
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
+}
+
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+}
+
+int
+glfs_loc_unlink(loc_t *loc)
+{
+    inode_unlink(loc->inode, loc->parent, loc->name);
+
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
+
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
+struct glfs_fd *
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
+int
+pub_glfs_close(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    fs = glfd->fs;
+
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
+int
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
+int
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
+int
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
+struct glfs_fd *
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+#ifdef HAVE_SEEK_HOLE
+static int
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
+
+    switch (whence) {
+        case SEEK_DATA:
+            what = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            what = GF_SEEK_HOLE;
+            break;
+        default:
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
+
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret != -1)
+        glfd->offset = off;
+
+done:
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+out:
+    return ret;
+}
+#endif
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
+off_t
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
+#ifdef HAVE_SEEK_HOLE
+        case SEEK_DATA:
+        case SEEK_HOLE:
+            ret = glfs_seek(glfd, offset, whence);
+            break;
+#endif
+        default:
+            errno = EINVAL;
+    }
+
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
+
+    if (ret != -1)
+        off = glfd->offset;
+
+    return off;
+
+invalid_fs:
+    return -1;
+}
+
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
+ssize_t
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
+{
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
+ssize_t
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
+ssize_t
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
+ssize_t
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
+ssize_t
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
+
+    return ret;
+}
+
+struct glfs_io {
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
+};
+
+static int
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
+{
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
+
+out:
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
+        }
+
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
+
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
+err:
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
+inval:
+    return ret;
+}
+
+static int
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
+
+    return 0;
+}
+
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
+
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+    return ret;
+
+invalid_fs:
+    return -1;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
+int
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
+int
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
+int
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
+int
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
+int
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
+int
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = buf;
+    iov.iov_len = count;
+
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
+int
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
+int
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
+}
+
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
+ssize_t
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
+
+out:
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd_in->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
+ssize_t
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
+{
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
+ssize_t
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
+ssize_t
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
+ssize_t
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
+ssize_t
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
+
+    return ret;
+}
+
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
+
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
+
+    return 0;
+}
+
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
+int
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
+int
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
+int
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
+int
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
+int
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
+int
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
+int
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
+}
+
+static int
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
+int
+pub_glfs_fsync34(struct glfs_fd *glfd)
+{
+    return glfs_fsync_common(glfd, NULL, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
+int
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
+{
+    return glfs_fsync_common(glfd, prestat, poststat);
+}
+
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
+
+    return 0;
+}
+
+static int
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
+
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
+int
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
+int
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
+int
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
+{
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
+int
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
+int
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
+int
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
+{
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
+int
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
+
+    return 0;
+}
+
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
+
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
+int
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
+int
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
+int
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
+int
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
+int
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
+int
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
+out:
+    loc_wipe(&loc);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
+int
+pub_glfs_unlink(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
+int
+pub_glfs_rmdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
+int
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
+
+    if (ret)
+        goto out;
+retrynew:
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+    }
+
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
+int
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
+
+    if (ret)
+        goto out;
+retrynew:
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
+int
+pub_glfs_closedir(struct glfs_fd *glfd)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
+
+    glfs_mark_glfd_for_deletion(glfd);
+
+    __GLFS_EXIT_FS;
+
+    ret = 0;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
+long
+pub_glfs_telldir(struct glfs_fd *fd)
+{
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    return fd->offset;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
+void
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
+{
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
+
+    if (fd->offset == offset)
+        return;
+
+    fd->offset = offset;
+    fd->next = NULL;
+
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
+
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
+
+static int
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
+
+    return 0;
+}
+
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
+int
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
+int
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
+
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
+
+    return 0;
+}
+
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
+{
+    dirent->d_ino = gf_dirent->d_ino;
+
+#ifdef _DIRENT_HAVE_D_OFF
+    dirent->d_off = gf_dirent->d_off;
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+    dirent->d_type = gf_dirent->d_type;
+#endif
+
+#ifdef _DIRENT_HAVE_D_NAMLEN
+    dirent->d_namlen = strlen(gf_dirent->d_name);
+#endif
+
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
+}
+
+int
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
+{
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
+                }
+            }
+
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
+
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
+
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
+
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
+
+    gf_dirent_free(&old);
+out:
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    return ret;
+}
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus)
+{
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
+
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
+
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
+
+    glfd->offset = entry->d_off;
+
+    return entry;
+}
+
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
+{
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
+
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
+        }
+
+        glfd->readdirbuf = buf;
+    }
+unlock:
+    UNLOCK(&glfd->fd->lock);
+
+    return buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
+int
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
+{
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    errno = 0;
+
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
+
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
+
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
+
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
+
+    return ret;
+
+invalid_fs:
+    return -1;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
+int
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
+{
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
+struct dirent *
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
+{
+    struct dirent *res = NULL;
+    int ret = -1;
+
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
+
+    return res;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
+struct dirent *
+pub_glfs_readdir(struct glfs_fd *glfd)
+{
+    return pub_glfs_readdirplus(glfd, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
+int
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
+int
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
+{
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
+int
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
+{
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
+int
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
+
+    ret = glfs_setattr(fs, path, &stat, 1);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
+int
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
+
+    ret = glfs_fsetattr(glfd, &stat);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
+int
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
+
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
+
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
+int
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
+
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
+
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
+int
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
+{
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
+
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
+
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
+int
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
+{
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
+
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
+
+    ret = glfs_setattr(fs, path, &stat, 1);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
+int
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
+{
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
+
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
+
+    ret = glfs_setattr(fs, path, &stat, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
+int
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
+{
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
+
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
+
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
+
+    ret = glfs_fsetattr(glfd, &stat);
+
+    return ret;
+}
+
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
+out:
+    return ret;
+}
+
+ssize_t
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
+out:
+    loc_wipe(&loc);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
+ssize_t
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
+{
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
+ssize_t
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
+{
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
+ssize_t
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
+{
+    int ret = -1;
+
+    if (!xattr)
+        goto out;
+
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
+
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
+
+out:
+    return ret;
+}
+
+ssize_t
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
+out:
+    loc_wipe(&loc);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
+ssize_t
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
+{
+    return glfs_listxattr_common(fs, path, value, size, 1);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
+ssize_t
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
+{
+    return glfs_listxattr_common(fs, path, value, size, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
+ssize_t
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
+int
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
+                   const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
+int
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
+int
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 1);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
+int
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
+int
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
+int
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
+int
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
+int
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
+int
+pub_glfs_chdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(fs, loc.inode);
+
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
+int
+pub_glfs_fchdir(struct glfs_fd *glfd)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
+
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
+{
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
+
+out:
+    loc_wipe(&loc);
+
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return retpath;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    inode = glfs_cwd_get(fs);
+
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
+out:
+    GF_FREE(path);
+
+    if (inode)
+        inode_unref(inode);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    if (ret < 0)
+        return NULL;
+
+    return buf;
+}
+
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
+}
+
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
+
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+
+        if (ret)
+            goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
+
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
+        }
+    }
+
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
+int
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+    return glfs_lock_common(glfd, cmd, flock, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
+int
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
+
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
+
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
+
+    glfd->lk_owner.len = len;
+
+    memcpy(glfd->lk_owner.data, data, len);
+
+    ret = 0;
+out:
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return dupfd;
+}
+
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    upcall_entry *u_list = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
+
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&u_list->upcall_list);
+
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
+
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    ret = 0;
+
+out:
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
+
+static void
+glfs_free_upcall_lease(void *to_free)
+{
+    struct glfs_upcall_lease *arg = to_free;
+
+    if (!arg)
+        return;
+
+    if (arg->object)
+        glfs_h_close(arg->object);
+
+    GF_FREE(arg);
+}
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
+
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
+        }
+    }
+
+out:
+    return ret;
+}
+
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
+
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
+
+    up_lease_arg->lease_type = recall_lease->lease_type;
+
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
+
+    ret = 0;
+
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
+}
+
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
+
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
+
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+
+    (void)upcall_syncop_args_free(args);
+
+    return 0;
+}
+
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
+
+out:
+    return ret;
+}
+
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
+
+    if (!src)
+        goto out;
+
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
+
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
+
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
+out:
+    return NULL;
+}
+
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
+{
+    struct gf_upcall_recall_lease *dst = NULL;
+
+    if (!src)
+        goto out;
+
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
+
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
+
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
+out:
+    return NULL;
+}
+
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
+
+    return NULL;
+}
+
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
+
+    args = upcall_syncop_args_init(fs, upcall_data);
+
+    if (!args)
+        goto out;
+
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
+
+out:
+    return;
+}
+
+/*
+ * This routine is called in case of any notification received
+ * from the server. All the upcall events are queued up in a list
+ * to be read by the applications.
+ *
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
+ * applications the way it desires for each event queued (for eg.,
+ * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
+ */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
+void
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    if (!data)
+        goto out;
+
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
+
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
+
+out:
+
+    if (fd)
+        fd_unref(fd);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static void
+glfs_release_xreaddirp_stat(void *ptr)
+{
+    struct glfs_xreaddirp_stat *to_free = ptr;
+
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
+
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
+int
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
+{
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
+
+    errno = 0;
+
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf)
+        goto out;
+
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
+
+    if (!xstat)
+        goto out;
+
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
+
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
+
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
+
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
+
+out:
+    GF_REF_PUT(glfd);
+
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
+
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
+
+    __GLFS_EXIT_FS;
+
+    return ret;
+
+invalid_fs:
+    return -1;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
+
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
+
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
+{
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
+                ret = -1;
+                errno = EINVAL;
+                goto out;
+            }
+            break;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
+
+out:
+
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
new file mode 100644
index 00000000000..53c2ee896f9
--- /dev/null
+++ b/api/src/glfs-handleops.c
@@ -0,0 +1,2655 @@
+/*
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include <glusterfs/syncop.h>
+#include "glfs.h"
+#include "glfs-handles.h"
+#include "gfapi-messages.h"
+
+int
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
+
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
+{
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
+        return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
+struct glfs_object *
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
+        }
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
+
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
+{
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
+int
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    loc_wipe(&loc);
+
+out:
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
+int
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
+int
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
+
+out:
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
+{
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!is_listxattr) {
+        if (!name || *name == '\0') {
+            errno = EINVAL;
+            return -1;
+        }
+
+        if (strlen(name) > GF_XATTR_NAME_MAX) {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
+int
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
+{
+    int ret = -1;
+    dict_t *xattr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
+
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
+int
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
+int
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
+int
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
+struct glfs_object *
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
+{
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (fd)
+        fd_unref(fd);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
+struct glfs_object *
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
+
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
+struct glfs_object *
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+out:
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
+int
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
+struct glfs_fd *
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return glfd;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
+int
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
+ssize_t
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
+{
+    ssize_t ret = -1;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
+
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+    ret = GFAPI_HANDLE_LENGTH;
+
+out:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
+struct glfs_object *
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
+
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
+
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
+        } else {
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
+
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
+
+found:
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+out:
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
+int
+pub_glfs_h_close(struct glfs_object *object)
+{
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
+
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
+int
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
+struct glfs_object *
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
+        }
+
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
+
+out:
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
+
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
+int
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (linkval)
+        GF_FREE(linkval);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
+int
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (inode)
+        inode_unref(inode);
+
+    if (pinode)
+        inode_unref(pinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
+int
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
+
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
+
+    if (oldpinode)
+        inode_unref(oldpinode);
+
+    if (newpinode)
+        inode_unref(newpinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+/*
+ * Given a handle/gfid, find if the corresponding inode is present in
+ * the inode table. If yes create and return the corresponding glfs_object.
+ */
+struct glfs_object *
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
+{
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
+
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
+
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+out:
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return object;
+}
+
+static void
+glfs_free_upcall_inode(void *to_free)
+{
+    struct glfs_upcall_inode *arg = to_free;
+
+    if (!arg)
+        return;
+
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
+
+    GF_FREE(arg);
+}
+
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
+
+    ret = 0;
+
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
+}
+
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
+
+    if (to_free->event)
+        to_free->free_event(to_free->event);
+}
+
+/*
+ * This API is used to poll for upcall events stored in the upcall list.
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
+ * will be mapped appropriately into 'glfs_upcall' along with the handle object
+ * to be passed to NFS-Ganesha.
+ *
+ * On success, applications need to check if up_arg is not-NULL or errno is not
+ * ENOENT. glfs_upcall_get_reason() can be used to decide what kind of event
+ * has been received.
+ *
+ * Current supported upcall_events:
+ *      GLFS_UPCALL_INODE_INVALIDATE
+ *
+ * After processing the event, applications need to free 'up_arg' by calling
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling before
+ * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
+ * cannot be freed while in this routine.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
+int
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
+{
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+        {
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
+                }
+
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
+
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
+                }
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
+                /* no 'default:' label, to force handling all upcall events */
+                errno = ENOENT;
+                break;
+        }
+
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
+
+    ret = 0;
+
+out:
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    glfs_subvol_done(fs, subvol);
+
+restore:
+    __GLFS_EXIT_FS;
+err:
+    return ret;
+}
+
+static gf_boolean_t log_upcall370 = _gf_true; /* log once */
+
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
+ * structures that are returned to the calling application. This is not
+ * recommended, as the returned structures need to returned correctly (handles
+ * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
+ *
+ * To the best of our knowledge, only NFS-Ganesha uses the upcall events
+ * through gfapi. We keep this backwards compatibility function around so that
+ * applications using the existing implementation do not break.
+ *
+ * WARNING: this function will be removed in the future.
+ */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
+int
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
+{
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
+
+            up_inode = upcall->event;
+
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
+
+            up_arg->event_arg = cb_inode;
+        }
+    }
+
+out:
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
+
+    return ret;
+}
+
+#ifdef HAVE_ACL_LIBACL_H
+#include <glusterfs/glusterfs-acl.h>
+#include <acl/libacl.h>
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
+{
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
+
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
+
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
+
+    acl_free(acl_s);
+
+out:
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
+
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
+
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
+
+    acl = acl_from_text(acl_s);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return acl;
+}
+#else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
+acl_t
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
+int
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
+
+/* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+/* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
+ssize_t
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
+{
+    struct glfs_object *object = NULL;
+
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
+
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
+
+out:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
+
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
+
+    return xstat->object;
+
+out:
+    return NULL;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
+int
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
new file mode 100644
index 00000000000..4d039b9c76b
--- /dev/null
+++ b/api/src/glfs-handles.h
@@ -0,0 +1,355 @@
+/*
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_HANDLES_H
+#define _GLFS_HANDLES_H
+
+#include "glfs.h"
+
+/* GLFS OBJECT BASED OPERATIONS
+ *
+ * The following APIs are introduced to provide an API framework that can work
+ * with gluster objects (files and directories), instead of absolute paths.
+ *
+ * The following API set can be related to the POSIX *at interfaces (like
+ * openat (2)). The intention of these APIs is to be able to operate based
+ * on parent object and looking up or creating child objects within, OR to be
+ * used on the actual object thus looked up or created, and retrieve information
+ * regarding the same.
+ *
+ * The APIs also provide for generating an opaque invariant handle to the
+ * object, that can later be used to lookup the object, instead of the regular
+ * glfs_h_* variants. The APIs that provide this behaviour are,
+ * glfs_h_extract_handle and glfs_h_create_from_handle.
+ *
+ * The object handles can be transitioned to fd based operations as supported
+ * by glfs.h calls, using the glfs_h_open call. This provides a way to move
+ * from objects to fd's akin to moving from path to fd for required operations.
+ *
+ * NOTE: The opaque invariant handle is the GFID of the object in reality, but
+ * maintained as an opaque data value, for potential internal changes to the
+ * same without impacting the caller.
+ *
+ * NOTE: Currently looking up an object can create multiple object handles to
+ * the same, i.e distinct glfs_object *. Hence each such looked up or received
+ * handle from other calls, would need to be closed. In the future, for a given
+ * object these pointers would be the same, and an ease of use API to forget all
+ * instances of this bject would be provided (instead of a per lookup close).
+ * This should not change the APIs in their current form.
+ *
+ */
+
+/* Handle length for object handles returned from glfs_h_extract_handle or
+ * glfs_h_create_from_handle */
+#define GFAPI_HANDLE_LENGTH 16
+
+/* These flags should be in sync to the ones defined in upcall.h */
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+
+/* Portability non glibc c++ build systems */
+#ifndef __THROW
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+__BEGIN_DECLS
+
+/*
+ * Notes:
+ *
+ * The file object handle. One per looked up, created file/directory
+ *
+ * This had been introduced to facilitate gfid/inode based gfapi
+ * - a requirement introduced by nfs-ganesha
+ */
+struct glfs_object;
+typedef struct glfs_object glfs_object_t;
+
+/* Functions for getting details about the glfs_upcall_inode
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
+
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
+
+/* Handle based operations */
+/* Operations that generate handles */
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
+
+/* Operations on the actual objects */
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
+
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
+
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
+
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
+
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
+
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
+
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
+
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
+
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
+
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
+
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
+
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
+
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
+
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
+
+/* Operations enabling opaque invariant handle to object transitions */
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
+
+/* Given a handle, looks up the inode and creates glfs_object.
+ * In addition, if provided 'stat', copies the inode attributes
+ */
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
+
+/* Operations enabling object handles to fd transitions */
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
+
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
+
+int
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
+/*
+  SYNOPSIS
+
+  glfs_h_poll_upcall: Poll for upcall events given a 'glfs' object.
+
+  DESCRIPTION
+
+  This API is used to poll for upcall events stored in the
+  upcall list. Current users of this API is NFS-Ganesha.
+  In case of any event received, it will be mapped appropriately
+  into 'glfs_upcall' along with the handle('glfs_object') to be
+  passed to NFS-Ganesha.
+
+  In case of success, applications need to check the value of
+  cbk->handle to be NON NULL before processing the upcall
+  events.
+
+  PARAMETERS
+
+  @fs: glfs object to poll the upcall events for
+  @cbk: Pointer that will contain an upcall event for use by the application.
+        Application is responsible for free'ing the structure with glfs_free().
+
+  RETURN VALUES
+
+  0   : Success.
+  -1  : Error condition, mostly due to out of memory.
+
+*/
+
+int
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
+
+int
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+
+acl_t
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+
+size_t
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+
+ssize_t
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
+
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
+
+__END_DECLS
+
+#endif /* !_GLFS_HANDLES_H */
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
new file mode 100644
index 00000000000..7cc3b18a104
--- /dev/null
+++ b/api/src/glfs-internal.h
@@ -0,0 +1,756 @@
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_INTERNAL_H
+#define _GLFS_INTERNAL_H
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
+#include "glfs-handles.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
+
+#define GLFS_SYMLINK_MAX_FOLLOW 2048
+
+#define DEFAULT_REVAL_COUNT 1
+
+/*
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * as part of fs inititialization.
+ *
+ * These flag bits are validated in glfs_fini() to destroy all or partially
+ * initialized mutex and conditional variables of glfs object.
+ * If you introduce new pthread mutex or conditional variable in glfs object,
+ * please make sure you have a flag bit intorduced here for proper cleanup
+ * in glfs_fini().
+ *
+ */
+
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
+
+#ifndef GF_DARWIN_HOST_OS
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) /**/
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver) /**/
+#endif
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
+
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
+#define STR(str) #str
+#else
+#ifndef GFAPI_PUBLIC
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
+#endif
+#ifndef GFAPI_PRIVATE
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
+#endif
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
+#endif
+
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+struct glfs;
+
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
+};
+typedef struct _upcall_entry upcall_entry;
+
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
+
+struct glfs {
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
+};
+
+/* This enum is used to maintain the state of glfd. In case of async fops
+ * fd might be closed before the actual fop is complete. Therefore we need
+ * to track whether the fd is closed or not, instead actually closing it.*/
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
+
+struct glfs_fd {
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
+};
+
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+    inode_t *inode;
+    uuid_t gfid;
+};
+
+struct glfs_upcall {
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
+};
+
+struct glfs_upcall_inode {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
+
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
+
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
+};
+
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
+};
+
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
+
+typedef void(glfs_mem_release_t)(void *ptr);
+
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
+};
+
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
+
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
+
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
+
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
+
+static inline void
+__glfs_free(void *free_ptr)
+{
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
+}
+
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
+
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
+
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
+
+/*
+  By default all lock attempts from user context must
+  use glfs_lock() and glfs_unlock(). This allows
+  for a safe implementation of graph migration where
+  we can give up the mutex during syncop calls so
+  that bottom up calls (particularly CHILD_UP notify)
+  can do a mutex_lock() on @glfs without deadlocking
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
+*/
+static inline int
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
+{
+    pthread_mutex_lock(&fs->mutex);
+
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
+
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
+
+    return 0;
+}
+
+static inline void
+glfs_unlock(struct glfs *fs)
+{
+    pthread_mutex_unlock(&fs->mutex);
+}
+
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
+
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
+
+/* Sends RPC call to glusterd to fetch required volume info */
+int
+glfs_get_volume_info(struct glfs *fs);
+
+/*
+  SYNOPSIS
+
+       glfs_new_from_ctx: Creates a virtual mount object by taking a
+       glusterfs_ctx_t object.
+
+  DESCRIPTION
+
+       glfs_new_from_ctx() is not same as glfs_new(). It takes the
+       glusterfs_ctx_t object instead of creating one by glusterfs_ctx_new().
+       Again the usage is restricted to NFS MOUNT over UDP i.e. in
+       glfs_resolve_at() which would take fs object as input but never use
+       (purpose is not to change the ABI of glfs_resolve_at()).
+
+  PARAMETERS
+
+       @ctx: glusterfs_ctx_t object
+
+  RETURN VALUES
+
+       fs     : Pointer to the newly created glfs_t object.
+       NULL   : Otherwise.
+*/
+
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
+
+/*
+  SYNOPSIS
+
+       glfs_free_from_ctx: Free up the memory occupied by glfs_t object
+       created by glfs_new_from_ctx().
+
+  DESCRIPTION
+
+       The glfs_t object allocated by glfs_new_from_ctx() must be released
+       by the caller using this routine. The usage can be found
+       at glfs_fini() or NFS, MOUNT over UDP i.e.
+                        __mnt3udp_get_export_subdir_inode ()
+                                => glfs_resolve_at().
+
+  PARAMETERS
+
+       @fs: The glfs_t object to be deallocated.
+
+  RETURN VALUES
+
+       void
+*/
+
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
+
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
+
+/* Deprecated structures that were passed to client applications, replaced by
+ * accessor functions. Do not use these in new applications, and update older
+ * usage.
+ *
+ * See http://review.gluster.org/14701 for more details.
+ *
+ * WARNING: These structures will be removed in the future.
+ */
+struct glfs_callback_arg {
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
+};
+
+struct glfs_callback_inode_arg {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
+};
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
+
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
+
+void
+unset_fop_attr(dict_t **fop_attr);
+
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
+#endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
new file mode 100644
index 00000000000..100dcc16cc0
--- /dev/null
+++ b/api/src/glfs-master.c
@@ -0,0 +1,183 @@
+/*
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdio.h>
+
+#include <glusterfs/glusterfs.h>
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "gfapi-messages.h"
+
+int
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
+{
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
+unlock:
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
+
+    return ret;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
+        case GF_EVENT_UPCALL:
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
+}
+
+int
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    return 0;
+}
+
+int
+init(xlator_t *this)
+{
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+}
+
+/* place-holder fops */
+int
+glfs_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int
+glfs_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int
+glfs_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+struct xlator_dumpops dumpops;
+
+struct xlator_fops fops;
+
+struct xlator_cbks cbks = {
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
+};
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
new file mode 100644
index 00000000000..bfa325a3ad9
--- /dev/null
+++ b/api/src/glfs-mem-types.h
@@ -0,0 +1,35 @@
+/*
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_MEM_TYPES_H
+#define _GLFS_MEM_TYPES_H
+
+#include <glusterfs/mem-types.h>
+
+#define GF_MEM_TYPE_START (gf_common_mt_end + 1)
+
+enum glfs_mem_types_ {
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
+};
+#endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
new file mode 100644
index 00000000000..7c82b8cd162
--- /dev/null
+++ b/api/src/glfs-mgmt.c
@@ -0,0 +1,1049 @@
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <pthread.h>
+
+#include <glusterfs/glusterfs.h>
+#include "glfs.h"
+#include <glusterfs/dict.h>
+
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "rpc-common-xdr.h"
+
+#include <glusterfs/syncop.h>
+
+#include "glfs-internal.h"
+#include "gfapi-messages.h"
+#include <glusterfs/syscall.h>
+
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
+
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp)
+{
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
+out:
+    if (fp)
+        fclose(fp);
+
+    if (!ctx->active) {
+        ret = -1;
+    }
+
+    return ret;
+}
+
+int
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+
+    this = mydata;
+    fs = this->private;
+
+    glfs_volfile_fetch(fs);
+
+    return 0;
+}
+
+int
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    return 0;
+}
+
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
+
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
+};
+
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
+};
+
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
+};
+
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
+};
+
+int
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
+{
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+
+out:
+    if (iobref)
+        iobref_unref(iobref);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
+}
+
+/*
+ * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
+ */
+int
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
+
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
+
+    if (dict)
+        dict_unref(dict);
+
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
+
+    __wake(args);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
+int
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
+{
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
+
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
+
+done:
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
+
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(volid, fs->vol_uuid, uuid_size);
+
+    __GLFS_EXIT_FS;
+
+    return uuid_size;
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
+
+int
+glfs_get_volume_info(struct glfs *fs)
+{
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
+
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = &args;
+
+    __yawn((&args));
+
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
+
+    __yield((&args));
+
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+
+out:
+    return ret;
+}
+
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
+{
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
+out:
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
+{
+    int ret = -1;
+
+    pthread_mutex_lock(&fs->mutex);
+
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
+
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
+
+    pthread_mutex_unlock(&fs->mutex);
+
+    return ret;
+}
+
+int
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+out:
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
+        }
+    }
+
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
+
+    return 0;
+}
+
+int
+glfs_volfile_fetch(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
+out:
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+static int
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
+{
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
+                }
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
+                }
+
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
+                }
+
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
+out:
+    return 0;
+}
+
+int
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
+{
+    int ret = 0;
+
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
+
+        default:
+            break;
+    }
+
+    return ret;
+}
+
+int
+glfs_mgmt_init(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
+out:
+    if (options)
+        dict_unref(options);
+    return ret;
+}
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
new file mode 100644
index 00000000000..8a393ecb464
--- /dev/null
+++ b/api/src/glfs-resolve.c
@@ -0,0 +1,1199 @@
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <limits.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
+#include "glfs-mem-types.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "gfapi-messages.h"
+#include <glusterfs/inode.h>
+#include "glfs-internal.h"
+
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
+int
+glfs_first_lookup_safe(xlator_t *subvol)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
+
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
+
+    return ret;
+}
+
+int
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
+{
+    int ret = -1;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
+
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return ret;
+}
+
+/**
+ * We have to check if need_lookup flag is set in both old and the new inodes.
+ * If its set in oldinode, then directly go ahead and do an explicit lookup.
+ * But if its not set in the oldinode, then check if the newinode is linked
+ * via readdirp. If so an explicit lookup is needed on the new inode, so that
+ * below xlators can set their respective contexts.
+ */
+inode_t *
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
+}
+
+inode_t *
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
+{
+    inode_t *newinode = NULL;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
+
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return newinode;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
+int
+priv_glfs_loc_touchup(loc_t *loc)
+{
+    int ret = 0;
+
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
+
+    return ret;
+}
+
+int
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
+{
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
+out:
+    loc_wipe(&loc);
+    return ret;
+}
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
+
+    return ret;
+}
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
+
+    return ret;
+}
+
+inode_t *
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
+{
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
+        } else {
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
+        }
+
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
+
+    if (!loc.name)
+        loc.name = component;
+
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
+
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
+        }
+
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        gf_uuid_generate(gfid);
+
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
+
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
+found:
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
+
+    return inode;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
+int
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
+{
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
+
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
+                ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
+        }
+
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
+
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
+{
+    int ret = -1;
+    inode_t *cwd = NULL;
+
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
+                                    follow, reval);
+
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
+
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
+
+out:
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
+int
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
+{
+    int ret = -1;
+
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
+
+    return ret;
+}
+
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
+{
+    int ret = -1;
+
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
+
+    return ret;
+}
+
+int
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
+{
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+out:
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
+}
+
+fd_t *
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+{
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
+out:
+    if (newinode)
+        inode_unref(newinode);
+
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return newfd;
+}
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+{
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
+
+    oldfd = glfd->fd;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
+
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return newfd;
+}
+
+fd_t *
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+    fd_t *fd = NULL;
+
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
+
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
+
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
+
+    return fd;
+}
+
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+    fd_t *fd = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
+
+    return fd;
+}
+
+void
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
+{
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
+
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
+}
+
+/* Note that though it appears that this function executes under fs->mutex,
+ * it is not fully executed under fs->mutex. i.e. there are functions like
+ * __glfs_first_lookup, __glfs_refresh_inode, __glfs_migrate_openfds which
+ * unlocks fs->mutex before sending any network fop, and reacquire fs->mutex
+ * once the fop is complete. Hence the variable read from fs at the start of the
+ * function need not have the same value by the end of the function.
+ */
+xlator_t *
+__glfs_active_subvol(struct glfs *fs)
+{
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
+
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
+
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
+
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
+
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
+
+    return new_subvol;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
+void
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+{
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
+xlator_t *
+priv_glfs_active_subvol(struct glfs *fs)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
+
+        if (subvol)
+            subvol->winds++;
+
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
+
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
+
+    return subvol;
+}
+
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
+{
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
+
+    if (fs->cwd)
+        inode_unref(fs->cwd);
+
+    fs->cwd = inode;
+
+    return 0;
+}
+
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
+{
+    int ret = 0;
+
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
+
+    return ret;
+}
+
+inode_t *
+__glfs_cwd_get(struct glfs *fs)
+{
+    inode_t *cwd = NULL;
+
+    if (!fs->cwd)
+        return NULL;
+
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
+
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
+
+    return cwd;
+}
+
+inode_t *
+glfs_cwd_get(struct glfs *fs)
+{
+    inode_t *cwd = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
+
+    return cwd;
+}
+
+inode_t *
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
+{
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
+
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
+
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
+
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
+
+    return inode;
+}
+
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
+{
+    inode_t *inode = NULL;
+
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
+
+    return inode;
+}
+
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
+{
+    struct glfs_object *object = NULL;
+
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
+
+    /* we hold the reference */
+    loc->inode = NULL;
+
+    *retobject = object;
+
+    return 0;
+}
+
+struct glfs_object *
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
+{
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
+
+out:
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
+}
diff --git a/api/src/glfs.c b/api/src/glfs.c
new file mode 100644
index 00000000000..b4bf1423f6d
--- /dev/null
+++ b/api/src/glfs.c
@@ -0,0 +1,1806 @@
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+  TODO:
+  - set proper pid/lk_owner to call frames (currently buried in syncop)
+  - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
+    reach it via THIS.
+  - update syncop functions to accept/return xdata. ???
+  - protocol/client to reconnect immediately after portmap disconnect.
+  - handle SEEK_END failure in _lseek()
+  - handle umask (per filesystem?)
+  - make itables LRU based
+  - 0-copy for readv/writev
+  - reconcile the open/creat mess
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
+#endif
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
+#include "glfs-mem-types.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
+
+#include "gfapi-messages.h"
+#include "glfs.h"
+#include "glfs-internal.h"
+
+static gf_boolean_t
+vol_assigned(cmd_args_t *args)
+{
+    return args->volfile || args->volfile_server;
+}
+
+static int
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    if (!ctx) {
+        goto err;
+    }
+
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
+err:
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
+}
+
+static int
+create_master(struct glfs *fs)
+{
+    int ret = 0;
+    xlator_t *master = NULL;
+
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
+
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
+
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
+
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
+
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
+
+    fs->ctx->master = master;
+    THIS = master;
+
+    return 0;
+
+err:
+    if (master) {
+        xlator_destroy(master);
+    }
+
+    return -1;
+}
+
+static FILE *
+get_volfp(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
+
+    return specfp;
+}
+
+int
+glfs_volumes_init(struct glfs *fs)
+{
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (!vol_assigned(cmd_args))
+        return -1;
+
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
+
+    fp = get_volfp(fs);
+
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
+
+out:
+    return ret;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
+int
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
+{
+    xlator_cmdline_option_t *option = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
+
+    INIT_LIST_HEAD(&option->cmd_args);
+
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
+
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+
+    __GLFS_EXIT_FS;
+
+    return 0;
+enomem:
+    errno = ENOMEM;
+
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
+
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
+int
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
+{
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
+        }
+    }
+
+out:
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
+int
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
+{
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
+
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
+        } else {
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
+        }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
+
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
+
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
+
+out:
+    if (server_host) {
+        GF_FREE(server_host);
+    }
+
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+/* *
+ * Used to free the arguments allocated by glfs_set_volfile_server()
+ */
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
+out:
+    return;
+}
+
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
+{
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
+
+    if (!&(cmd_args->xlator_options))
+        return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
+int
+pub_glfs_setfsuid(uid_t fsuid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
+int
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
+
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
+}
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
+{
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
+
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
+{
+    char *lease_id = NULL;
+    int ret = -1;
+
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr)
+{
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
+
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
+}
+
+void
+unset_fop_attr(dict_t **fop_attr)
+{
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
+struct glfs *
+pub_glfs_from_glfd(struct glfs_fd *glfd)
+{
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
+
+    return glfd->fs;
+}
+
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
+{
+    if (!glfd)
+        return;
+
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
+
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    GF_FREE(glfd->readdirbuf);
+
+    GF_FREE(glfd);
+}
+
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs)
+{
+    struct glfs_fd *glfd = NULL;
+
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
+
+    glfd->fs = fs;
+
+    INIT_LIST_HEAD(&glfd->openfds);
+
+    GF_REF_INIT(glfd, glfs_fd_destroy);
+
+    return glfd;
+}
+
+void
+glfs_fd_bind(struct glfs_fd *glfd)
+{
+    struct glfs *fs = NULL;
+
+    fs = glfd->fs;
+
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
+}
+
+static void *
+glfs_poller(void *data)
+{
+    struct glfs *fs = NULL;
+
+    fs = data;
+
+    gf_event_dispatch(fs->ctx->event_pool);
+
+    return NULL;
+}
+
+static struct glfs *
+glfs_new_fs(const char *volname)
+{
+    struct glfs *fs = NULL;
+
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
+
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
+
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
+
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
+
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
+
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
+
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
+
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
+
+    return fs;
+
+err:
+    glfs_free_from_ctx(fs);
+    return NULL;
+}
+
+extern xlator_t global_xlator;
+extern glusterfs_ctx_t *global_ctx;
+extern pthread_mutex_t global_ctx_mutex;
+
+static int
+glfs_init_global_ctx()
+{
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
+
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
+
+        gf_log_globals_init(ctx, GF_LOG_NONE);
+
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
+
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&global_ctx_mutex);
+
+    if (ret)
+        FREE(ctx);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
+struct glfs *
+pub_glfs_new(const char *volname)
+{
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
+        }
+        errno = EINVAL;
+        return NULL;
+    }
+
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
+
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
+
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
+
+    /* first globals init, for gf_mem_acct_enable_set () */
+
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
+
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
+
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
+
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
+
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
+        goto out;
+
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
+#endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
+
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
+
+    if (old_THIS)
+        THIS = old_THIS;
+
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
+struct glfs *
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
+{
+    struct glfs *fs = NULL;
+
+    if (!ctx)
+        goto out;
+
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
+
+    fs->ctx = ctx;
+
+out:
+    return fs;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
+void
+priv_glfs_free_from_ctx(struct glfs *fs)
+{
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+
+    if (!fs)
+        return;
+
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
+
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
+
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
+
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
+
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
+
+    FREE(fs->volname);
+
+    FREE(fs);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
+int
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
+{
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &fs->ctx->cmd_args;
+
+    if (vol_assigned(cmd_args))
+        return -1;
+
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
+int
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
+{
+    int ret = -1;
+    char *tmplog = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
+        if (ret)
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
+
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
+
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+int
+glfs_init_wait(struct glfs *fs)
+{
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
+void
+priv_glfs_init_done(struct glfs *fs, int ret)
+{
+    glfs_init_cbk init_cbk;
+
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
+
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (init_cbk)
+        init_cbk(fs, ret);
+out:
+    return;
+}
+
+int
+glfs_init_common(struct glfs *fs)
+{
+    int ret = -1;
+
+    ret = create_master(fs);
+    if (ret)
+        return ret;
+
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
+
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
+
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
+}
+
+int
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
+{
+    int ret = -1;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
+
+    fs->init_cbk = cbk;
+
+    ret = glfs_init_common(fs);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
+int
+pub_glfs_init(struct glfs *fs)
+{
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
+
+    ret = glfs_init_wait(fs);
+out:
+    __GLFS_EXIT_FS;
+
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
+int
+pub_glfs_fini(struct glfs *fs)
+{
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
+        {
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
+        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
+
+    /* leaked frames may exist, we ignore */
+
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
+                }
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
+         */
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
+
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
+
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
+
+free_fs:
+    glfs_free_from_ctx(fs);
+
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
+
+fail:
+    if (!ret)
+        ret = err;
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
+ssize_t
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
+{
+    ssize_t res = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return res;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
+int
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
+
+out:
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
+int
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
+{
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
+void
+pub_glfs_free(void *ptr)
+{
+    GLFS_FREE(ptr);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
+{
+    return arg->reason;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
+{
+    return arg->object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
+{
+    return arg->expire_time_attr;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
+{
+    return arg->p_object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->p_buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
+{
+    return arg->oldp_object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->oldp_buf;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
+{
+    return arg->object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
+{
+    return arg->lease_type;
+}
+
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
+int
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
+        }
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
+{
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
+        }
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
+        }
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
+{
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
+int
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
+{
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
+        }
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
+        }
+    }
+
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
+
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
+
+    __GLFS_EXIT_FS;
+
+    return 0;
+err:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return -1;
+}
diff --git a/api/src/glfs.h b/api/src/glfs.h
new file mode 100644
index 00000000000..279d11d58ee
--- /dev/null
+++ b/api/src/glfs.h
@@ -0,0 +1,1485 @@
+/*
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_H
+#define _GLFS_H
+
+/*
+  Enforce the following flags as libgfapi is built
+  with them, and we want programs linking against them to also
+  be built with these flags. This is necessary as it affects
+  some of the structures defined in libc headers (like struct stat)
+  and those definitions need to be consistently compiled in
+  both the library and the application.
+*/
+
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifndef __USE_FILE_OFFSET64
+#define __USE_FILE_OFFSET64
+#endif
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <sys/cdefs.h>
+#include <dirent.h>
+#include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
+
+#if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
+#include <sys/acl.h>
+#else
+typedef void *acl_t;
+typedef int acl_type_t;
+#endif
+
+/* Portability non glibc c++ build systems */
+#ifndef __THROW
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
+#endif
+
+#ifndef GF_DARWIN_HOST_OS
+#define GFAPI_PUBLIC(sym, ver)  /**/
+#define GFAPI_PRIVATE(sym, ver) /**/
+#else
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#endif
+
+__BEGIN_DECLS
+
+/* The filesystem object. One object per 'virtual mount' */
+struct glfs;
+typedef struct glfs glfs_t;
+
+/*
+  SYNOPSIS
+
+  glfs_new: Create a new 'virtual mount' object.
+
+  DESCRIPTION
+
+  This is most likely the very first function you will use. This function
+  will create a new glfs_t (virtual mount) object in memory.
+
+  On this newly created glfs_t, you need to be either set a volfile path
+  (glfs_set_volfile) or a volfile server (glfs_set_volfile_server).
+
+  The glfs_t object needs to be initialized with glfs_init() before you
+  can start issuing file operations on it.
+
+  PARAMETERS
+
+  @volname: Name of the volume. This identifies the server-side volume and
+            the fetched volfile (equivalent of --volfile-id command line
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
+
+  RETURN VALUES
+
+  NULL   : Out of memory condition.
+  Others : Pointer to the newly created glfs_t virtual mount object.
+
+*/
+
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile: Specify the path to the volume specification file.
+
+  DESCRIPTION
+
+  If you are using a static volume specification file (without dynamic
+  volume management abilities from the CLI), then specify the path to
+  the volume specification file.
+
+  This is incompatible with glfs_set_volfile_server().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @volfile: Path to the locally available volume specification file.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile_server: Specify the list of addresses for management server.
+
+  DESCRIPTION
+
+  This function specifies the list of addresses for the management server
+  (glusterd) to connect, and establish the volume configuration. The @volname
+  parameter passed to glfs_new() is the volume which will be virtually
+  mounted as the glfs_t object. All operations performed by the CLI at
+  the management server will automatically be reflected in the 'virtual
+  mount' object as it maintains a connection to glusterd and polls on
+  configuration change notifications.
+
+  This is incompatible with glfs_set_volfile().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @transport: String specifying the transport used to connect to the
+              management daemon. Specifying NULL will result in the
+              usage of the default (tcp) transport type. Permitted
+              values are "tcp" or "unix".
+
+  @host:      String specifying the address where to find the management daemon.
+              Socket path, while using Unix domain socket as transport type.
+              This would either be
+              - FQDN (e.g : "storage01.company.com") or
+              - ASCII (e.g : "192.168.22.1") or
+              - Socket path (e.g : "/var/run/glusterd.socket")
+
+  NOTE: This API is special, multiple calls to this function with different
+        volfile servers, port or transport-type would create a list of volfile
+        servers which would be polled during `volfile_fetch_attempts()`
+
+  @port: The TCP port number where gluster management daemon is listening.
+         Specifying 0 uses the default port number GF_DEFAULT_BASE_PORT.
+         This parameter is unused if you are using a UNIX domain socket.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
+
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
+/*
+  SYNOPSIS
+
+  glfs_set_logging: Specify logging parameters.
+
+  DESCRIPTION
+
+  This function specifies logging parameters for the virtual mount.
+  Default log file is /dev/null.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the logging parameters.
+
+  @logfile: The logfile to be used for logging. Will be created if it does not
+            already exist (provided system permissions allow). If NULL, a new
+            logfile will be created in default log directory associated with
+            the glusterfs installation.
+
+  @loglevel: Numerical value specifying the degree of verbosity. Higher the
+             value, more verbose the logging.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_init: Initialize the 'virtual mount'
+
+  DESCRIPTION
+
+  This function initializes the glfs_t object. This consists of many steps:
+  - Spawn a poll-loop thread.
+  - Establish connection to management daemon and receive volume specification.
+  - Construct translator graph and initialize graph.
+  - Wait for initialization (connecting to all bricks) to complete.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_fini: Cleanup and destroy the 'virtual mount'
+
+  DESCRIPTION
+
+  This function attempts to gracefully destroy glfs_t object. An attempt is
+  made to wait for all background processing to complete before returning.
+
+  glfs_fini() must be called after all operations on glfs_t is finished.
+
+  IMPORTANT
+
+  IT IS NECESSARY TO CALL glfs_fini() ON ALL THE INITIALIZED glfs_t
+  OBJECTS BEFORE TERMINATING THE PROGRAM. THERE MAY BE CACHED AND
+  UNWRITTEN / INCOMPLETE OPERATIONS STILL IN PROGRESS EVEN THOUGH THE
+  API CALLS HAVE RETURNED. glfs_fini() WILL WAIT FOR BACKGROUND OPERATIONS
+  TO COMPLETE BEFORE RETURNING, THEREBY MAKING IT SAFE FOR THE PROGRAM TO
+  EXIT.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be destroyed.
+
+  RETURN VALUES
+
+   0 : Success.
+*/
+
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
+
+/*
+  SYNOPSIS
+
+      glfs_getvol: Get the volfile associated with a 'virtual mount'
+
+  DESCRIPTION
+
+      Sometimes it's useful e.g. for scripts to see the volfile, so that they
+      can parse it and find subvolumes to do things like split-brain resolution
+      or custom layouts.  The API here was specifically intended to make access
+      e.g. from Python as simple as possible.
+
+      Note that the volume must be started (not necessarily mounted) for this
+      to work.
+
+  PARAMETERS
+
+      @fs:  The 'virtual mount' object for which a volfile is desired
+      @buf: Pointer to a place for the volfile length to be stored
+      @len: Length of @buf
+
+  RETURN VALUES
+
+      >0: filled N bytes of buffer
+       0: no volfile available
+      <0: volfile length exceeds @len by N bytes (@buf unchanged)
+*/
+
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
+
+/*
+  SYNOPSIS
+
+       glfs_get_volumeid: Copy the Volume UUID stored in the glfs object fs.
+
+  DESCRIPTION
+
+       This function when invoked for the first time sends RPC call to the
+       the management server (glusterd) to fetch volume uuid and stores it
+       in the glusterfs_context linked to the glfs object fs which can be used
+       in the subsequent calls. Later it parses that UUID to convert it from
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
+
+  PARAMETERS
+
+       @fs: The 'virtual mount' object to be used to retrieve and store
+            volume's UUID.
+       @volid: Pointer to a place for the volume UUID to be stored
+       @size: Length of @volid
+
+  RETURN VALUES
+
+       -1 : Failure. @errno will be set with the type of failure.
+        Others : length of the volume UUID stored.
+*/
+
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
+
+/*
+ * FILE OPERATION
+ *
+ * What follows are filesystem operations performed on the
+ * 'virtual mount'. The calls here are kept as close to
+ * the POSIX system calls as possible.
+ *
+ * Notes:
+ *
+ * - All paths specified, even if absolute, are relative to the
+ *   root of the virtual mount and not the system root (/).
+ *
+ */
+
+/* The file descriptor object. One per open file/directory. */
+
+struct glfs_fd;
+typedef struct glfs_fd glfs_fd_t;
+
+/*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
+ * PER THREAD IDENTITY MODIFIERS
+ *
+ * The following operations enable to set a per thread identity context
+ * for the glfs APIs to perform operations as. The calls here are kept as close
+ * to POSIX equivalents as possible.
+ *
+ * NOTES:
+ *
+ *  - setgroups is a per thread setting, hence this is named as fsgroups to be
+ *    close in naming to the fs(u/g)id APIs
+ *  - Typical mode of operation is to set the IDs as required, with the
+ *    supplementary groups being optionally set, make the glfs call and post the
+ *    glfs operation set them back to eu/gid or uid/gid as appropriate to the
+ *    caller
+ *  - The groups once set, need to be unset by setting the size to 0 (in which
+ *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
+ *  - Once a process for a thread of operation choses to set the IDs, all glfs
+ *    calls made from that thread would default to the IDs set for the thread.
+ *    As a result use these APIs with care and ensure that the set IDs are
+ *    reverted to global process defaults as required.
+ *
+ */
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
+
+/*
+  SYNOPSIS
+
+  glfs_open: Open a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @flags: Open flags. See open(2). O_CREAT is not supported.
+          Use glfs_creat() for creating files.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_creat: Create a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @mode: Permission of the file to be created.
+
+  @flags: Create flags. See open(2). O_EXCL is supported.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
+
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
+
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
+
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
+
+/*
+
+  glfs_io_cbk
+
+  The following is the function type definition of the callback
+  function pointer which has to be provided by the caller to the
+  *_async() versions of the IO calls.
+
+  The callback function is called on completion of the requested
+  IO, and the appropriate return value is returned in @ret.
+
+  In case of an error in completing the IO, @ret will be -1 and
+  @errno will be set with the appropriate error.
+
+  @ret will be same as the return value of the non _async() variant
+  of the particular call
+
+  @data is the same context pointer provided by the caller at the
+  time of issuing the async IO call. This can be used by the
+  caller to differentiate different instances of the async requests
+  in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
+*/
+
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
+
+// glfs_{read,write}[_async]
+
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
+
+// glfs_{read,write}v[_async]
+
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
+
+// glfs_p{read,write}[_async]
+
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
+
+// glfs_p{read,write}v[_async]
+
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
+
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
+
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
+
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
+
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
+
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
+
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
+
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
+
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
+
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
+
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
+
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
+
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
+
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
+
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
+
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
+
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
+
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
+
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
+
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
+
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
+
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
+
+/*
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_file_lock: Request extended byte range lock on a file
+
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
+
+  PARAMETERS
+
+  @fd: File descriptor
+
+  @cmd: As specified in man fcntl(2).
+
+  @flock: As specified in man fcntl(2).
+
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
+
+  RETURN VALUES
+
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
+
+ */
+
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
+
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
+
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
+
+/*
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
+ */
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
+
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
+
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
+
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
+
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
+ */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
+
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
+
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
+
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
+
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
+
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
+
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
+
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
+
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
+
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
+
+/*
+  SYNOPSIS
+
+  glfs_lease: Takes a lease on a file.
+
+  DESCRIPTION
+
+  This function takes lease on an open file.
+
+  PARAMETERS
+
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
+
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
+
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
+
+      @lease.lease_id - A unique identification of lease, 128bits.
+
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
+
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
+
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
+
+/*
+  SYNOPSIS
+
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
+
+  DESCRIPTION
+
+  The functions are used to set attributes on the file.
+
+  PARAMETERS
+
+  @glfs_fsetattr
+
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
+
+  @glfs_setattr
+
+         @fs: File object.
+
+         @path: The path of the file that is being operated on.
+
+         @follow: Flag used to resolve symlink.
+
+
+  @stat: Struct that has information about the file.
+
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
+
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+
+ */
+
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
+
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
+
+/*
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+ */
+
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
+
+__END_DECLS
+#endif /* !_GLFS_H */
diff --git a/argp-standalone/Makefile.am b/argp-standalone/Makefile.am
deleted file mode 100644
index 4775d4876aa..00000000000
--- a/argp-standalone/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-# From glibc
-
-# Copyright (C) 1997, 2003, 2004 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Library General Public License for more details.
-
-# You should have received a copy of the GNU Library General Public
-# License along with the GNU C Library; see the file COPYING.LIB.  If
-# not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = .
-
-LIBOBJS = @LIBOBJS@
-
-noinst_LIBRARIES = libargp.a
-
-noinst_HEADERS = argp.h argp-fmtstream.h argp-namefrob.h
-
-EXTRA_DIST = mempcpy.c strchrnul.c strndup.c strcasecmp.c vsnprintf.c autogen.sh
-
-# Leaves out argp-fs-xinl.c and argp-xinl.c 
-libargp_a_SOURCES = argp-ba.c argp-eexst.c argp-fmtstream.c \
-		    argp-help.c argp-parse.c argp-pv.c \
-		    argp-pvh.c
-
-libargp_a_LIBADD = $(LIBOBJS)
-
-
diff --git a/argp-standalone/acinclude.m4 b/argp-standalone/acinclude.m4
deleted file mode 100644
index fb61e957dfa..00000000000
--- a/argp-standalone/acinclude.m4
+++ /dev/null
@@ -1,1084 +0,0 @@
-dnl Try to detect the type of the third arg to getsockname() et al
-AC_DEFUN([LSH_TYPE_SOCKLEN_T],
-[AH_TEMPLATE([socklen_t], [Length type used by getsockopt])
-AC_CACHE_CHECK([for socklen_t in sys/socket.h], ac_cv_type_socklen_t,
-[AC_EGREP_HEADER(socklen_t, sys/socket.h,
-  [ac_cv_type_socklen_t=yes], [ac_cv_type_socklen_t=no])])
-if test $ac_cv_type_socklen_t = no; then
-        AC_MSG_CHECKING(for AIX)
-        AC_EGREP_CPP(yes, [
-#ifdef _AIX
- yes
-#endif
-],[
-AC_MSG_RESULT(yes)
-AC_DEFINE(socklen_t, size_t)
-],[
-AC_MSG_RESULT(no)
-AC_DEFINE(socklen_t, int)
-])
-fi
-])
-
-dnl Choose cc flags for compiling position independent code
-AC_DEFUN([LSH_CCPIC],
-[AC_MSG_CHECKING(CCPIC)
-AC_CACHE_VAL(lsh_cv_sys_ccpic,[
-  if test -z "$CCPIC" ; then
-    if test "$GCC" = yes ; then
-      case `uname -sr` in
-	BSD/OS*)
-         case `uname -r` in
-           4.*) CCPIC="-fPIC";;
-           *) CCPIC="";;
-         esac
-	;;
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-	SunOS\ 5.*)
-	  # Could also use -fPIC, if there are a large number of symbol reference
-	  CCPIC="-fPIC"
-	;;
-	CYGWIN*)
-	  CCPIC=""
-	;;
-	*)
-	  CCPIC="-fpic"
-	;;
-      esac
-    else
-      case `uname -sr` in
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-        IRIX*)
-          CCPIC="-share"
-        ;;
-	hp*|HP*) CCPIC="+z"; ;;
-	FreeBSD*) CCPIC="-fpic";;
-	SCO_SV*) CCPIC="-KPIC -dy -Bdynamic";;
-        UnixWare*|OpenUNIX*) CCPIC="-KPIC -dy -Bdynamic";;
-	Solaris*) CCPIC="-KPIC -Bdynamic";;
-	Windows_NT*) CCPIC="-shared" ;;
-      esac
-    fi
-  fi
-  OLD_CFLAGS="$CFLAGS"
-  CFLAGS="$CFLAGS $CCPIC"
-  AC_TRY_COMPILE([], [exit(0);],
-    lsh_cv_sys_ccpic="$CCPIC", lsh_cv_sys_ccpic='')
-  CFLAGS="$OLD_CFLAGS"
-])
-CCPIC="$lsh_cv_sys_ccpic"
-AC_MSG_RESULT($CCPIC)
-AC_SUBST([CCPIC])])
-
-dnl LSH_PATH_ADD(path-id, directory)
-AC_DEFUN([LSH_PATH_ADD],
-[AC_MSG_CHECKING($2)
-ac_exists=no
-if test -d "$2/." ; then
-  ac_real_dir=`cd $2 && pwd`
-  if test -n "$ac_real_dir" ; then
-    ac_exists=yes
-    for old in $1_REAL_DIRS ; do
-      ac_found=no
-      if test x$ac_real_dir = x$old ; then
-        ac_found=yes;
-	break;
-      fi
-    done
-    if test $ac_found = yes ; then
-      AC_MSG_RESULT(already added)
-    else
-      AC_MSG_RESULT(added)
-      # LDFLAGS="$LDFLAGS -L $2"
-      $1_REAL_DIRS="$ac_real_dir [$]$1_REAL_DIRS"
-      $1_DIRS="$2 [$]$1_DIRS"
-    fi
-  fi
-fi
-if test $ac_exists = no ; then
-  AC_MSG_RESULT(not found)
-fi
-])
-
-dnl LSH_RPATH_ADD(dir)
-AC_DEFUN([LSH_RPATH_ADD], [LSH_PATH_ADD(RPATH_CANDIDATE, $1)])
-
-dnl LSH_RPATH_INIT(candidates)
-AC_DEFUN([LSH_RPATH_INIT],
-[AC_MSG_CHECKING([for -R flag])
-RPATHFLAG=''
-case `uname -sr` in
-  OSF1\ V4.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 6.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 5.*)
-    RPATHFLAG="-rpath "
-    ;;
-  SunOS\ 5.*)
-    if test "$TCC" = "yes"; then
-      # tcc doesn't know about -R
-      RPATHFLAG="-Wl,-R,"
-    else
-      RPATHFLAG=-R
-    fi
-    ;;
-  Linux\ 2.*)
-    RPATHFLAG="-Wl,-rpath,"
-    ;;
-  *)
-    :
-    ;;
-esac
-
-if test x$RPATHFLAG = x ; then
-  AC_MSG_RESULT(none)
-else
-  AC_MSG_RESULT([using $RPATHFLAG])
-fi
-
-RPATH_CANDIDATE_REAL_DIRS=''
-RPATH_CANDIDATE_DIRS=''
-
-AC_MSG_RESULT([Searching for libraries])
-
-for d in $1 ; do
-  LSH_RPATH_ADD($d)
-done
-])    
-
-dnl Try to execute a main program, and if it fails, try adding some
-dnl -R flag.
-dnl LSH_RPATH_FIX
-AC_DEFUN([LSH_RPATH_FIX],
-[if test $cross_compiling = no -a "x$RPATHFLAG" != x ; then
-  ac_success=no
-  AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-    ac_success=yes, ac_success=no, :)
-  
-  if test $ac_success = no ; then
-    AC_MSG_CHECKING([Running simple test program failed. Trying -R flags])
-dnl echo RPATH_CANDIDATE_DIRS = $RPATH_CANDIDATE_DIRS
-    ac_remaining_dirs=''
-    ac_rpath_save_LDFLAGS="$LDFLAGS"
-    for d in $RPATH_CANDIDATE_DIRS ; do
-      if test $ac_success = yes ; then
-  	ac_remaining_dirs="$ac_remaining_dirs $d"
-      else
-  	LDFLAGS="$RPATHFLAG$d $LDFLAGS"
-dnl echo LDFLAGS = $LDFLAGS
-  	AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-  	  [ac_success=yes
-  	  ac_rpath_save_LDFLAGS="$LDFLAGS"
-  	  AC_MSG_RESULT([adding $RPATHFLAG$d])
-  	  ],
-  	  [ac_remaining_dirs="$ac_remaining_dirs $d"], :)
-  	LDFLAGS="$ac_rpath_save_LDFLAGS"
-      fi
-    done
-    RPATH_CANDIDATE_DIRS=$ac_remaining_dirs
-  fi
-  if test $ac_success = no ; then
-    AC_MSG_RESULT(failed)
-  fi
-fi
-])
-
-dnl Like AC_CHECK_LIB, but uses $KRB_LIBS rather than $LIBS.
-dnl LSH_CHECK_KRB_LIB(LIBRARY, FUNCTION, [, ACTION-IF-FOUND [,
-dnl                  ACTION-IF-NOT-FOUND [, OTHER-LIBRARIES]]])
-
-AC_DEFUN([LSH_CHECK_KRB_LIB],
-[AC_CHECK_LIB([$1], [$2],
-  ifelse([$3], ,
-      [[ac_tr_lib=HAVE_LIB`echo $1 | sed -e 's/[^a-zA-Z0-9_]/_/g' \
-     	    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
-        AC_DEFINE_UNQUOTED($ac_tr_lib)
-        KRB_LIBS="-l$1 $KRB_LIBS"
-      ]], [$3]),
-  ifelse([$4], , , [$4
-])dnl
-, [$5 $KRB_LIBS])
-])
-
-dnl LSH_LIB_ARGP(ACTION-IF-OK, ACTION-IF-BAD)
-AC_DEFUN([LSH_LIB_ARGP],
-[ ac_argp_save_LIBS="$LIBS"
-  ac_argp_save_LDFLAGS="$LDFLAGS"
-  ac_argp_ok=no
-  # First check if we can link with argp.
-  AC_SEARCH_LIBS(argp_parse, argp,
-  [ LSH_RPATH_FIX
-    AC_CACHE_CHECK([for working argp],
-      lsh_cv_lib_argp_works,
-      [ AC_TRY_RUN(
-[#include <argp.h>
-#include <stdlib.h>
-
-static const struct argp_option
-options[] =
-{
-  { NULL, 0, NULL, 0, NULL, 0 }
-};
-
-struct child_state
-{
-  int n;
-};
-
-static error_t
-child_parser(int key, char *arg, struct argp_state *state)
-{
-  struct child_state *input = (struct child_state *) state->input;
-  
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_END:
-      if (!input->n)
-	input->n = 1;
-      break;
-    }
-  return 0;
-}
-
-const struct argp child_argp =
-{
-  options,
-  child_parser,
-  NULL, NULL, NULL, NULL, NULL
-};
-
-struct main_state
-{
-  struct child_state child;
-  int m;
-};
-
-static error_t
-main_parser(int key, char *arg, struct argp_state *state)
-{
-  struct main_state *input = (struct main_state *) state->input;
-
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_INIT:
-      state->child_inputs[0] = &input->child;
-      break;
-    case ARGP_KEY_END:
-      if (!input->m)
-	input->m = input->child.n;
-      
-      break;
-    }
-  return 0;
-}
-
-static const struct argp_child
-main_children[] =
-{
-  { &child_argp, 0, "", 0 },
-  { NULL, 0, NULL, 0}
-};
-
-static const struct argp
-main_argp =
-{ options, main_parser, 
-  NULL,
-  NULL,
-  main_children,
-  NULL, NULL
-};
-
-int main(int argc, char **argv)
-{
-  struct main_state input = { { 0 }, 0 };
-  char *v[2] = { "foo", NULL };
-
-  argp_parse(&main_argp, 1, v, 0, NULL, &input);
-
-  if ( (input.m == 1) && (input.child.n == 1) )
-    return 0;
-  else
-    return 1;
-}
-], lsh_cv_lib_argp_works=yes,
-   lsh_cv_lib_argp_works=no,
-   lsh_cv_lib_argp_works=no)])
-
-  if test x$lsh_cv_lib_argp_works = xyes ; then
-    ac_argp_ok=yes
-  else
-    # Reset link flags
-    LIBS="$ac_argp_save_LIBS"
-    LDFLAGS="$ac_argp_save_LDFLAGS"
-  fi])
-
-  if test x$ac_argp_ok = xyes ; then
-    ifelse([$1],, true, [$1])
-  else
-    ifelse([$2],, true, [$2])
-  fi   
-])
-
-dnl LSH_GCC_ATTRIBUTES
-dnl Check for gcc's __attribute__ construction
-
-AC_DEFUN([LSH_GCC_ATTRIBUTES],
-[AC_CACHE_CHECK(for __attribute__,
-	       lsh_cv_c_attribute,
-[ AC_TRY_COMPILE([
-#include <stdlib.h>
-],
-[
-static void foo(void) __attribute__ ((noreturn));
-
-static void __attribute__ ((noreturn))
-foo(void)
-{
-  exit(1);
-}
-],
-lsh_cv_c_attribute=yes,
-lsh_cv_c_attribute=no)])
-
-AH_TEMPLATE([HAVE_GCC_ATTRIBUTE], [Define if the compiler understands __attribute__])
-if test "x$lsh_cv_c_attribute" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_ATTRIBUTE)
-fi
-
-AH_BOTTOM(
-[#if __GNUC__ || HAVE_GCC_ATTRIBUTE
-# define NORETURN __attribute__ ((__noreturn__))
-# define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# define UNUSED __attribute__ ((__unused__))
-#else
-# define NORETURN
-# define PRINTF_STYLE(f, a)
-# define UNUSED
-#endif
-])])
-
-AC_DEFUN([LSH_GCC_FUNCTION_NAME],
-[# Check for gcc's __FUNCTION__ variable
-AH_TEMPLATE([HAVE_GCC_FUNCTION],
-	    [Define if the compiler understands __FUNCTION__])
-AH_BOTTOM(
-[#if HAVE_GCC_FUNCTION
-# define FUNCTION_NAME __FUNCTION__
-#else
-# define FUNCTION_NAME "Unknown"
-#endif
-])
-
-AC_CACHE_CHECK(for __FUNCTION__,
-	       lsh_cv_c_FUNCTION,
-  [ AC_TRY_COMPILE(,
-      [ #if __GNUC__ == 3
-	#  error __FUNCTION__ is broken in gcc-3
-	#endif
-        void foo(void) { char c = __FUNCTION__[0]; } ],
-      lsh_cv_c_FUNCTION=yes,
-      lsh_cv_c_FUNCTION=no)])
-
-if test "x$lsh_cv_c_FUNCTION" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_FUNCTION)
-fi
-])
-
-# Check for alloca, and include the standard blurb in config.h
-AC_DEFUN([LSH_FUNC_ALLOCA],
-[AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([malloc.h])
-AH_BOTTOM(
-[/* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# else
-#  ifdef _AIX
- #pragma alloca
-#  else
-#   ifndef alloca /* predefined by HP cc +Olibcalls */
-char *alloca ();
-#   endif
-#  endif
-# endif
-#else /* defined __GNUC__ */
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# endif
-#endif
-/* Needed for alloca on windows */
-#if HAVE_MALLOC_H
-# include <malloc.h>
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRERROR],
-[AC_CHECK_FUNCS(strerror)
-AH_BOTTOM(
-[#if HAVE_STRERROR
-#define STRERROR strerror
-#else
-#define STRERROR(x) (sys_errlist[x])
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRSIGNAL],
-[AC_CHECK_FUNCS(strsignal)
-AC_CHECK_DECLS([sys_siglist, _sys_siglist])
-AH_BOTTOM(
-[#if HAVE_STRSIGNAL
-# define STRSIGNAL strsignal
-#else /* !HAVE_STRSIGNAL */
-# if HAVE_DECL_SYS_SIGLIST
-#  define STRSIGNAL(x) (sys_siglist[x])
-# else
-#  if HAVE_DECL__SYS_SIGLIST
-#   define STRSIGNAL(x) (_sys_siglist[x])
-#  else
-#   define STRSIGNAL(x) "Unknown signal"
-#   if __GNUC__
-#    warning Using dummy STRSIGNAL
-#   endif
-#  endif
-# endif
-#endif /* !HAVE_STRSIGNAL */
-])])
-
-dnl LSH_MAKE_CONDITIONAL(symbol, test)
-AC_DEFUN([LSH_MAKE_CONDITIONAL],
-[if $2 ; then
-  IF_$1=''
-  UNLESS_$1='# '
-else
-  IF_$1='# '
-  UNLESS_$1=''
-fi 
-AC_SUBST(IF_$1)
-AC_SUBST(UNLESS_$1)])
-
-dnl LSH_DEPENDENCY_TRACKING
-
-dnl Defines compiler flags DEP_FLAGS to generate dependency
-dnl information, and DEP_PROCESS that is any shell commands needed for
-dnl massaging the dependency information further. Dependencies are
-dnl generated as a side effect of compilation. Dependency files
-dnl themselves are not treated as targets.
-
-AC_DEFUN([LSH_DEPENDENCY_TRACKING],
-[AC_ARG_ENABLE(dependency_tracking,
-  AC_HELP_STRING([--disable-dependency-tracking],
-    [Disable dependency tracking. Dependency tracking doesn't work with BSD make]),,
-  [enable_dependency_tracking=yes])
-
-DEP_FLAGS=''
-DEP_PROCESS='true'
-if test x$enable_dependency_tracking = xyes ; then
-  if test x$GCC = xyes ; then
-    gcc_version=`gcc --version | head -1`
-    case "$gcc_version" in
-      2.*|*[[!0-9.]]2.*)
-        enable_dependency_tracking=no
-        AC_MSG_WARN([Dependency tracking disabled, gcc-3.x is needed])
-      ;;
-      *)
-        DEP_FLAGS='-MT $[]@ -MD -MP -MF $[]@.d'
-        DEP_PROCESS='true'
-      ;;
-    esac
-  else
-    enable_dependency_tracking=no
-    AC_MSG_WARN([Dependency tracking disabled])
-  fi
-fi
-
-if test x$enable_dependency_tracking = xyes ; then
-  DEP_INCLUDE='include '
-else
-  DEP_INCLUDE='# '
-fi
-
-AC_SUBST([DEP_INCLUDE])
-AC_SUBST([DEP_FLAGS])
-AC_SUBST([DEP_PROCESS])])
-
-dnl @synopsis AX_CREATE_STDINT_H [( HEADER-TO-GENERATE [, HEADERS-TO-CHECK])]
-dnl
-dnl the "ISO C9X: 7.18 Integer types <stdint.h>" section requires the
-dnl existence of an include file <stdint.h> that defines a set of 
-dnl typedefs, especially uint8_t,int32_t,uintptr_t.
-dnl Many older installations will not provide this file, but some will
-dnl have the very same definitions in <inttypes.h>. In other enviroments
-dnl we can use the inet-types in <sys/types.h> which would define the
-dnl typedefs int8_t and u_int8_t respectivly.
-dnl
-dnl This macros will create a local "_stdint.h" or the headerfile given as 
-dnl an argument. In many cases that file will just "#include <stdint.h>" 
-dnl or "#include <inttypes.h>", while in other environments it will provide 
-dnl the set of basic 'stdint's definitions/typedefs: 
-dnl   int8_t,uint8_t,int16_t,uint16_t,int32_t,uint32_t,intptr_t,uintptr_t
-dnl   int_least32_t.. int_fast32_t.. intmax_t
-dnl which may or may not rely on the definitions of other files,
-dnl or using the AC_CHECK_SIZEOF macro to determine the actual
-dnl sizeof each type.
-dnl
-dnl if your header files require the stdint-types you will want to create an
-dnl installable file mylib-int.h that all your other installable header
-dnl may include. So if you have a library package named "mylib", just use
-dnl      AX_CREATE_STDINT_H(mylib-int.h) 
-dnl in configure.ac and go to install that very header file in Makefile.am
-dnl along with the other headers (mylib.h) - and the mylib-specific headers
-dnl can simply use "#include <mylib-int.h>" to obtain the stdint-types.
-dnl
-dnl Remember, if the system already had a valid <stdint.h>, the generated
-dnl file will include it directly. No need for fuzzy HAVE_STDINT_H things...
-dnl
-dnl @, (status: used on new platforms) (see http://ac-archive.sf.net/gstdint/)
-dnl @version $Id: acinclude.m4,v 1.27 2004/11/23 21:27:35 nisse Exp $
-dnl @author  Guido Draheim <guidod@gmx.de> 
-
-AC_DEFUN([AX_CREATE_STDINT_H],
-[# ------ AX CREATE STDINT H -------------------------------------
-AC_MSG_CHECKING([for stdint types])
-ac_stdint_h=`echo ifelse($1, , _stdint.h, $1)`
-# try to shortcircuit - if the default include path of the compiler
-# can find a "stdint.h" header then we assume that all compilers can.
-AC_CACHE_VAL([ac_cv_header_stdint_t],[
-old_CXXFLAGS="$CXXFLAGS" ; CXXFLAGS=""
-old_CPPFLAGS="$CPPFLAGS" ; CPPFLAGS=""
-old_CFLAGS="$CFLAGS"     ; CFLAGS=""
-AC_TRY_COMPILE([#include <stdint.h>],[int_least32_t v = 0;],
-[ac_cv_stdint_result="(assuming C99 compatible system)"
- ac_cv_header_stdint_t="stdint.h"; ],
-[ac_cv_header_stdint_t=""])
-CXXFLAGS="$old_CXXFLAGS"
-CPPFLAGS="$old_CPPFLAGS"
-CFLAGS="$old_CFLAGS" ])
-
-v="... $ac_cv_header_stdint_h"
-if test "$ac_stdint_h" = "stdint.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./stdint.h?)])
-elif test "$ac_stdint_h" = "inttypes.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./inttypes.h?)])
-elif test "_$ac_cv_header_stdint_t" = "_" ; then
- AC_MSG_RESULT([(putting them into $ac_stdint_h)$v])
-else
- ac_cv_header_stdint="$ac_cv_header_stdint_t"
- AC_MSG_RESULT([$ac_cv_header_stdint (shortcircuit)])
-fi
-
-if test "_$ac_cv_header_stdint_t" = "_" ; then # can not shortcircuit..
-
-dnl .....intro message done, now do a few system checks.....
-dnl btw, all CHECK_TYPE macros do automatically "DEFINE" a type, therefore
-dnl we use the autoconf implementation detail _AC CHECK_TYPE_NEW instead
-
-inttype_headers=`echo $2 | sed -e 's/,/ /g'`
-
-ac_cv_stdint_result="(no helpful system typedefs seen)"
-AC_CACHE_CHECK([for stdint uintptr_t], [ac_cv_header_stdint_x],[
- ac_cv_header_stdint_x="" # the 1997 typedefs (inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in stdint.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_uintptr_t 
-   unset ac_cv_type_uint64_t
-   _AC_CHECK_TYPE_NEW(uintptr_t,[ac_cv_header_stdint_x=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uintptr_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uintptr_t])
- ])
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-AC_CACHE_CHECK([for stdint uint32_t], [ac_cv_header_stdint_o],[
- ac_cv_header_stdint_o="" # the 1995 typedefs (sys/inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in inttypes.h sys/inttypes.h stdint.h $inttype_headers ; do
-   unset ac_cv_type_uint32_t
-   unset ac_cv_type_uint64_t
-   AC_CHECK_TYPE(uint32_t,[ac_cv_header_stdint_o=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uint32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uint32_t])
- ])
-fi
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-if test "_$ac_cv_header_stdint_o" = "_" ; then
-AC_CACHE_CHECK([for stdint u_int32_t], [ac_cv_header_stdint_u],[
- ac_cv_header_stdint_u="" # the BSD typedefs (sys/types.h)
-  AC_MSG_RESULT([(..)])
-  for i in sys/types.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_u_int32_t
-   unset ac_cv_type_u_int64_t
-   AC_CHECK_TYPE(u_int32_t,[ac_cv_header_stdint_u=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(u_int64_t,[and64="/u_int64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen u_int32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint u_int32_t])
- ])
-fi fi
-
-dnl if there was no good C99 header file, do some typedef checks...
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-   AC_MSG_CHECKING([for stdint datatype model])
-   AC_MSG_RESULT([(..)])
-   AC_CHECK_SIZEOF(char)
-   AC_CHECK_SIZEOF(short)
-   AC_CHECK_SIZEOF(int)
-   AC_CHECK_SIZEOF(long)
-   AC_CHECK_SIZEOF(void*)
-   ac_cv_stdint_char_model=""
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_char"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_short"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model=""
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_long"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_voidp"
-   name="$ac_cv_stdint_long_model"
-   case "$ac_cv_stdint_char_model/$ac_cv_stdint_long_model" in
-    122/242)     name="$name,  IP16 (standard 16bit machine)" ;;
-    122/244)     name="$name,  LP32 (standard 32bit mac/win)" ;;
-    122/*)       name="$name        (unusual int16 model)" ;; 
-    124/444)     name="$name, ILP32 (standard 32bit unixish)" ;;
-    124/488)     name="$name,  LP64 (standard 64bit unixish)" ;;
-    124/448)     name="$name, LLP64 (unusual  64bit unixish)" ;;
-    124/*)       name="$name        (unusual int32 model)" ;; 
-    128/888)     name="$name, ILP64 (unusual  64bit numeric)" ;;
-    128/*)       name="$name        (unusual int64 model)" ;; 
-    222/*|444/*) name="$name        (unusual dsptype)" ;;
-     *)          name="$name        (very unusal model)" ;;
-   esac
-   AC_MSG_RESULT([combined for stdint datatype model...  $name])
-fi
-
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_x"
-elif  test "_$ac_cv_header_stdint_o" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_o"
-elif  test "_$ac_cv_header_stdint_u" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_u"
-else
-   ac_cv_header_stdint="stddef.h"
-fi
-
-AC_MSG_CHECKING([for extra inttypes in chosen header])
-AC_MSG_RESULT([($ac_cv_header_stdint)])
-dnl see if int_least and int_fast types are present in _this_ header.
-unset ac_cv_type_int_least32_t
-unset ac_cv_type_int_fast32_t
-AC_CHECK_TYPE(int_least32_t,,,[#include <$ac_cv_header_stdint>])
-AC_CHECK_TYPE(int_fast32_t,,,[#include<$ac_cv_header_stdint>])
-AC_CHECK_TYPE(intmax_t,,,[#include <$ac_cv_header_stdint>])
-
-fi # shortcircut to system "stdint.h"
-# ------------------ PREPARE VARIABLES ------------------------------
-if test "$GCC" = "yes" ; then
-ac_cv_stdint_message="using gnu compiler "`$CC --version | head -1` 
-else
-ac_cv_stdint_message="using $CC"
-fi
-
-AC_MSG_RESULT([make use of $ac_cv_header_stdint in $ac_stdint_h dnl
-$ac_cv_stdint_result])
-
-# ----------------- DONE inttypes.h checks START header -------------
-AC_CONFIG_COMMANDS([$ac_stdint_h],[
-AC_MSG_NOTICE(creating $ac_stdint_h : $_ac_stdint_h)
-ac_stdint=$tmp/_stdint.h
-
-echo "#ifndef" $_ac_stdint_h >$ac_stdint
-echo "#define" $_ac_stdint_h "1" >>$ac_stdint
-echo "#ifndef" _GENERATED_STDINT_H >>$ac_stdint
-echo "#define" _GENERATED_STDINT_H '"'$PACKAGE $VERSION'"' >>$ac_stdint
-echo "/* generated $ac_cv_stdint_message */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_t" != "_" ; then 
-echo "#define _STDINT_HAVE_STDINT_H" "1" >>$ac_stdint
-fi
-
-cat >>$ac_stdint <<STDINT_EOF
-
-/* ................... shortcircuit part ........................... */
-
-#if defined HAVE_STDINT_H || defined _STDINT_HAVE_STDINT_H
-#include <stdint.h>
-#else
-#include <stddef.h>
-
-/* .................... configured part ............................ */
-
-STDINT_EOF
-
-echo "/* whether we have a C99 compatible stdint header file */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-  ac_header="$ac_cv_header_stdint_x"
-  echo "#define _STDINT_HEADER_INTPTR" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_INTPTR */" >>$ac_stdint
-fi
-
-echo "/* whether we have a C96 compatible inttypes header file */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_o" != "_" ; then
-  ac_header="$ac_cv_header_stdint_o"
-  echo "#define _STDINT_HEADER_UINT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_UINT32 */" >>$ac_stdint
-fi
-
-echo "/* whether we have a BSD compatible inet types header */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_u" != "_" ; then
-  ac_header="$ac_cv_header_stdint_u"
-  echo "#define _STDINT_HEADER_U_INT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_U_INT32 */" >>$ac_stdint
-fi
-
-echo "" >>$ac_stdint
-
-if test "_$ac_header" != "_" ; then if test "$ac_header" != "stddef.h" ; then
-  echo "#include <$ac_header>" >>$ac_stdint
-  echo "" >>$ac_stdint
-fi fi
-
-echo "/* which 64bit typedef has been found */" >>$ac_stdint
-if test "$ac_cv_type_uint64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_UINT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_UINT64_T */" >>$ac_stdint
-fi
-if test "$ac_cv_type_u_int64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_U_INT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_U_INT64_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* which type model has been detected */" >>$ac_stdint
-if test "_$ac_cv_stdint_char_model" != "_" ; then
-echo "#define   _STDINT_CHAR_MODEL" "$ac_cv_stdint_char_model" >>$ac_stdint
-echo "#define   _STDINT_LONG_MODEL" "$ac_cv_stdint_long_model" >>$ac_stdint
-else
-echo "/* #undef _STDINT_CHAR_MODEL // skipped */" >>$ac_stdint
-echo "/* #undef _STDINT_LONG_MODEL // skipped */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* whether int_least types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_least32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_LEAST32_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_LEAST32_T */" >>$ac_stdint
-fi
-echo "/* whether int_fast types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_fast32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_FAST32_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_FAST32_T */" >>$ac_stdint
-fi
-echo "/* whether intmax_t type was detected */" >>$ac_stdint
-if test "$ac_cv_type_intmax_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INTMAX_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INTMAX_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-  cat >>$ac_stdint <<STDINT_EOF
-/* .................... detections part ............................ */
-
-/* whether we need to define bitspecific types from compiler base types */
-#ifndef _STDINT_HEADER_INTPTR
-#ifndef _STDINT_HEADER_UINT32
-#ifndef _STDINT_HEADER_U_INT32
-#define _STDINT_NEED_INT_MODEL_T
-#else
-#define _STDINT_HAVE_U_INT_TYPES
-#endif
-#endif
-#endif
-
-#ifdef _STDINT_HAVE_U_INT_TYPES
-#undef _STDINT_NEED_INT_MODEL_T
-#endif
-
-#ifdef  _STDINT_CHAR_MODEL
-#if     _STDINT_CHAR_MODEL+0 == 122 || _STDINT_CHAR_MODEL+0 == 124
-#ifndef _STDINT_BYTE_MODEL
-#define _STDINT_BYTE_MODEL 12
-#endif
-#endif
-#endif
-
-#ifndef _STDINT_HAVE_INT_LEAST32_T
-#define _STDINT_NEED_INT_LEAST_T
-#endif
-
-#ifndef _STDINT_HAVE_INT_FAST32_T
-#define _STDINT_NEED_INT_FAST_T
-#endif
-
-#ifndef _STDINT_HEADER_INTPTR
-#define _STDINT_NEED_INTPTR_T
-#ifndef _STDINT_HAVE_INTMAX_T
-#define _STDINT_NEED_INTMAX_T
-#endif
-#endif
-
-
-/* .................... definition part ............................ */
-
-/* some system headers have good uint64_t */
-#ifndef _HAVE_UINT64_T
-#if     defined _STDINT_HAVE_UINT64_T  || defined HAVE_UINT64_T
-#define _HAVE_UINT64_T
-#elif   defined _STDINT_HAVE_U_INT64_T || defined HAVE_U_INT64_T
-#define _HAVE_UINT64_T
-typedef u_int64_t uint64_t;
-#endif
-#endif
-
-#ifndef _HAVE_UINT64_T
-/* .. here are some common heuristics using compiler runtime specifics */
-#if defined __STDC_VERSION__ && defined __STDC_VERSION__ >= 199901L
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-#elif !defined __STRICT_ANSI__
-#if defined _MSC_VER || defined __WATCOMC__ || defined __BORLANDC__
-#define _HAVE_UINT64_T
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-
-#elif defined __GNUC__ || defined __MWERKS__ || defined __ELF__
-/* note: all ELF-systems seem to have loff-support which needs 64-bit */
-#if !defined _NO_LONGLONG
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-#endif
-
-#elif defined __alpha || (defined __mips && defined _ABIN32)
-#if !defined _NO_LONGLONG
-typedef long int64_t;
-typedef unsigned long uint64_t;
-#endif
-  /* compiler/cpu type to define int64_t */
-#endif
-#endif
-#endif
-
-#if defined _STDINT_HAVE_U_INT_TYPES
-/* int8_t int16_t int32_t defined by inet code, redeclare the u_intXX types */
-typedef u_int8_t uint8_t;
-typedef u_int16_t uint16_t;
-typedef u_int32_t uint32_t;
-
-/* glibc compatibility */
-#ifndef __int8_t_defined
-#define __int8_t_defined
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INT_MODEL_T
-/* we must guess all the basic types. Apart from byte-adressable system, */
-/* there a few 32-bit-only dsp-systems that we guard with BYTE_MODEL 8-} */
-/* (btw, those nibble-addressable systems are way off, or so we assume) */
-
-dnl   /* have a look at "64bit and data size neutrality" at */
-dnl   /* http://unix.org/version2/whatsnew/login_64bit.html */
-dnl   /* (the shorthand "ILP" types always have a "P" part) */
-
-#if defined _STDINT_BYTE_MODEL
-#if _STDINT_LONG_MODEL+0 == 242
-/* 2:4:2 =  IP16 = a normal 16-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned long   uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          long    int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL == 444
-/* 2:4:4 =  LP32 = a 32-bit system derived from a 16-bit */
-/* 4:4:4 = ILP32 = a normal 32-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 484 || _STDINT_LONG_MODEL+0 == 488
-/* 4:8:4 =  IP32 = a 32-bit system prepared for 64-bit    */
-/* 4:8:8 =  LP64 = a normal 64-bit system                 */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* this system has a "long" of 64bit */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long   uint64_t;
-typedef          long    int64_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 448
-/*      LLP64   a 64-bit system derived from a 32-bit system */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* assuming the system has a "long long" */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-typedef          long long  int64_t;
-#endif
-#else
-#define _STDINT_NO_INT32_T
-#endif
-#else
-#define _STDINT_NO_INT8_T
-#define _STDINT_NO_INT32_T
-#endif
-#endif
-
-/*
- * quote from SunOS-5.8 sys/inttypes.h:
- * Use at your own risk.  As of February 1996, the committee is squarely
- * behind the fixed sized types; the "least" and "fast" types are still being
- * discussed.  The probability that the "fast" types may be removed before
- * the standard is finalized is high enough that they are not currently
- * implemented.
- */
-
-#if defined _STDINT_NEED_INT_LEAST_T
-typedef  int8_t    int_least8_t;
-typedef  int16_t   int_least16_t;
-typedef  int32_t   int_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_least64_t;
-#endif
-
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_least64_t;
-#endif
-  /* least types */
-#endif
-
-#if defined _STDINT_NEED_INT_FAST_T
-typedef  int8_t    int_fast8_t; 
-typedef  int       int_fast16_t;
-typedef  int32_t   int_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_fast64_t;
-#endif
-
-typedef uint8_t   uint_fast8_t; 
-typedef unsigned  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_fast64_t;
-#endif
-  /* fast types */
-#endif
-
-#ifdef _STDINT_NEED_INTMAX_T
-#ifdef _HAVE_UINT64_T
-typedef  int64_t       intmax_t;
-typedef uint64_t      uintmax_t;
-#else
-typedef          long  intmax_t;
-typedef unsigned long uintmax_t;
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INTPTR_T
-#ifndef __intptr_t_defined
-#define __intptr_t_defined
-/* we encourage using "long" to store pointer values, never use "int" ! */
-#if   _STDINT_LONG_MODEL+0 == 242 || _STDINT_LONG_MODEL+0 == 484
-typedef  unsinged int   uintptr_t;
-typedef           int    intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL+0 == 444
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 448 && defined _HAVE_UINT64_T
-typedef        uint64_t uintptr_t;
-typedef         int64_t  intptr_t;
-#else /* matches typical system types ILP32 and LP64 - but not IP16 or LLP64 */
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#endif
-#endif
-#endif
-
-  /* shortcircuit*/
-#endif
-  /* once */
-#endif
-#endif
-STDINT_EOF
-    if cmp -s $ac_stdint_h $ac_stdint 2>/dev/null; then
-      AC_MSG_NOTICE([$ac_stdint_h is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$ac_stdint_h"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f $ac_stdint_h
-      mv $ac_stdint $ac_stdint_h
-    fi
-],[# variables for create stdint.h replacement
-PACKAGE="$PACKAGE"
-VERSION="$VERSION"
-ac_stdint_h="$ac_stdint_h"
-_ac_stdint_h=AS_TR_CPP(_$PACKAGE-$ac_stdint_h)
-ac_cv_stdint_message="$ac_cv_stdint_message"
-ac_cv_header_stdint_t="$ac_cv_header_stdint_t"
-ac_cv_header_stdint_x="$ac_cv_header_stdint_x"
-ac_cv_header_stdint_o="$ac_cv_header_stdint_o"
-ac_cv_header_stdint_u="$ac_cv_header_stdint_u"
-ac_cv_type_uint64_t="$ac_cv_type_uint64_t"
-ac_cv_type_u_int64_t="$ac_cv_type_u_int64_t"
-ac_cv_stdint_char_model="$ac_cv_stdint_char_model"
-ac_cv_stdint_long_model="$ac_cv_stdint_long_model"
-ac_cv_type_int_least32_t="$ac_cv_type_int_least32_t"
-ac_cv_type_int_fast32_t="$ac_cv_type_int_fast32_t"
-ac_cv_type_intmax_t="$ac_cv_type_intmax_t"
-])
-])
diff --git a/argp-standalone/argp-ba.c b/argp-standalone/argp-ba.c
deleted file mode 100644
index 0d3958c1151..00000000000
--- a/argp-standalone/argp-ba.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Default definition for ARGP_PROGRAM_BUG_ADDRESS.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program, it should point to string that is the
-   bug-reporting address for the program.  It will be printed by argp_help if
-   the ARGP_HELP_BUG_ADDR flag is set (as it is by various standard help
-   messages), embedded in a sentence that says something like `Report bugs to
-   ADDR.'.  */
-const char *argp_program_bug_address = 0;
diff --git a/argp-standalone/argp-eexst.c b/argp-standalone/argp-eexst.c
deleted file mode 100644
index 46b27847ad4..00000000000
--- a/argp-standalone/argp-eexst.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Default definition for ARGP_ERR_EXIT_STATUS
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_SYSEXITS_H
-# include <sysexits.h>
-#else
-# define EX_USAGE 64
-#endif
-
-#include "argp.h"
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-error_t argp_err_exit_status = EX_USAGE;
diff --git a/argp-standalone/argp-fmtstream.c b/argp-standalone/argp-fmtstream.c
deleted file mode 100644
index 7f792854fc3..00000000000
--- a/argp-standalone/argp-fmtstream.c
+++ /dev/null
@@ -1,475 +0,0 @@
-/* Word-wrapping and line-truncating streams
-   Copyright (C) 1997, 1998, 1999, 2001 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-#ifndef ARGP_FMTSTREAM_USE_LINEWRAP
-
-#ifndef isblank
-#define isblank(ch) ((ch)==' ' || (ch)=='\t')
-#endif
-
-#if defined _LIBC && defined USE_IN_LIBIO
-# include <libio/libioP.h>
-# define __vsnprintf(s, l, f, a) _IO_vsnprintf (s, l, f, a)
-#endif
-
-#define INIT_BUF_SIZE 200
-#define PRINTF_SIZE_GUESS 150
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-argp_fmtstream_t
-__argp_make_fmtstream (FILE *stream,
-		       size_t lmargin, size_t rmargin, ssize_t wmargin)
-{
-  argp_fmtstream_t fs = malloc (sizeof (struct argp_fmtstream));
-  if (fs)
-    {
-      fs->stream = stream;
-
-      fs->lmargin = lmargin;
-      fs->rmargin = rmargin;
-      fs->wmargin = wmargin;
-      fs->point_col = 0;
-      fs->point_offs = 0;
-
-      fs->buf = malloc (INIT_BUF_SIZE);
-      if (! fs->buf)
-	{
-	  free (fs);
-	  fs = 0;
-	}
-      else
-	{
-	  fs->p = fs->buf;
-	  fs->end = fs->buf + INIT_BUF_SIZE;
-	}
-    }
-
-  return fs;
-}
-#ifdef weak_alias
-weak_alias (__argp_make_fmtstream, argp_make_fmtstream)
-#endif
-
-/* Flush FS to its stream, and free it (but don't close the stream).  */
-void
-__argp_fmtstream_free (argp_fmtstream_t fs)
-{
-  __argp_fmtstream_update (fs);
-  if (fs->p > fs->buf)
-    FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-  free (fs->buf);
-  free (fs);
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_free, argp_fmtstream_free)
-#endif
-
-/* Process FS's buffer so that line wrapping is done from POINT_OFFS to the
-   end of its buffer.  This code is mostly from glibc stdio/linewrap.c.  */
-void
-__argp_fmtstream_update (argp_fmtstream_t fs)
-{
-  char *buf, *nl;
-  size_t len;
-
-  /* Scan the buffer for newlines.  */
-  buf = fs->buf + fs->point_offs;
-  while (buf < fs->p)
-    {
-      size_t r;
-
-      if (fs->point_col == 0 && fs->lmargin != 0)
-	{
-	  /* We are starting a new line.  Print spaces to the left margin.  */
-	  const size_t pad = fs->lmargin;
-	  if (fs->p + pad < fs->end)
-	    {
-	      /* We can fit in them in the buffer by moving the
-		 buffer text up and filling in the beginning.  */
-	      memmove (buf + pad, buf, fs->p - buf);
-	      fs->p += pad; /* Compensate for bigger buffer. */
-	      memset (buf, ' ', pad); /* Fill in the spaces.  */
-	      buf += pad; /* Don't bother searching them.  */
-	    }
-	  else
-	    {
-	      /* No buffer space for spaces.  Must flush.  */
-	      size_t i;
-	      for (i = 0; i < pad; i++)
-		PUTC_UNLOCKED (' ', fs->stream);
-	    }
-	  fs->point_col = pad;
-	}
-
-      len = fs->p - buf;
-      nl = memchr (buf, '\n', len);
-
-      if (fs->point_col < 0)
-	fs->point_col = 0;
-
-      if (!nl)
-	{
-	  /* The buffer ends in a partial line.  */
-
-	  if (fs->point_col + len < fs->rmargin)
-	    {
-	      /* The remaining buffer text is a partial line and fits
-		 within the maximum line width.  Advance point for the
-		 characters to be written and stop scanning.  */
-	      fs->point_col += len;
-	      break;
-	    }
-	  else
-	    /* Set the end-of-line pointer for the code below to
-	       the end of the buffer.  */
-	    nl = fs->p;
-	}
-      else if (fs->point_col + (nl - buf) < (ssize_t) fs->rmargin)
-	{
-	  /* The buffer contains a full line that fits within the maximum
-	     line width.  Reset point and scan the next line.  */
-	  fs->point_col = 0;
-	  buf = nl + 1;
-	  continue;
-	}
-
-      /* This line is too long.  */
-      r = fs->rmargin - 1;
-
-      if (fs->wmargin < 0)
-	{
-	  /* Truncate the line by overwriting the excess with the
-	     newline and anything after it in the buffer.  */
-	  if (nl < fs->p)
-	    {
-	      memmove (buf + (r - fs->point_col), nl, fs->p - nl);
-	      fs->p -= buf + (r - fs->point_col) - nl;
-	      /* Reset point for the next line and start scanning it.  */
-	      fs->point_col = 0;
-	      buf += r + 1; /* Skip full line plus \n. */
-	    }
-	  else
-	    {
-	      /* The buffer ends with a partial line that is beyond the
-		 maximum line width.  Advance point for the characters
-		 written, and discard those past the max from the buffer.  */
-	      fs->point_col += len;
-	      fs->p -= fs->point_col - r;
-	      break;
-	    }
-	}
-      else
-	{
-	  /* Do word wrap.  Go to the column just past the maximum line
-	     width and scan back for the beginning of the word there.
-	     Then insert a line break.  */
-
-	  char *p, *nextline;
-	  int i;
-
-	  p = buf + (r + 1 - fs->point_col);
-	  while (p >= buf && !isblank (*p))
-	    --p;
-	  nextline = p + 1;	/* This will begin the next line.  */
-
-	  if (nextline > buf)
-	    {
-	      /* Swallow separating blanks.  */
-	      if (p >= buf)
-		do
-		  --p;
-		while (p >= buf && isblank (*p));
-	      nl = p + 1;	/* The newline will replace the first blank. */
-	    }
-	  else
-	    {
-	      /* A single word that is greater than the maximum line width.
-		 Oh well.  Put it on an overlong line by itself.  */
-	      p = buf + (r + 1 - fs->point_col);
-	      /* Find the end of the long word.  */
-	      do
-		++p;
-	      while (p < nl && !isblank (*p));
-	      if (p == nl)
-		{
-		  /* It already ends a line.  No fussing required.  */
-		  fs->point_col = 0;
-		  buf = nl + 1;
-		  continue;
-		}
-	      /* We will move the newline to replace the first blank.  */
-	      nl = p;
-	      /* Swallow separating blanks.  */
-	      do
-		++p;
-	      while (isblank (*p));
-	      /* The next line will start here.  */
-	      nextline = p;
-	    }
-
-	  /* Note: There are a bunch of tests below for
-	     NEXTLINE == BUF + LEN + 1; this case is where NL happens to fall
-	     at the end of the buffer, and NEXTLINE is in fact empty (and so
-	     we need not be careful to maintain its contents).  */
-
-	  if (nextline == buf + len + 1
-	      ? fs->end - nl < fs->wmargin + 1
-	      : nextline - (nl + 1) < fs->wmargin)
-	    {
-	      /* The margin needs more blanks than we removed.  */
-	      if (fs->end - fs->p > fs->wmargin + 1)
-		/* Make some space for them.  */
-		{
-		  size_t mv = fs->p - nextline;
-		  memmove (nl + 1 + fs->wmargin, nextline, mv);
-		  nextline = nl + 1 + fs->wmargin;
-		  len = nextline + mv - buf;
-		  *nl++ = '\n';
-		}
-	      else
-		/* Output the first line so we can use the space.  */
-		{
-		  if (nl > fs->buf)
-		    FWRITE_UNLOCKED (fs->buf, 1, nl - fs->buf, fs->stream);
-		  PUTC_UNLOCKED ('\n', fs->stream);
-		  len += buf - fs->buf;
-		  nl = buf = fs->buf;
-		}
-	    }
-	  else
-	    /* We can fit the newline and blanks in before
-	       the next word.  */
-	    *nl++ = '\n';
-
-	  if (nextline - nl >= fs->wmargin
-	      || (nextline == buf + len + 1 && fs->end - nextline >= fs->wmargin))
-	    /* Add blanks up to the wrap margin column.  */
-	    for (i = 0; i < fs->wmargin; ++i)
-	      *nl++ = ' ';
-	  else
-	    for (i = 0; i < fs->wmargin; ++i)
-	      PUTC_UNLOCKED (' ', fs->stream);
-
-	  /* Copy the tail of the original buffer into the current buffer
-	     position.  */
-	  if (nl < nextline)
-	    memmove (nl, nextline, buf + len - nextline);
-	  len -= nextline - buf;
-
-	  /* Continue the scan on the remaining lines in the buffer.  */
-	  buf = nl;
-
-	  /* Restore bufp to include all the remaining text.  */
-	  fs->p = nl + len;
-
-	  /* Reset the counter of what has been output this line.  If wmargin
-	     is 0, we want to avoid the lmargin getting added, so we set
-	     point_col to a magic value of -1 in that case.  */
-	  fs->point_col = fs->wmargin ? fs->wmargin : -1;
-	}
-    }
-
-  /* Remember that we've scanned as far as the end of the buffer.  */
-  fs->point_offs = fs->p - fs->buf;
-}
-
-/* Ensure that FS has space for AMOUNT more bytes in its buffer, either by
-   growing the buffer, or by flushing it.  True is returned iff we succeed. */
-int
-__argp_fmtstream_ensure (struct argp_fmtstream *fs, size_t amount)
-{
-  if ((size_t) (fs->end - fs->p) < amount)
-    {
-      ssize_t wrote;
-
-      /* Flush FS's buffer.  */
-      __argp_fmtstream_update (fs);
-
-      wrote = FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-      if (wrote == fs->p - fs->buf)
-	{
-	  fs->p = fs->buf;
-	  fs->point_offs = 0;
-	}
-      else
-	{
-	  fs->p -= wrote;
-	  fs->point_offs -= wrote;
-	  memmove (fs->buf, fs->buf + wrote, fs->p - fs->buf);
-	  return 0;
-	}
-
-      if ((size_t) (fs->end - fs->buf) < amount)
-	/* Gotta grow the buffer.  */
-	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
-
-	  if (! new_buf)
-	    {
-	      __set_errno (ENOMEM);
-	      return 0;
-	    }
-
-	  fs->buf = new_buf;
-	  fs->end = new_buf + new_size;
-	  fs->p = fs->buf;
-	}
-    }
-
-  return 1;
-}
-
-ssize_t
-__argp_fmtstream_printf (struct argp_fmtstream *fs, const char *fmt, ...)
-{
-  size_t out;
-  size_t avail;
-  size_t size_guess = PRINTF_SIZE_GUESS; /* How much space to reserve. */
-
-  do
-    {
-      va_list args;
-
-      if (! __argp_fmtstream_ensure (fs, size_guess))
-	return -1;
-
-      va_start (args, fmt);
-      avail = fs->end - fs->p;
-      out = __vsnprintf (fs->p, avail, fmt, args);
-      va_end (args);
-      if (out >= avail)
-	size_guess = out + 1;
-    }
-  while (out >= avail);
-
-  fs->p += out;
-
-  return out;
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_printf, argp_fmtstream_printf)
-#endif
-
-/* Duplicate the inline definitions in argp-fmtstream.h, for compilers
- * that don't do inlining. */
-size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#endif /* !ARGP_FMTSTREAM_USE_LINEWRAP */
diff --git a/argp-standalone/argp-fmtstream.h b/argp-standalone/argp-fmtstream.h
deleted file mode 100644
index e797b119ebd..00000000000
--- a/argp-standalone/argp-fmtstream.h
+++ /dev/null
@@ -1,319 +0,0 @@
-/* Word-wrapping and line-truncating streams.
-   Copyright (C) 1997, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  If the system does have it, it is just a wrapper for
-   that.  This header file is only used internally while compiling argp, and
-   shouldn't be installed.  */
-
-#ifndef _ARGP_FMTSTREAM_H
-#define _ARGP_FMTSTREAM_H
-
-#include <stdio.h>
-#include <string.h>
-
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#else
-/* This is a kludge to make the code compile on windows. Perhaps it
-   would be better to just replace ssize_t with int through out the
-   code. */
-# define ssize_t int
-#endif
-
-#if _LIBC || (defined (HAVE_FLOCKFILE) && defined(HAVE_PUTC_UNLOCKED) \
-     && defined (HAVE_FPUTS_UNLOCKED) && defined (HAVE_FWRITE_UNLOCKED) )
-/* Use locking funxtions */
-# define FLOCKFILE(f) flockfile(f)
-# define FUNLOCKFILE(f) funlockfile(f)
-# define PUTC_UNLOCKED(c, f) putc_unlocked((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs_unlocked((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite_unlocked((b), (s), (n), (f))
-#else
-/* Disable stdio locking */
-# define FLOCKFILE(f)
-# define FUNLOCKFILE(f)
-# define PUTC_UNLOCKED(c, f) putc((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite((b), (s), (n), (f))
-#endif /* No thread safe i/o */
-
-#if    (_LIBC - 0 && !defined (USE_IN_LIBIO)) \
-    || (defined (__GNU_LIBRARY__) && defined (HAVE_LINEWRAP_H))
-/* line_wrap_stream is available, so use that.  */
-#define ARGP_FMTSTREAM_USE_LINEWRAP
-#endif
-
-#ifdef ARGP_FMTSTREAM_USE_LINEWRAP
-/* Just be a simple wrapper for line_wrap_stream; the semantics are
-   *slightly* different, as line_wrap_stream doesn't actually make a new
-   object, it just modifies the given stream (reversibly) to do
-   line-wrapping.  Since we control who uses this code, it doesn't matter.  */
-
-#include <linewrap.h>
-
-typedef FILE *argp_fmtstream_t;
-
-#define argp_make_fmtstream line_wrap_stream
-#define __argp_make_fmtstream line_wrap_stream
-#define argp_fmtstream_free line_unwrap_stream
-#define __argp_fmtstream_free line_unwrap_stream
-
-#define __argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define __argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define __argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define __argp_fmtstream_printf fprintf
-#define argp_fmtstream_printf fprintf
-
-#define __argp_fmtstream_lmargin line_wrap_lmargin
-#define argp_fmtstream_lmargin line_wrap_lmargin
-#define __argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define __argp_fmtstream_rmargin line_wrap_rmargin
-#define argp_fmtstream_rmargin line_wrap_rmargin
-#define __argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define __argp_fmtstream_wmargin line_wrap_wmargin
-#define argp_fmtstream_wmargin line_wrap_wmargin
-#define __argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define __argp_fmtstream_point line_wrap_point
-#define argp_fmtstream_point line_wrap_point
-
-#else /* !ARGP_FMTSTREAM_USE_LINEWRAP */
-/* Guess we have to define our own version.  */
-
-#ifndef __const
-#define __const const
-#endif
-
-
-struct argp_fmtstream
-{
-  FILE *stream;			/* The stream we're outputting to.  */
-
-  size_t lmargin, rmargin;	/* Left and right margins.  */
-  ssize_t wmargin;		/* Margin to wrap to, or -1 to truncate.  */
-
-  /* Point in buffer to which we've processed for wrapping, but not output.  */
-  size_t point_offs;
-  /* Output column at POINT_OFFS, or -1 meaning 0 but don't add lmargin.  */
-  ssize_t point_col;
-
-  char *buf;			/* Output buffer.  */
-  char *p;			/* Current end of text in BUF. */
-  char *end;			/* Absolute end of BUF.  */
-};
-
-typedef struct argp_fmtstream *argp_fmtstream_t;
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-extern argp_fmtstream_t __argp_make_fmtstream (FILE *__stream,
-					       size_t __lmargin,
-					       size_t __rmargin,
-					       ssize_t __wmargin);
-extern argp_fmtstream_t argp_make_fmtstream (FILE *__stream,
-					     size_t __lmargin,
-					     size_t __rmargin,
-					     ssize_t __wmargin);
-
-/* Flush __FS to its stream, and free it (but don't close the stream).  */
-extern void __argp_fmtstream_free (argp_fmtstream_t __fs);
-extern void argp_fmtstream_free (argp_fmtstream_t __fs);
-
-extern ssize_t __argp_fmtstream_printf (argp_fmtstream_t __fs,
-				       __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-extern ssize_t argp_fmtstream_printf (argp_fmtstream_t __fs,
-				      __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-
-extern int __argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-extern int argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-
-extern int __argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-extern int argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-
-extern size_t __argp_fmtstream_write (argp_fmtstream_t __fs,
-				      __const char *__str, size_t __len);
-extern size_t argp_fmtstream_write (argp_fmtstream_t __fs,
-				    __const char *__str, size_t __len);
-
-/* Access macros for various bits of state.  */
-#define argp_fmtstream_lmargin(__fs) ((__fs)->lmargin)
-#define argp_fmtstream_rmargin(__fs) ((__fs)->rmargin)
-#define argp_fmtstream_wmargin(__fs) ((__fs)->wmargin)
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* Set __FS's left margin to LMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					  size_t __lmargin);
-extern size_t __argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					    size_t __lmargin);
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					  size_t __rmargin);
-extern size_t __argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					    size_t __rmargin);
-
-/* Set __FS's wrap margin to __WMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					  size_t __wmargin);
-extern size_t __argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					    size_t __wmargin);
-
-/* Return the column number of the current output point in __FS.  */
-extern size_t argp_fmtstream_point (argp_fmtstream_t __fs);
-extern size_t __argp_fmtstream_point (argp_fmtstream_t __fs);
-
-/* Internal routines.  */
-extern void _argp_fmtstream_update (argp_fmtstream_t __fs);
-extern void __argp_fmtstream_update (argp_fmtstream_t __fs);
-extern int _argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-extern int __argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-
-#ifdef __OPTIMIZE__
-/* Inline versions of above routines.  */
-
-#if !_LIBC
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#define __argp_fmtstream_write argp_fmtstream_write
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#define __argp_fmtstream_point argp_fmtstream_point
-#define __argp_fmtstream_update _argp_fmtstream_update
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#endif
-
-#ifndef ARGP_FS_EI
-#define ARGP_FS_EI extern inline
-#endif
-
-ARGP_FS_EI size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#if !_LIBC
-#undef __argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#undef __argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#undef __argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#endif
-
-#endif /* __OPTIMIZE__ */
-
-#endif /* ARGP_FMTSTREAM_USE_LINEWRAP */
-
-#endif /* argp-fmtstream.h */
diff --git a/argp-standalone/argp-help.c b/argp-standalone/argp-help.c
deleted file mode 100644
index ced78c4cb26..00000000000
--- a/argp-standalone/argp-help.c
+++ /dev/null
@@ -1,1849 +0,0 @@
-/* Hierarchial argument parsing help output
-   Copyright (C) 1995,96,97,98,99,2000, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <ctype.h>
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-# endif
-#endif
-
-#include "argp.h"
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-
-#ifndef _LIBC
-# ifndef __strchrnul
-#  define __strchrnul strchrnul
-# endif
-# ifndef __mempcpy
-#  define __mempcpy mempcpy
-# endif
-/* We need to use a different name, as __strndup is likely a macro. */
-# define STRNDUP strndup
-# if HAVE_STRERROR
-#  define STRERROR strerror
-# else
-#  define STRERROR(x) (sys_errlist[x])
-# endif
-#else /* _LIBC */
-# define FLOCKFILE __flockfile
-# define FUNLOCKFILE __funlockfile
-# define STRNDUP __strndup
-# define STRERROR strerror
-#endif
-
-#if !_LIBC
-# if !HAVE_STRNDUP
-char *strndup (const char *s, size_t size);
-# endif /* !HAVE_STRNDUP */
-
-# if !HAVE_MEMPCPY
-void *mempcpy (void *to, const void *from, size_t size);
-# endif /* !HAVE_MEMPCPY */
-
-# if !HAVE_STRCHRNUL
-char *strchrnul(const char *s, int c);
-# endif /* !HAVE_STRCHRNUL */
-
-# if !HAVE_STRCASECMP
-int strcasecmp(const char *s1, const char *s2);
-#endif
-
-#endif /* !_LIBC */
-
-
-/* User-selectable (using an environment variable) formatting parameters.
-
-   These may be specified in an environment variable called `ARGP_HELP_FMT',
-   with a contents like:  VAR1=VAL1,VAR2=VAL2,BOOLVAR2,no-BOOLVAR2
-   Where VALn must be a positive integer.  The list of variables is in the
-   UPARAM_NAMES vector, below.  */
-
-/* Default parameters.  */
-#define DUP_ARGS      0		/* True if option argument can be duplicated. */
-#define DUP_ARGS_NOTE 1		/* True to print a note about duplicate args. */
-#define SHORT_OPT_COL 2		/* column in which short options start */
-#define LONG_OPT_COL  6		/* column in which long options start */
-#define DOC_OPT_COL   2		/* column in which doc options start */
-#define OPT_DOC_COL  29		/* column in which option text starts */
-#define HEADER_COL    1		/* column in which group headers are printed */
-#define USAGE_INDENT 12		/* indentation of wrapped usage lines */
-#define RMARGIN      79		/* right margin used for wrapping */
-
-/* User-selectable (using an environment variable) formatting parameters.
-   They must all be of type `int' for the parsing code to work.  */
-struct uparams
-{
-  /* If true, arguments for an option are shown with both short and long
-     options, even when a given option has both, e.g. `-x ARG, --longx=ARG'.
-     If false, then if an option has both, the argument is only shown with
-     the long one, e.g., `-x, --longx=ARG', and a message indicating that
-     this really means both is printed below the options.  */
-  int dup_args;
-
-  /* This is true if when DUP_ARGS is false, and some duplicate arguments have
-     been suppressed, an explanatory message should be printed.  */
-  int dup_args_note;
-
-  /* Various output columns.  */
-  int short_opt_col;
-  int long_opt_col;
-  int doc_opt_col;
-  int opt_doc_col;
-  int header_col;
-  int usage_indent;
-  int rmargin;
-
-  int valid;			/* True when the values in here are valid.  */
-};
-
-/* This is a global variable, as user options are only ever read once.  */
-static struct uparams uparams = {
-  DUP_ARGS, DUP_ARGS_NOTE,
-  SHORT_OPT_COL, LONG_OPT_COL, DOC_OPT_COL, OPT_DOC_COL, HEADER_COL,
-  USAGE_INDENT, RMARGIN,
-  0
-};
-
-/* A particular uparam, and what the user name is.  */
-struct uparam_name
-{
-  const char *name;		/* User name.  */
-  int is_bool;			/* Whether it's `boolean'.  */
-  size_t uparams_offs;		/* Location of the (int) field in UPARAMS.  */
-};
-
-/* The name-field mappings we know about.  */
-static const struct uparam_name uparam_names[] =
-{
-  { "dup-args",       1, offsetof (struct uparams, dup_args) },
-  { "dup-args-note",  1, offsetof (struct uparams, dup_args_note) },
-  { "short-opt-col",  0, offsetof (struct uparams, short_opt_col) },
-  { "long-opt-col",   0, offsetof (struct uparams, long_opt_col) },
-  { "doc-opt-col",    0, offsetof (struct uparams, doc_opt_col) },
-  { "opt-doc-col",    0, offsetof (struct uparams, opt_doc_col) },
-  { "header-col",     0, offsetof (struct uparams, header_col) },
-  { "usage-indent",   0, offsetof (struct uparams, usage_indent) },
-  { "rmargin",        0, offsetof (struct uparams, rmargin) },
-  { 0, 0, 0 }
-};
-
-/* Read user options from the environment, and fill in UPARAMS appropiately.  */
-static void
-fill_in_uparams (const struct argp_state *state)
-{
-
-  const char *var = getenv ("ARGP_HELP_FMT");
-
-#define SKIPWS(p) do { while (isspace (*p)) p++; } while (0);
-
-  if (var)
-    /* Parse var. */
-    while (*var)
-      {
-	SKIPWS (var);
-
-	if (isalpha (*var))
-	  {
-	    size_t var_len;
-	    const struct uparam_name *un;
-	    int unspec = 0, val = 0;
-	    const char *arg = var;
-
-	    while (isalnum (*arg) || *arg == '-' || *arg == '_')
-	      arg++;
-	    var_len = arg - var;
-
-	    SKIPWS (arg);
-
-	    if (*arg == '\0' || *arg == ',')
-	      unspec = 1;
-	    else if (*arg == '=')
-	      {
-		arg++;
-		SKIPWS (arg);
-	      }
-
-	    if (unspec)
-	      {
-		if (var[0] == 'n' && var[1] == 'o' && var[2] == '-')
-		  {
-		    val = 0;
-		    var += 3;
-		    var_len -= 3;
-		  }
-		else
-		  val = 1;
-	      }
-	    else if (isdigit (*arg))
-	      {
-		val = atoi (arg);
-		while (isdigit (*arg))
-		  arg++;
-		SKIPWS (arg);
-	      }
-
-	    for (un = uparam_names; un->name; un++)
-	      if (strlen (un->name) == var_len
-		  && strncmp (var, un->name, var_len) == 0)
-		{
-		  if (unspec && !un->is_bool)
-		    __argp_failure (state, 0, 0,
-				    dgettext (state->root_argp->argp_domain, "\
-%.*s: ARGP_HELP_FMT parameter requires a value"),
-				    (int) var_len, var);
-		  else
-		    *(int *)((char *)&uparams + un->uparams_offs) = val;
-		  break;
-		}
-	    if (! un->name)
-	      __argp_failure (state, 0, 0,
-			      dgettext (state->root_argp->argp_domain, "\
-%.*s: Unknown ARGP_HELP_FMT parameter"),
-			      (int) var_len, var);
-
-	    var = arg;
-	    if (*var == ',')
-	      var++;
-	  }
-	else if (*var)
-	  {
-	    __argp_failure (state, 0, 0,
-			    dgettext (state->root_argp->argp_domain,
-				      "Garbage in ARGP_HELP_FMT: %s"), var);
-	    break;
-	  }
-      }
-}
-
-/* Returns true if OPT hasn't been marked invisible.  Visibility only affects
-   whether OPT is displayed or used in sorting, not option shadowing.  */
-#define ovisible(opt) (! ((opt)->flags & OPTION_HIDDEN))
-
-/* Returns true if OPT is an alias for an earlier option.  */
-#define oalias(opt) ((opt)->flags & OPTION_ALIAS)
-
-/* Returns true if OPT is an documentation-only entry.  */
-#define odoc(opt) ((opt)->flags & OPTION_DOC)
-
-/* Returns true if OPT is the end-of-list marker for a list of options.  */
-#define oend(opt) __option_is_end (opt)
-
-/* Returns true if OPT has a short option.  */
-#define oshort(opt) __option_is_short (opt)
-
-/*
-   The help format for a particular option is like:
-
-     -xARG, -yARG, --long1=ARG, --long2=ARG        Documentation...
-
-   Where ARG will be omitted if there's no argument, for this option, or
-   will be surrounded by "[" and "]" appropiately if the argument is
-   optional.  The documentation string is word-wrapped appropiately, and if
-   the list of options is long enough, it will be started on a separate line.
-   If there are no short options for a given option, the first long option is
-   indented slighly in a way that's supposed to make most long options appear
-   to be in a separate column.
-
-   For example, the following output (from ps):
-
-     -p PID, --pid=PID          List the process PID
-	 --pgrp=PGRP            List processes in the process group PGRP
-     -P, -x, --no-parent        Include processes without parents
-     -Q, --all-fields           Don't elide unusable fields (normally if there's
-				some reason ps can't print a field for any
-				process, it's removed from the output entirely)
-     -r, --reverse, --gratuitously-long-reverse-option
-				Reverse the order of any sort
-	 --session[=SID]        Add the processes from the session SID (which
-				defaults to the sid of the current process)
-
-    Here are some more options:
-     -f ZOT, --foonly=ZOT       Glork a foonly
-     -z, --zaza                 Snit a zar
-
-     -?, --help                 Give this help list
-	 --usage                Give a short usage message
-     -V, --version              Print program version
-
-   The struct argp_option array for the above could look like:
-
-   {
-     {"pid",       'p',      "PID",  0, "List the process PID"},
-     {"pgrp",      OPT_PGRP, "PGRP", 0, "List processes in the process group PGRP"},
-     {"no-parent", 'P',	      0,     0, "Include processes without parents"},
-     {0,           'x',       0,     OPTION_ALIAS},
-     {"all-fields",'Q',       0,     0, "Don't elide unusable fields (normally"
-                                        " if there's some reason ps can't"
-					" print a field for any process, it's"
-                                        " removed from the output entirely)" },
-     {"reverse",   'r',       0,     0, "Reverse the order of any sort"},
-     {"gratuitously-long-reverse-option", 0, 0, OPTION_ALIAS},
-     {"session",   OPT_SESS,  "SID", OPTION_ARG_OPTIONAL,
-                                        "Add the processes from the session"
-					" SID (which defaults to the sid of"
-					" the current process)" },
-
-     {0,0,0,0, "Here are some more options:"},
-     {"foonly", 'f', "ZOT", 0, "Glork a foonly"},
-     {"zaza", 'z', 0, 0, "Snit a zar"},
-
-     {0}
-   }
-
-   Note that the last three options are automatically supplied by argp_parse,
-   unless you tell it not to with ARGP_NO_HELP.
-
-*/
-
-/* Returns true if CH occurs between BEG and END.  */
-static int
-find_char (char ch, char *beg, char *end)
-{
-  while (beg < end)
-    if (*beg == ch)
-      return 1;
-    else
-      beg++;
-  return 0;
-}
-
-struct hol_cluster;		/* fwd decl */
-
-struct hol_entry
-{
-  /* First option.  */
-  const struct argp_option *opt;
-  /* Number of options (including aliases).  */
-  unsigned num;
-
-  /* A pointers into the HOL's short_options field, to the first short option
-     letter for this entry.  The order of the characters following this point
-     corresponds to the order of options pointed to by OPT, and there are at
-     most NUM.  A short option recorded in a option following OPT is only
-     valid if it occurs in the right place in SHORT_OPTIONS (otherwise it's
-     probably been shadowed by some other entry).  */
-  char *short_options;
-
-  /* Entries are sorted by their group first, in the order:
-       1, 2, ..., n, 0, -m, ..., -2, -1
-     and then alphabetically within each group.  The default is 0.  */
-  int group;
-
-  /* The cluster of options this entry belongs to, or 0 if none.  */
-  struct hol_cluster *cluster;
-
-  /* The argp from which this option came.  */
-  const struct argp *argp;
-};
-
-/* A cluster of entries to reflect the argp tree structure.  */
-struct hol_cluster
-{
-  /* A descriptive header printed before options in this cluster.  */
-  const char *header;
-
-  /* Used to order clusters within the same group with the same parent,
-     according to the order in which they occurred in the parent argp's child
-     list.  */
-  int index;
-
-  /* How to sort this cluster with respect to options and other clusters at the
-     same depth (clusters always follow options in the same group).  */
-  int group;
-
-  /* The cluster to which this cluster belongs, or 0 if it's at the base
-     level.  */
-  struct hol_cluster *parent;
-
-  /* The argp from which this cluster is (eventually) derived.  */
-  const struct argp *argp;
-
-  /* The distance this cluster is from the root.  */
-  int depth;
-
-  /* Clusters in a given hol are kept in a linked list, to make freeing them
-     possible.  */
-  struct hol_cluster *next;
-};
-
-/* A list of options for help.  */
-struct hol
-{
-  /* An array of hol_entry's.  */
-  struct hol_entry *entries;
-  /* The number of entries in this hol.  If this field is zero, the others
-     are undefined.  */
-  unsigned num_entries;
-
-  /* A string containing all short options in this HOL.  Each entry contains
-     pointers into this string, so the order can't be messed with blindly.  */
-  char *short_options;
-
-  /* Clusters of entries in this hol.  */
-  struct hol_cluster *clusters;
-};
-
-/* Create a struct hol from the options in ARGP.  CLUSTER is the
-   hol_cluster in which these entries occur, or 0, if at the root.  */
-static struct hol *
-make_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  char *so;
-  const struct argp_option *o;
-  const struct argp_option *opts = argp->options;
-  struct hol_entry *entry;
-  unsigned num_short_options = 0;
-  struct hol *hol = malloc (sizeof (struct hol));
-
-  assert (hol);
-
-  hol->num_entries = 0;
-  hol->clusters = 0;
-
-  if (opts)
-    {
-      int cur_group = 0;
-
-      /* The first option must not be an alias.  */
-      assert (! oalias (opts));
-
-      /* Calculate the space needed.  */
-      for (o = opts; ! oend (o); o++)
-	{
-	  if (! oalias (o))
-	    hol->num_entries++;
-	  if (oshort (o))
-	    num_short_options++;	/* This is an upper bound.  */
-	}
-
-      hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
-      hol->short_options = malloc (num_short_options + 1);
-
-      assert (hol->entries && hol->short_options);
-
-      /* Fill in the entries.  */
-      so = hol->short_options;
-      for (o = opts, entry = hol->entries; ! oend (o); entry++)
-	{
-	  entry->opt = o;
-	  entry->num = 0;
-	  entry->short_options = so;
-	  entry->group = cur_group =
-	    o->group
-	    ? o->group
-	    : ((!o->name && !o->key)
-	       ? cur_group + 1
-	       : cur_group);
-	  entry->cluster = cluster;
-	  entry->argp = argp;
-
-	  do
-	    {
-	      entry->num++;
-	      if (oshort (o) && ! find_char (o->key, hol->short_options, so))
-		/* O has a valid short option which hasn't already been used.*/
-		*so++ = o->key;
-	      o++;
-	    }
-	  while (! oend (o) && oalias (o));
-	}
-      *so = '\0';		/* null terminated so we can find the length */
-    }
-
-  return hol;
-}
-
-/* Add a new cluster to HOL, with the given GROUP and HEADER (taken from the
-   associated argp child list entry), INDEX, and PARENT, and return a pointer
-   to it.  ARGP is the argp that this cluster results from.  */
-static struct hol_cluster *
-hol_add_cluster (struct hol *hol, int group, const char *header, int index,
-		 struct hol_cluster *parent, const struct argp *argp)
-{
-  struct hol_cluster *cl = malloc (sizeof (struct hol_cluster));
-  if (cl)
-    {
-      cl->group = group;
-      cl->header = header;
-
-      cl->index = index;
-      cl->parent = parent;
-      cl->argp = argp;
-      cl->depth = parent ? parent->depth + 1 : 0;
-
-      cl->next = hol->clusters;
-      hol->clusters = cl;
-    }
-  return cl;
-}
-
-/* Free HOL and any resources it uses.  */
-static void
-hol_free (struct hol *hol)
-{
-  struct hol_cluster *cl = hol->clusters;
-
-  while (cl)
-    {
-      struct hol_cluster *next = cl->next;
-      free (cl);
-      cl = next;
-    }
-
-  if (hol->num_entries > 0)
-    {
-      free (hol->entries);
-      free (hol->short_options);
-    }
-
-  free (hol);
-}
-
-static inline int
-hol_entry_short_iterate (const struct hol_entry *entry,
-			 int (*func)(const struct argp_option *opt,
-				     const struct argp_option *real,
-				     const char *domain, void *cookie),
-			 const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-  char *so = entry->short_options;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (oshort (opt) && *so == opt->key)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-	so++;
-      }
-
-  return val;
-}
-
-static inline int
-hol_entry_long_iterate (const struct hol_entry *entry,
-			int (*func)(const struct argp_option *opt,
-				    const struct argp_option *real,
-				    const char *domain, void *cookie),
-			const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (opt->name)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-      }
-
-  return val;
-}
-
-/* Iterator that returns true for the first short option.  */
-static inline int
-until_short (const struct argp_option *opt, const struct argp_option *real UNUSED,
-	     const char *domain UNUSED, void *cookie UNUSED)
-{
-  return oshort (opt) ? opt->key : 0;
-}
-
-/* Returns the first valid short option in ENTRY, or 0 if there is none.  */
-static char
-hol_entry_first_short (const struct hol_entry *entry)
-{
-  return hol_entry_short_iterate (entry, until_short,
-				  entry->argp->argp_domain, 0);
-}
-
-/* Returns the first valid long option in ENTRY, or 0 if there is none.  */
-static const char *
-hol_entry_first_long (const struct hol_entry *entry)
-{
-  const struct argp_option *opt;
-  unsigned num;
-  for (opt = entry->opt, num = entry->num; num > 0; opt++, num--)
-    if (opt->name && ovisible (opt))
-      return opt->name;
-  return 0;
-}
-
-/* Returns the entry in HOL with the long option name NAME, or 0 if there is
-   none.  */
-static struct hol_entry *
-hol_find_entry (struct hol *hol, const char *name)
-{
-  struct hol_entry *entry = hol->entries;
-  unsigned num_entries = hol->num_entries;
-
-  while (num_entries-- > 0)
-    {
-      const struct argp_option *opt = entry->opt;
-      unsigned num_opts = entry->num;
-
-      while (num_opts-- > 0)
-	if (opt->name && ovisible (opt) && strcmp (opt->name, name) == 0)
-	  return entry;
-	else
-	  opt++;
-
-      entry++;
-    }
-
-  return 0;
-}
-
-/* If an entry with the long option NAME occurs in HOL, set it's special
-   sort position to GROUP.  */
-static void
-hol_set_group (struct hol *hol, const char *name, int group)
-{
-  struct hol_entry *entry = hol_find_entry (hol, name);
-  if (entry)
-    entry->group = group;
-}
-
-/* Order by group:  0, 1, 2, ..., n, -m, ..., -2, -1.
-   EQ is what to return if GROUP1 and GROUP2 are the same.  */
-static int
-group_cmp (int group1, int group2, int eq)
-{
-  if (group1 == group2)
-    return eq;
-  else if ((group1 < 0 && group2 < 0) || (group1 >= 0 && group2 >= 0))
-    return group1 - group2;
-  else
-    return group2 - group1;
-}
-
-/* Compare clusters CL1 & CL2 by the order that they should appear in
-   output.  */
-static int
-hol_cluster_cmp (const struct hol_cluster *cl1, const struct hol_cluster *cl2)
-{
-  /* If one cluster is deeper than the other, use its ancestor at the same
-     level, so that finding the common ancestor is straightforward.  */
-  while (cl1->depth < cl2->depth)
-    cl1 = cl1->parent;
-  while (cl2->depth < cl1->depth)
-    cl2 = cl2->parent;
-
-  /* Now reduce both clusters to their ancestors at the point where both have
-     a common parent; these can be directly compared.  */
-  while (cl1->parent != cl2->parent)
-    cl1 = cl1->parent, cl2 = cl2->parent;
-
-  return group_cmp (cl1->group, cl2->group, cl2->index - cl1->index);
-}
-
-/* Return the ancestor of CL that's just below the root (i.e., has a parent
-   of 0).  */
-static struct hol_cluster *
-hol_cluster_base (struct hol_cluster *cl)
-{
-  while (cl->parent)
-    cl = cl->parent;
-  return cl;
-}
-
-/* Return true if CL1 is a child of CL2.  */
-static int
-hol_cluster_is_child (const struct hol_cluster *cl1,
-		      const struct hol_cluster *cl2)
-{
-  while (cl1 && cl1 != cl2)
-    cl1 = cl1->parent;
-  return cl1 == cl2;
-}
-
-/* Given the name of a OPTION_DOC option, modifies NAME to start at the tail
-   that should be used for comparisons, and returns true iff it should be
-   treated as a non-option.  */
-
-/* FIXME: Can we use unsigned char * for the argument? */
-static int
-canon_doc_option (const char **name)
-{
-  int non_opt;
-  /* Skip initial whitespace.  */
-  while (isspace ( (unsigned char) **name))
-    (*name)++;
-  /* Decide whether this looks like an option (leading `-') or not.  */
-  non_opt = (**name != '-');
-  /* Skip until part of name used for sorting.  */
-  while (**name && !isalnum ( (unsigned char) **name))
-    (*name)++;
-  return non_opt;
-}
-
-/* Order ENTRY1 & ENTRY2 by the order which they should appear in a help
-   listing.  */
-static int
-hol_entry_cmp (const struct hol_entry *entry1,
-	       const struct hol_entry *entry2)
-{
-  /* The group numbers by which the entries should be ordered; if either is
-     in a cluster, then this is just the group within the cluster.  */
-  int group1 = entry1->group, group2 = entry2->group;
-
-  if (entry1->cluster != entry2->cluster)
-    {
-      /* The entries are not within the same cluster, so we can't compare them
-	 directly, we have to use the appropiate clustering level too.  */
-      if (! entry1->cluster)
-	/* ENTRY1 is at the `base level', not in a cluster, so we have to
-	   compare it's group number with that of the base cluster in which
-	   ENTRY2 resides.  Note that if they're in the same group, the
-	   clustered option always comes laster.  */
-	return group_cmp (group1, hol_cluster_base (entry2->cluster)->group, -1);
-      else if (! entry2->cluster)
-	/* Likewise, but ENTRY2's not in a cluster.  */
-	return group_cmp (hol_cluster_base (entry1->cluster)->group, group2, 1);
-      else
-	/* Both entries are in clusters, we can just compare the clusters.  */
-	return hol_cluster_cmp (entry1->cluster, entry2->cluster);
-    }
-  else if (group1 == group2)
-    /* The entries are both in the same cluster and group, so compare them
-       alphabetically.  */
-    {
-      int short1 = hol_entry_first_short (entry1);
-      int short2 = hol_entry_first_short (entry2);
-      int doc1 = odoc (entry1->opt);
-      int doc2 = odoc (entry2->opt);
-      /* FIXME: Can we use unsigned char * instead? */
-      const char *long1 = hol_entry_first_long (entry1);
-      const char *long2 = hol_entry_first_long (entry2);
-
-      if (doc1)
-	doc1 = canon_doc_option (&long1);
-      if (doc2)
-	doc2 = canon_doc_option (&long2);
-
-      if (doc1 != doc2)
-	/* `documentation' options always follow normal options (or
-	   documentation options that *look* like normal options).  */
-	return doc1 - doc2;
-      else if (!short1 && !short2 && long1 && long2)
-	/* Only long options.  */
-	return __strcasecmp (long1, long2);
-      else
-	/* Compare short/short, long/short, short/long, using the first
-	   character of long options.  Entries without *any* valid
-	   options (such as options with OPTION_HIDDEN set) will be put
-	   first, but as they're not displayed, it doesn't matter where
-	   they are.  */
-	{
-	  unsigned char first1 = short1 ? short1 : long1 ? *long1 : 0;
-	  unsigned char first2 = short2 ? short2 : long2 ? *long2 : 0;
-#ifdef _tolower
-	  int lower_cmp = _tolower (first1) - _tolower (first2);
-#else
-	  int lower_cmp = tolower (first1) - tolower (first2);
-#endif
-	  /* Compare ignoring case, except when the options are both the
-	     same letter, in which case lower-case always comes first.  */
-	  /* NOTE: The subtraction below does the right thing
-	     even with eight-bit chars: first1 and first2 are
-	     converted to int *before* the subtraction. */
-	  return lower_cmp ? lower_cmp : first2 - first1;
-	}
-    }
-  else
-    /* Within the same cluster, but not the same group, so just compare
-       groups.  */
-    return group_cmp (group1, group2, 0);
-}
-
-/* Version of hol_entry_cmp with correct signature for qsort.  */
-static int
-hol_entry_qcmp (const void *entry1_v, const void *entry2_v)
-{
-  return hol_entry_cmp (entry1_v, entry2_v);
-}
-
-/* Sort HOL by group and alphabetically by option name (with short options
-   taking precedence over long).  Since the sorting is for display purposes
-   only, the shadowing of options isn't effected.  */
-static void
-hol_sort (struct hol *hol)
-{
-  if (hol->num_entries > 0)
-    qsort (hol->entries, hol->num_entries, sizeof (struct hol_entry),
-	   hol_entry_qcmp);
-}
-
-/* Append MORE to HOL, destroying MORE in the process.  Options in HOL shadow
-   any in MORE with the same name.  */
-static void
-hol_append (struct hol *hol, struct hol *more)
-{
-  struct hol_cluster **cl_end = &hol->clusters;
-
-  /* Steal MORE's cluster list, and add it to the end of HOL's.  */
-  while (*cl_end)
-    cl_end = &(*cl_end)->next;
-  *cl_end = more->clusters;
-  more->clusters = 0;
-
-  /* Merge entries.  */
-  if (more->num_entries > 0)
-    {
-      if (hol->num_entries == 0)
-	{
-	  hol->num_entries = more->num_entries;
-	  hol->entries = more->entries;
-	  hol->short_options = more->short_options;
-	  more->num_entries = 0;	/* Mark MORE's fields as invalid.  */
-	}
-      else
-	/* Append the entries in MORE to those in HOL, taking care to only add
-	   non-shadowed SHORT_OPTIONS values.  */
-	{
-	  unsigned left;
-	  char *so, *more_so;
-	  struct hol_entry *e;
-	  unsigned num_entries = hol->num_entries + more->num_entries;
-	  struct hol_entry *entries =
-	    malloc (num_entries * sizeof (struct hol_entry));
-	  unsigned hol_so_len = strlen (hol->short_options);
-	  char *short_options =
-	    malloc (hol_so_len + strlen (more->short_options) + 1);
-
-	  __mempcpy (__mempcpy (entries, hol->entries,
-				hol->num_entries * sizeof (struct hol_entry)),
-		     more->entries,
-		     more->num_entries * sizeof (struct hol_entry));
-
-	  __mempcpy (short_options, hol->short_options, hol_so_len);
-
-	  /* Fix up the short options pointers from HOL.  */
-	  for (e = entries, left = hol->num_entries; left > 0; e++, left--)
-	    e->short_options += (short_options - hol->short_options);
-
-	  /* Now add the short options from MORE, fixing up its entries
-	     too.  */
-	  so = short_options + hol_so_len;
-	  more_so = more->short_options;
-	  for (left = more->num_entries; left > 0; e++, left--)
-	    {
-	      int opts_left;
-	      const struct argp_option *opt;
-
-	      e->short_options = so;
-
-	      for (opts_left = e->num, opt = e->opt; opts_left; opt++, opts_left--)
-		{
-		  int ch = *more_so;
-		  if (oshort (opt) && ch == opt->key)
-		    /* The next short option in MORE_SO, CH, is from OPT.  */
-		    {
-		      if (! find_char (ch, short_options,
-				       short_options + hol_so_len))
-			/* The short option CH isn't shadowed by HOL's options,
-			   so add it to the sum.  */
-			*so++ = ch;
-		      more_so++;
-		    }
-		}
-	    }
-
-	  *so = '\0';
-
-	  free (hol->entries);
-	  free (hol->short_options);
-
-	  hol->entries = entries;
-	  hol->num_entries = num_entries;
-	  hol->short_options = short_options;
-	}
-    }
-
-  hol_free (more);
-}
-
-/* Inserts enough spaces to make sure STREAM is at column COL.  */
-static void
-indent_to (argp_fmtstream_t stream, unsigned col)
-{
-  int needed = col - __argp_fmtstream_point (stream);
-  while (needed-- > 0)
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* Output to STREAM either a space, or a newline if there isn't room for at
-   least ENSURE characters before the right margin.  */
-static void
-space (argp_fmtstream_t stream, size_t ensure)
-{
-  if (__argp_fmtstream_point (stream) + ensure
-      >= __argp_fmtstream_rmargin (stream))
-    __argp_fmtstream_putc (stream, '\n');
-  else
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* If the option REAL has an argument, we print it in using the printf
-   format REQ_FMT or OPT_FMT depending on whether it's a required or
-   optional argument.  */
-static void
-arg (const struct argp_option *real, const char *req_fmt, const char *opt_fmt,
-     const char *domain UNUSED, argp_fmtstream_t stream)
-{
-  if (real->arg)
-    {
-      if (real->flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, opt_fmt,
-				 dgettext (domain, real->arg));
-      else
-	__argp_fmtstream_printf (stream, req_fmt,
-				 dgettext (domain, real->arg));
-    }
-}
-
-/* Helper functions for hol_entry_help.  */
-
-/* State used during the execution of hol_help.  */
-struct hol_help_state
-{
-  /* PREV_ENTRY should contain the previous entry printed, or 0.  */
-  struct hol_entry *prev_entry;
-
-  /* If an entry is in a different group from the previous one, and SEP_GROUPS
-     is true, then a blank line will be printed before any output. */
-  int sep_groups;
-
-  /* True if a duplicate option argument was suppressed (only ever set if
-     UPARAMS.dup_args is false).  */
-  int suppressed_dup_arg;
-};
-
-/* Some state used while printing a help entry (used to communicate with
-   helper functions).  See the doc for hol_entry_help for more info, as most
-   of the fields are copied from its arguments.  */
-struct pentry_state
-{
-  const struct hol_entry *entry;
-  argp_fmtstream_t stream;
-  struct hol_help_state *hhstate;
-
-  /* True if nothing's been printed so far.  */
-  int first;
-
-  /* If non-zero, the state that was used to print this help.  */
-  const struct argp_state *state;
-};
-
-/* If a user doc filter should be applied to DOC, do so.  */
-static const char *
-filter_doc (const char *doc, int key, const struct argp *argp,
-	    const struct argp_state *state)
-{
-  if (argp->help_filter)
-    /* We must apply a user filter to this output.  */
-    {
-      void *input = __argp_input (argp, state);
-      return (*argp->help_filter) (key, doc, input);
-    }
-  else
-    /* No filter.  */
-    return doc;
-}
-
-/* Prints STR as a header line, with the margin lines set appropiately, and
-   notes the fact that groups should be separated with a blank line.  ARGP is
-   the argp that should dictate any user doc filtering to take place.  Note
-   that the previous wrap margin isn't restored, but the left margin is reset
-   to 0.  */
-static void
-print_header (const char *str, const struct argp *argp,
-	      struct pentry_state *pest)
-{
-  const char *tstr = dgettext (argp->argp_domain, str);
-  const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_HEADER, argp, pest->state);
-
-  if (fstr)
-    {
-      if (*fstr)
-	{
-	  if (pest->hhstate->prev_entry)
-	    /* Precede with a blank line.  */
-	    __argp_fmtstream_putc (pest->stream, '\n');
-	  indent_to (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_lmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_wmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_puts (pest->stream, fstr);
-	  __argp_fmtstream_set_lmargin (pest->stream, 0);
-	  __argp_fmtstream_putc (pest->stream, '\n');
-	}
-
-      pest->hhstate->sep_groups = 1; /* Separate subsequent groups. */
-    }
-
-  if (fstr != tstr)
-    free ((char *) fstr);
-}
-
-/* Inserts a comma if this isn't the first item on the line, and then makes
-   sure we're at least to column COL.  If this *is* the first item on a line,
-   prints any pending whitespace/headers that should precede this line. Also
-   clears FIRST.  */
-static void
-comma (unsigned col, struct pentry_state *pest)
-{
-  if (pest->first)
-    {
-      const struct hol_entry *pe = pest->hhstate->prev_entry;
-      const struct hol_cluster *cl = pest->entry->cluster;
-
-      if (pest->hhstate->sep_groups && pe && pest->entry->group != pe->group)
-	__argp_fmtstream_putc (pest->stream, '\n');
-
-      if (cl && cl->header && *cl->header
-	  && (!pe
-	      || (pe->cluster != cl
-		  && !hol_cluster_is_child (pe->cluster, cl))))
-	/* If we're changing clusters, then this must be the start of the
-	   ENTRY's cluster unless that is an ancestor of the previous one
-	   (in which case we had just popped into a sub-cluster for a bit).
-	   If so, then print the cluster's header line.  */
-	{
-	  int old_wm = __argp_fmtstream_wmargin (pest->stream);
-	  print_header (cl->header, cl->argp, pest);
-	  __argp_fmtstream_set_wmargin (pest->stream, old_wm);
-	}
-
-      pest->first = 0;
-    }
-  else
-    __argp_fmtstream_puts (pest->stream, ", ");
-
-  indent_to (pest->stream, col);
-}
-
-/* Print help for ENTRY to STREAM.  */
-static void
-hol_entry_help (struct hol_entry *entry, const struct argp_state *state,
-		argp_fmtstream_t stream, struct hol_help_state *hhstate)
-{
-  unsigned num;
-  const struct argp_option *real = entry->opt, *opt;
-  char *so = entry->short_options;
-  int have_long_opt = 0;	/* We have any long options.  */
-  /* Saved margins.  */
-  int old_lm = __argp_fmtstream_set_lmargin (stream, 0);
-  int old_wm = __argp_fmtstream_wmargin (stream);
-  /* PEST is a state block holding some of our variables that we'd like to
-     share with helper functions.  */
-
-  /* Decent initializers are a GNU extension, so don't use it here. */
-  struct pentry_state pest;
-  pest.entry = entry;
-  pest.stream = stream;
-  pest.hhstate = hhstate;
-  pest.first = 1;
-  pest.state = state;
-
-  if (! odoc (real))
-    for (opt = real, num = entry->num; num > 0; opt++, num--)
-      if (opt->name && ovisible (opt))
-	{
-	  have_long_opt = 1;
-	  break;
-	}
-
-  /* First emit short options.  */
-  __argp_fmtstream_set_wmargin (stream, uparams.short_opt_col); /* For truly bizarre cases. */
-  for (opt = real, num = entry->num; num > 0; opt++, num--)
-    if (oshort (opt) && opt->key == *so)
-      /* OPT has a valid (non shadowed) short option.  */
-      {
-	if (ovisible (opt))
-	  {
-	    comma (uparams.short_opt_col, &pest);
-	    __argp_fmtstream_putc (stream, '-');
-	    __argp_fmtstream_putc (stream, *so);
-	    if (!have_long_opt || uparams.dup_args)
-	      arg (real, " %s", "[%s]", state->root_argp->argp_domain, stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-	so++;
-      }
-
-  /* Now, long options.  */
-  if (odoc (real))
-    /* A `documentation' option.  */
-    {
-      __argp_fmtstream_set_wmargin (stream, uparams.doc_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.doc_opt_col, &pest);
-	    /* Calling gettext here isn't quite right, since sorting will
-	       have been done on the original; but documentation options
-	       should be pretty rare anyway...  */
-	    __argp_fmtstream_puts (stream,
-				   dgettext (state->root_argp->argp_domain,
-					     opt->name));
-	  }
-    }
-  else
-    /* A real long option.  */
-    {
-      int first_long_opt = 1;
-
-      __argp_fmtstream_set_wmargin (stream, uparams.long_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.long_opt_col, &pest);
-	    __argp_fmtstream_printf (stream, "--%s", opt->name);
-	    if (first_long_opt || uparams.dup_args)
-	      arg (real, "=%s", "[=%s]", state->root_argp->argp_domain,
-		   stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-    }
-
-  /* Next, documentation strings.  */
-  __argp_fmtstream_set_lmargin (stream, 0);
-
-  if (pest.first)
-    {
-      /* Didn't print any switches, what's up?  */
-      if (!oshort (real) && !real->name)
-	/* This is a group header, print it nicely.  */
-	print_header (real->doc, entry->argp, &pest);
-      else
-	/* Just a totally shadowed option or null header; print nothing.  */
-	goto cleanup;		/* Just return, after cleaning up.  */
-    }
-  else
-    {
-      const char *tstr = real->doc ? dgettext (state->root_argp->argp_domain,
-					       real->doc) : 0;
-      const char *fstr = filter_doc (tstr, real->key, entry->argp, state);
-      if (fstr && *fstr)
-	{
-	  unsigned int col = __argp_fmtstream_point (stream);
-
-	  __argp_fmtstream_set_lmargin (stream, uparams.opt_doc_col);
-	  __argp_fmtstream_set_wmargin (stream, uparams.opt_doc_col);
-
-	  if (col > (unsigned int) (uparams.opt_doc_col + 3))
-	    __argp_fmtstream_putc (stream, '\n');
-	  else if (col >= (unsigned int) uparams.opt_doc_col)
-	    __argp_fmtstream_puts (stream, "   ");
-	  else
-	    indent_to (stream, uparams.opt_doc_col);
-
-	  __argp_fmtstream_puts (stream, fstr);
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-
-      /* Reset the left margin.  */
-      __argp_fmtstream_set_lmargin (stream, 0);
-      __argp_fmtstream_putc (stream, '\n');
-    }
-
-  hhstate->prev_entry = entry;
-
-cleanup:
-  __argp_fmtstream_set_lmargin (stream, old_lm);
-  __argp_fmtstream_set_wmargin (stream, old_wm);
-}
-
-/* Output a long help message about the options in HOL to STREAM.  */
-static void
-hol_help (struct hol *hol, const struct argp_state *state,
-	  argp_fmtstream_t stream)
-{
-  unsigned num;
-  struct hol_entry *entry;
-  struct hol_help_state hhstate = { 0, 0, 0 };
-
-  for (entry = hol->entries, num = hol->num_entries; num > 0; entry++, num--)
-    hol_entry_help (entry, state, stream, &hhstate);
-
-  if (hhstate.suppressed_dup_arg && uparams.dup_args_note)
-    {
-      const char *tstr = dgettext (state->root_argp->argp_domain, "\
-Mandatory or optional arguments to long options are also mandatory or \
-optional for any corresponding short options.");
-      const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_DUP_ARGS_NOTE,
-				     state ? state->root_argp : 0, state);
-      if (fstr && *fstr)
-	{
-	  __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, fstr);
-	  __argp_fmtstream_putc (stream, '\n');
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-    }
-}
-
-/* Helper functions for hol_usage.  */
-
-/* If OPT is a short option without an arg, append its key to the string
-   pointer pointer to by COOKIE, and advance the pointer.  */
-static int
-add_argless_short_opt (const struct argp_option *opt,
-		       const struct argp_option *real,
-		       const char *domain UNUSED, void *cookie)
-{
-  char **snao_end = cookie;
-  if (!(opt->arg || real->arg)
-      && !((opt->flags | real->flags) & OPTION_NO_USAGE))
-    *(*snao_end)++ = opt->key;
-  return 0;
-}
-
-/* If OPT is a short option with an arg, output a usage entry for it to the
-   stream pointed at by COOKIE.  */
-static int
-usage_argful_short_opt (const struct argp_option *opt,
-			const struct argp_option *real,
-			const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (arg && !(flags & OPTION_NO_USAGE))
-    {
-      arg = dgettext (domain, arg);
-
-      if (flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, " [-%c[%s]]", opt->key, arg);
-      else
-	{
-	  /* Manually do line wrapping so that it (probably) won't
-	     get wrapped at the embedded space.  */
-	  space (stream, 6 + strlen (arg));
-	  __argp_fmtstream_printf (stream, "[-%c %s]", opt->key, arg);
-	}
-    }
-
-  return 0;
-}
-
-/* Output a usage entry for the long option opt to the stream pointed at by
-   COOKIE.  */
-static int
-usage_long_opt (const struct argp_option *opt,
-		const struct argp_option *real,
-		const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (! (flags & OPTION_NO_USAGE))
-    {
-      if (arg)
-	{
-	  arg = dgettext (domain, arg);
-	  if (flags & OPTION_ARG_OPTIONAL)
-	    __argp_fmtstream_printf (stream, " [--%s[=%s]]", opt->name, arg);
-	  else
-	    __argp_fmtstream_printf (stream, " [--%s=%s]", opt->name, arg);
-	}
-      else
-	__argp_fmtstream_printf (stream, " [--%s]", opt->name);
-    }
-
-  return 0;
-}
-
-/* Print a short usage description for the arguments in HOL to STREAM.  */
-static void
-hol_usage (struct hol *hol, argp_fmtstream_t stream)
-{
-  if (hol->num_entries > 0)
-    {
-      unsigned nentries;
-      struct hol_entry *entry;
-      char *short_no_arg_opts = alloca (strlen (hol->short_options) + 1);
-      char *snao_end = short_no_arg_opts;
-
-      /* First we put a list of short options without arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, add_argless_short_opt,
-				 entry->argp->argp_domain, &snao_end);
-      if (snao_end > short_no_arg_opts)
-	{
-	  *snao_end++ = 0;
-	  __argp_fmtstream_printf (stream, " [-%s]", short_no_arg_opts);
-	}
-
-      /* Now a list of short options *with* arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, usage_argful_short_opt,
-				 entry->argp->argp_domain, stream);
-
-      /* Finally, a list of long options (whew!).  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_long_iterate (entry, usage_long_opt,
-				entry->argp->argp_domain, stream);
-    }
-}
-
-/* Make a HOL containing all levels of options in ARGP.  CLUSTER is the
-   cluster in which ARGP's entries should be clustered, or 0.  */
-static struct hol *
-argp_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  const struct argp_child *child = argp->children;
-  struct hol *hol = make_hol (argp, cluster);
-  if (child)
-    while (child->argp)
-      {
-	struct hol_cluster *child_cluster =
-	  ((child->group || child->header)
-	   /* Put CHILD->argp within its own cluster.  */
-	   ? hol_add_cluster (hol, child->group, child->header,
-			      child - argp->children, cluster, argp)
-	   /* Just merge it into the parent's cluster.  */
-	   : cluster);
-	hol_append (hol, argp_hol (child->argp, child_cluster)) ;
-	child++;
-      }
-  return hol;
-}
-
-/* Calculate how many different levels with alternative args strings exist in
-   ARGP.  */
-static size_t
-argp_args_levels (const struct argp *argp)
-{
-  size_t levels = 0;
-  const struct argp_child *child = argp->children;
-
-  if (argp->args_doc && strchr (argp->args_doc, '\n'))
-    levels++;
-
-  if (child)
-    while (child->argp)
-      levels += argp_args_levels ((child++)->argp);
-
-  return levels;
-}
-
-/* Print all the non-option args documented in ARGP to STREAM.  Any output is
-   preceded by a space.  LEVELS is a pointer to a byte vector the length
-   returned by argp_args_levels; it should be initialized to zero, and
-   updated by this routine for the next call if ADVANCE is true.  True is
-   returned as long as there are more patterns to output.  */
-static int
-argp_args_usage (const struct argp *argp, const struct argp_state *state,
-		 char **levels, int advance, argp_fmtstream_t stream)
-{
-  char *our_level = *levels;
-  int multiple = 0;
-  const struct argp_child *child = argp->children;
-  const char *tdoc = dgettext (argp->argp_domain, argp->args_doc), *nl = 0;
-  const char *fdoc = filter_doc (tdoc, ARGP_KEY_HELP_ARGS_DOC, argp, state);
-
-  if (fdoc)
-    {
-      const char *cp = fdoc;
-      nl = __strchrnul (cp, '\n');
-      if (*nl != '\0')
-	/* This is a `multi-level' args doc; advance to the correct position
-	   as determined by our state in LEVELS, and update LEVELS.  */
-	{
-	  int i;
-	  multiple = 1;
-	  for (i = 0; i < *our_level; i++)
-	    cp = nl + 1, nl = __strchrnul (cp, '\n');
-	  (*levels)++;
-	}
-
-      /* Manually do line wrapping so that it (probably) won't get wrapped at
-	 any embedded spaces.  */
-      space (stream, 1 + nl - cp);
-
-      __argp_fmtstream_write (stream, cp, nl - cp);
-    }
-  if (fdoc && fdoc != tdoc)
-    free ((char *)fdoc);	/* Free user's modified doc string.  */
-
-  if (child)
-    while (child->argp)
-      advance = !argp_args_usage ((child++)->argp, state, levels, advance, stream);
-
-  if (advance && multiple)
-    {
-      /* Need to increment our level.  */
-      if (*nl)
-	/* There's more we can do here.  */
-	{
-	  (*our_level)++;
-	  advance = 0;		/* Our parent shouldn't advance also. */
-	}
-      else if (*our_level > 0)
-	/* We had multiple levels, but used them up; reset to zero.  */
-	*our_level = 0;
-    }
-
-  return !advance;
-}
-
-/* Print the documentation for ARGP to STREAM; if POST is false, then
-   everything preceeding a `\v' character in the documentation strings (or
-   the whole string, for those with none) is printed, otherwise, everything
-   following the `\v' character (nothing for strings without).  Each separate
-   bit of documentation is separated a blank line, and if PRE_BLANK is true,
-   then the first is as well.  If FIRST_ONLY is true, only the first
-   occurrence is output.  Returns true if anything was output.  */
-static int
-argp_doc (const struct argp *argp, const struct argp_state *state,
-	  int post, int pre_blank, int first_only,
-	  argp_fmtstream_t stream)
-{
-  const char *text;
-  const char *inp_text;
-  void *input = 0;
-  int anything = 0;
-  size_t inp_text_limit = 0;
-  const char *doc = dgettext (argp->argp_domain, argp->doc);
-  const struct argp_child *child = argp->children;
-
-  if (doc)
-    {
-      char *vt = strchr (doc, '\v');
-      inp_text = post ? (vt ? vt + 1 : 0) : doc;
-      inp_text_limit = (!post && vt) ? (vt - doc) : 0;
-    }
-  else
-    inp_text = 0;
-
-  if (argp->help_filter)
-    /* We have to filter the doc strings.  */
-    {
-      if (inp_text_limit)
-	/* Copy INP_TEXT so that it's nul-terminated.  */
-	inp_text = STRNDUP (inp_text, inp_text_limit);
-      input = __argp_input (argp, state);
-      text =
-	(*argp->help_filter) (post
-			      ? ARGP_KEY_HELP_POST_DOC
-			      : ARGP_KEY_HELP_PRE_DOC,
-			      inp_text, input);
-    }
-  else
-    text = (const char *) inp_text;
-
-  if (text)
-    {
-      if (pre_blank)
-	__argp_fmtstream_putc (stream, '\n');
-
-      if (text == inp_text && inp_text_limit)
-	__argp_fmtstream_write (stream, inp_text, inp_text_limit);
-      else
-	__argp_fmtstream_puts (stream, text);
-
-      if (__argp_fmtstream_point (stream) > __argp_fmtstream_lmargin (stream))
-	__argp_fmtstream_putc (stream, '\n');
-
-      anything = 1;
-    }
-
-  if (text && text != inp_text)
-    free ((char *) text);	/* Free TEXT returned from the help filter.  */
-  if (inp_text && inp_text_limit && argp->help_filter)
-    free ((char *) inp_text);	/* We copied INP_TEXT, so free it now.  */
-
-  if (post && argp->help_filter)
-    /* Now see if we have to output a ARGP_KEY_HELP_EXTRA text.  */
-    {
-      text = (*argp->help_filter) (ARGP_KEY_HELP_EXTRA, 0, input);
-      if (text)
-	{
-	  if (anything || pre_blank)
-	    __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, text);
-	  free ((char *) text);
-	  if (__argp_fmtstream_point (stream)
-	      > __argp_fmtstream_lmargin (stream))
-	    __argp_fmtstream_putc (stream, '\n');
-	  anything = 1;
-	}
-    }
-
-  if (child)
-    while (child->argp && !(first_only && anything))
-      anything |=
-	argp_doc ((child++)->argp, state,
-		  post, anything || pre_blank, first_only,
-		  stream);
-
-  return anything;
-}
-
-/* Output a usage message for ARGP to STREAM.  If called from
-   argp_state_help, STATE is the relevent parsing state.  FLAGS are from the
-   set ARGP_HELP_*.  NAME is what to use wherever a `program name' is
-   needed. */
-
-static void
-_help (const struct argp *argp, const struct argp_state *state, FILE *stream,
-       unsigned flags, const char *name)
-{
-  int anything = 0;		/* Whether we've output anything.  */
-  struct hol *hol = 0;
-  argp_fmtstream_t fs;
-
-  if (! stream)
-    return;
-
-  FLOCKFILE (stream);
-
-  if (! uparams.valid)
-    fill_in_uparams (state);
-
-  fs = __argp_make_fmtstream (stream, 0, uparams.rmargin, 0);
-  if (! fs)
-    {
-      FUNLOCKFILE (stream);
-      return;
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG))
-    {
-      hol = argp_hol (argp, 0);
-
-      /* If present, these options always come last.  */
-      hol_set_group (hol, "help", -1);
-      hol_set_group (hol, "version", -1);
-
-      hol_sort (hol);
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE))
-    /* Print a short `Usage:' message.  */
-    {
-      int first_pattern = 1, more_patterns;
-      size_t num_pattern_levels = argp_args_levels (argp);
-      char *pattern_levels = alloca (num_pattern_levels);
-
-      memset (pattern_levels, 0, num_pattern_levels);
-
-      do
-	{
-	  int old_lm;
-	  int old_wm = __argp_fmtstream_set_wmargin (fs, uparams.usage_indent);
-	  char *levels = pattern_levels;
-
-	  if (first_pattern)
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "Usage:"),
-				     name);
-	  else
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "  or: "),
-				     name);
-
-	  /* We set the lmargin as well as the wmargin, because hol_usage
-	     manually wraps options with newline to avoid annoying breaks.  */
-	  old_lm = __argp_fmtstream_set_lmargin (fs, uparams.usage_indent);
-
-	  if (flags & ARGP_HELP_SHORT_USAGE)
-	    /* Just show where the options go.  */
-	    {
-	      if (hol->num_entries > 0)
-		__argp_fmtstream_puts (fs, dgettext (argp->argp_domain,
-						     " [OPTION...]"));
-	    }
-	  else
-	    /* Actually print the options.  */
-	    {
-	      hol_usage (hol, fs);
-	      flags |= ARGP_HELP_SHORT_USAGE; /* But only do so once.  */
-	    }
-
-	  more_patterns = argp_args_usage (argp, state, &levels, 1, fs);
-
-	  __argp_fmtstream_set_wmargin (fs, old_wm);
-	  __argp_fmtstream_set_lmargin (fs, old_lm);
-
-	  __argp_fmtstream_putc (fs, '\n');
-	  anything = 1;
-
-	  first_pattern = 0;
-	}
-      while (more_patterns);
-    }
-
-  if (flags & ARGP_HELP_PRE_DOC)
-    anything |= argp_doc (argp, state, 0, 0, 1, fs);
-
-  if (flags & ARGP_HELP_SEE)
-    {
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain, "\
-Try `%s --help' or `%s --usage' for more information.\n"),
-			       name, name);
-      anything = 1;
-    }
-
-  if (flags & ARGP_HELP_LONG)
-    /* Print a long, detailed help message.  */
-    {
-      /* Print info about all the options.  */
-      if (hol->num_entries > 0)
-	{
-	  if (anything)
-	    __argp_fmtstream_putc (fs, '\n');
-	  hol_help (hol, state, fs);
-	  anything = 1;
-	}
-    }
-
-  if (flags & ARGP_HELP_POST_DOC)
-    /* Print any documentation strings at the end.  */
-    anything |= argp_doc (argp, state, 1, anything, 0, fs);
-
-  if ((flags & ARGP_HELP_BUG_ADDR) && argp_program_bug_address)
-    {
-      if (anything)
-	__argp_fmtstream_putc (fs, '\n');
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain,
-					     "Report bugs to %s.\n"),
- 			       argp_program_bug_address);
-      anything = 1;
-    }
-
-  FUNLOCKFILE (stream);
-
-  if (hol)
-    hol_free (hol);
-
-  __argp_fmtstream_free (fs);
-}
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  NAME is what to use wherever a `program name' is needed. */
-void __argp_help (const struct argp *argp, FILE *stream,
-		  unsigned flags, char *name)
-{
-  _help (argp, 0, stream, flags, name);
-}
-#ifdef weak_alias
-weak_alias (__argp_help, argp_help)
-#endif
-
-char *__argp_basename(char *name)
-{
-  char *short_name = strrchr(name, '/');
-  return short_name ? short_name + 1 : name;
-}
-
-char *
-__argp_short_program_name(const struct argp_state *state)
-{
-  if (state)
-    return state->name;
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-  return program_invocation_short_name;
-#elif HAVE_DECL_PROGRAM_INVOCATION_NAME
-  return __argp_basename(program_invocation_name);
-#else /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-  /* FIXME: What now? Miles suggests that it is better to use NULL,
-     but currently the value is passed on directly to fputs_unlocked,
-     so that requires more changes. */
-# if __GNUC__
-  return "";
-# endif /* __GNUC__ */
-#endif /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-}
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-void
-__argp_state_help (const struct argp_state *state, FILE *stream, unsigned flags)
-{
-  if ((!state || ! (state->flags & ARGP_NO_ERRS)) && stream)
-    {
-      if (state && (state->flags & ARGP_LONG_ONLY))
-	flags |= ARGP_HELP_LONG_ONLY;
-
-      _help (state ? state->root_argp : 0, state, stream, flags,
-	     __argp_short_program_name(state));
-
-      if (!state || ! (state->flags & ARGP_NO_EXIT))
-	{
-	  if (flags & ARGP_HELP_EXIT_ERR)
-	    exit (argp_err_exit_status);
-	  if (flags & ARGP_HELP_EXIT_OK)
-	    exit (0);
-	}
-  }
-}
-#ifdef weak_alias
-weak_alias (__argp_state_help, argp_state_help)
-#endif
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-void
-__argp_error (const struct argp_state *state, const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  va_list ap;
-
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-	  PUTC_UNLOCKED (':', stream);
-	  PUTC_UNLOCKED (' ', stream);
-
-	  va_start (ap, fmt);
-	  vfprintf (stream, fmt, ap);
-	  va_end (ap);
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  __argp_state_help (state, stream, ARGP_HELP_STD_ERR);
-
-	  FUNLOCKFILE (stream);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_error, argp_error)
-#endif
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-void
-__argp_failure (const struct argp_state *state, int status, int errnum,
-		const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-
-	  if (fmt)
-	    {
-	      va_list ap;
-
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-
-	      va_start (ap, fmt);
-	      vfprintf (stream, fmt, ap);
-	      va_end (ap);
-	    }
-
-	  if (errnum)
-	    {
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-	      fputs (STRERROR (errnum), stream);
-	    }
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  FUNLOCKFILE (stream);
-
-	  if (status && (!state || !(state->flags & ARGP_NO_EXIT)))
-	    exit (status);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_failure, argp_failure)
-#endif
diff --git a/argp-standalone/argp-namefrob.h b/argp-standalone/argp-namefrob.h
deleted file mode 100644
index 0ce11481a7b..00000000000
--- a/argp-standalone/argp-namefrob.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Name frobnication for compiling argp outside of glibc
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#if !_LIBC
-/* This code is written for inclusion in gnu-libc, and uses names in the
-   namespace reserved for libc.  If we're not compiling in libc, define those
-   names to be the normal ones instead.  */
-
-/* argp-parse functions */
-#undef __argp_parse
-#define __argp_parse argp_parse
-#undef __option_is_end
-#define __option_is_end _option_is_end
-#undef __option_is_short
-#define __option_is_short _option_is_short
-#undef __argp_input
-#define __argp_input _argp_input
-
-/* argp-help functions */
-#undef __argp_help
-#define __argp_help argp_help
-#undef __argp_error
-#define __argp_error argp_error
-#undef __argp_failure
-#define __argp_failure argp_failure
-#undef __argp_state_help
-#define __argp_state_help argp_state_help
-#undef __argp_usage
-#define __argp_usage argp_usage
-#undef __argp_basename
-#define __argp_basename _argp_basename
-#undef __argp_short_program_name
-#define __argp_short_program_name _argp_short_program_name
-
-/* argp-fmtstream functions */
-#undef __argp_make_fmtstream
-#define __argp_make_fmtstream argp_make_fmtstream
-#undef __argp_fmtstream_free
-#define __argp_fmtstream_free argp_fmtstream_free
-#undef __argp_fmtstream_putc
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#define __argp_fmtstream_write argp_fmtstream_write
-#undef __argp_fmtstream_printf
-#define __argp_fmtstream_printf argp_fmtstream_printf
-#undef __argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#define __argp_fmtstream_point argp_fmtstream_point
-#undef __argp_fmtstream_update
-#define __argp_fmtstream_update _argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#undef __argp_fmtstream_lmargin
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#undef __argp_fmtstream_rmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#undef __argp_fmtstream_wmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* normal libc functions we call */
-#undef __sleep
-#define __sleep sleep
-#undef __strcasecmp
-#define __strcasecmp strcasecmp
-#undef __vsnprintf
-#define __vsnprintf vsnprintf
-
-#endif /* !_LIBC */
-
-#ifndef __set_errno
-#define __set_errno(e) (errno = (e))
-#endif
diff --git a/argp-standalone/argp-parse.c b/argp-standalone/argp-parse.c
deleted file mode 100644
index 78f7bf139b6..00000000000
--- a/argp-standalone/argp-parse.c
+++ /dev/null
@@ -1,1305 +0,0 @@
-/* Hierarchial argument parsing
-   Copyright (C) 1995, 96, 97, 98, 99, 2000,2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <limits.h>
-#include <assert.h>
-
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.
-   When compiling libc, the _ macro is predefined.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-#  define gettext(msgid) (msgid)
-# endif
-#endif
-#ifndef N_
-# define N_(msgid) (msgid)
-#endif
-
-#if _LIBC - 0
-#include <bits/libc-lock.h>
-#else
-#ifdef HAVE_CTHREADS_H
-#include <cthreads.h>
-#endif
-#endif /* _LIBC */
-
-#include "argp.h"
-#include "argp-namefrob.h"
-
-
-/* The meta-argument used to prevent any further arguments being interpreted
-   as options.  */
-#define QUOTE "--"
-
-/* EZ alias for ARGP_ERR_UNKNOWN.  */
-#define EBADKEY ARGP_ERR_UNKNOWN
-
-
-/* Default options.  */
-
-/* When argp is given the --HANG switch, _ARGP_HANG is set and argp will sleep
-   for one second intervals, decrementing _ARGP_HANG until it's zero.  Thus
-   you can force the program to continue by attaching a debugger and setting
-   it to 0 yourself.  */
-volatile int _argp_hang;
-
-#define OPT_PROGNAME	-2
-#define OPT_USAGE	-3
-#if HAVE_SLEEP && HAVE_GETPID
-#define OPT_HANG	-4
-#endif
-
-static const struct argp_option argp_default_options[] =
-{
-  {"help",	  '?',    	0, 0,  N_("Give this help list"), -1},
-  {"usage",	  OPT_USAGE,	0, 0,  N_("Give a short usage message"), 0 },
-  {"program-name",OPT_PROGNAME,"NAME", OPTION_HIDDEN,
-     N_("Set the program name"), 0},
-#if OPT_HANG
-  {"HANG",	  OPT_HANG,    "SECS", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
-     N_("Hang for SECS seconds (default 3600)"), 0 },
-#endif
-  {0, 0, 0, 0, 0, 0}
-};
-
-static error_t
-argp_default_parser (int key, char *arg, struct argp_state *state)
-{
-  switch (key)
-    {
-    case '?':
-      __argp_state_help (state, state->out_stream, ARGP_HELP_STD_HELP);
-      break;
-    case OPT_USAGE:
-      __argp_state_help (state, state->out_stream,
-		       ARGP_HELP_USAGE | ARGP_HELP_EXIT_OK);
-      break;
-
-    case OPT_PROGNAME:		/* Set the program name.  */
-#if HAVE_DECL_PROGRAM_INVOCATION_NAME
-      program_invocation_name = arg;
-#endif
-      /* [Note that some systems only have PROGRAM_INVOCATION_SHORT_NAME (aka
-	 __PROGNAME), in which case, PROGRAM_INVOCATION_NAME is just defined
-	 to be that, so we have to be a bit careful here.]  */
-
-      /* Update what we use for messages.  */
-
-      state->name = __argp_basename(arg);
-      
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-      program_invocation_short_name = state->name;
-#endif
-
-      if ((state->flags & (ARGP_PARSE_ARGV0 | ARGP_NO_ERRS))
-	  == ARGP_PARSE_ARGV0)
-	/* Update what getopt uses too.  */
-	state->argv[0] = arg;
-
-      break;
-      
-#if OPT_HANG
-    case OPT_HANG:
-      _argp_hang = atoi (arg ? arg : "3600");
-      fprintf(state->err_stream, "%s: pid = %ld\n",
-	      state->name, (long) getpid());
-      while (_argp_hang-- > 0)
-	__sleep (1);
-      break;
-#endif
-      
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_default_argp =
-  {argp_default_options, &argp_default_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-static const struct argp_option argp_version_options[] =
-{
-  {"version",	  'V',    	0, 0,  N_("Print program version"), -1},
-  {0, 0, 0, 0, 0, 0 }
-};
-
-static error_t
-argp_version_parser (int key, char *arg UNUSED, struct argp_state *state)
-{
-  switch (key)
-    {
-    case 'V':
-      if (argp_program_version_hook)
-	(*argp_program_version_hook) (state->out_stream, state);
-      else if (argp_program_version)
-	fprintf (state->out_stream, "%s\n", argp_program_version);
-      else
-	__argp_error (state, dgettext (state->root_argp->argp_domain,
-				       "(PROGRAM ERROR) No version known!?"));
-      if (! (state->flags & ARGP_NO_EXIT))
-	exit (0);
-      break;
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_version_argp =
-  {argp_version_options, &argp_version_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-
-/* The state of a `group' during parsing.  Each group corresponds to a
-   particular argp structure from the tree of such descending from the top
-   level argp passed to argp_parse.  */
-struct group
-{
-  /* This group's parsing function.  */
-  argp_parser_t parser;
-
-  /* Which argp this group is from.  */
-  const struct argp *argp;
-
-  /* The number of non-option args sucessfully handled by this parser.  */
-  unsigned args_processed;
-
-  /* This group's parser's parent's group.  */
-  struct group *parent;
-  unsigned parent_index;	/* And the our position in the parent.   */
-
-  /* These fields are swapped into and out of the state structure when
-     calling this group's parser.  */
-  void *input, **child_inputs;
-  void *hook;
-};
-
-/* Call GROUP's parser with KEY and ARG, swapping any group-specific info
-   from STATE before calling, and back into state afterwards.  If GROUP has
-   no parser, EBADKEY is returned.  */
-static error_t
-group_parse (struct group *group, struct argp_state *state, int key, char *arg)
-{
-  if (group->parser)
-    {
-      error_t err;
-      state->hook = group->hook;
-      state->input = group->input;
-      state->child_inputs = group->child_inputs;
-      state->arg_num = group->args_processed;
-      err = (*group->parser)(key, arg, state);
-      group->hook = state->hook;
-      return err;
-    }
-  else
-    return EBADKEY;
-}
-
-struct parser
-{
-  const struct argp *argp;
-
-  const char *posixly_correct;
-  
-  /* True if there are only no-option arguments left, which are just
-     passed verbatim with ARGP_KEY_ARG. This is set if we encounter a
-     quote, or the end of the proper options, but may be cleared again
-     if the user moves the next argument pointer backwards. */
-  int args_only;
-
-  /* Describe how to deal with options that follow non-option ARGV-elements.
-
-     If the caller did not specify anything, the default is
-     REQUIRE_ORDER if the environment variable POSIXLY_CORRECT is
-     defined, PERMUTE otherwise.
-
-     REQUIRE_ORDER means don't recognize them as options; stop option
-     processing when the first non-option is seen. This is what Unix
-     does. This mode of operation is selected by either setting the
-     environment variable POSIXLY_CORRECT, or using `+' as the first
-     character of the list of option characters.
-
-     PERMUTE is the default. We permute the contents of ARGV as we
-     scan, so that eventually all the non-options are at the end. This
-     allows options to be given in any order, even with programs that
-     were not written to expect this.
-
-     RETURN_IN_ORDER is an option available to programs that were
-     written to expect options and other ARGV-elements in any order
-     and that care about the ordering of the two. We describe each
-     non-option ARGV-element as if it were the argument of an option
-     with character code 1. Using `-' as the first character of the
-     list of option characters selects this mode of operation.
-
-  */
-  enum { REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER } ordering;
-
-  /* A segment of non-option arguments that have been skipped for
-     later processing, after all options. `first_nonopt' is the index
-     in ARGV of the first of them; `last_nonopt' is the index after
-     the last of them.
-
-     If quoted or args_only is non-zero, this segment should be empty. */
-
-  /* FIXME: I'd prefer to use unsigned, but it's more consistent to
-     use the same type as for state.next. */
-  int first_nonopt;
-  int last_nonopt;
-  
-  /* String of all recognized short options. Needed for ARGP_LONG_ONLY. */
-  /* FIXME: Perhaps change to a pointer to a suitable bitmap instead? */
-  char *short_opts;
-
-  /* For parsing combined short options. */
-  char *nextchar;
-  
-  /* States of the various parsing groups.  */
-  struct group *groups;
-  /* The end of the GROUPS array.  */
-  struct group *egroup;
-  /* An vector containing storage for the CHILD_INPUTS field in all groups.  */
-  void **child_inputs;
-  
-  /* State block supplied to parsing routines.  */
-  struct argp_state state;
-
-  /* Memory used by this parser.  */
-  void *storage;
-};
-
-/* Search for a group defining a short option. */
-static const struct argp_option *
-find_short_option(struct parser *parser, int key, struct group **p)
-{
-  struct group *group;
-  
-  assert(key >= 0);
-  assert(isascii(key));
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	if (opts->key == key)
-	  {
-	    *p = group;
-	    return opts;
-	  }
-    }
-  return NULL;
-}
-
-enum match_result { MATCH_EXACT, MATCH_PARTIAL, MATCH_NO };
-
-/* If defined, allow complete.el-like abbreviations of long options. */
-#ifndef ARGP_COMPLETE
-#define ARGP_COMPLETE 0
-#endif
-
-/* Matches an encountern long-option argument ARG against an option NAME.
- * ARG is terminated by NUL or '='. */
-static enum match_result
-match_option(const char *arg, const char *name)
-{
-  unsigned i, j;
-  for (i = j = 0;; i++, j++)
-    {
-      switch(arg[i])
-	{
-	case '\0':
-	case '=':
-	  return name[j] ? MATCH_PARTIAL : MATCH_EXACT;
-#if ARGP_COMPLETE
-	case '-':
-	  while (name[j] != '-')
-	    if (!name[j++])
-	      return MATCH_NO;
-	  break;
-#endif
-	default:
-	  if (arg[i] != name[j])
-	    return MATCH_NO;
-	}
-    }
-}
-
-static const struct argp_option *
-find_long_option(struct parser *parser,
-		 const char *arg,
-		 struct group **p)
-{
-  struct group *group;
-
-  /* Partial match found so far. */
-  struct group *matched_group = NULL;
-  const struct argp_option *matched_option = NULL;
-
-  /* Number of partial matches. */
-  int num_partial = 0;
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	{
-	  if (!opts->name)
-	    continue;
-	  switch (match_option(arg, opts->name))
-	    {
-	    case MATCH_NO:
-	      break;
-	    case MATCH_PARTIAL:
-	      num_partial++;
-
-	      matched_group = group;
-	      matched_option = opts;
-
-	      break;
-	    case MATCH_EXACT:
-	      /* Exact match. */
-	      *p = group;
-	      return opts;
-	    }
-	}
-    }
-  if (num_partial == 1)
-    {
-      *p = matched_group;
-      return matched_option;
-    }
-
-  return NULL;
-}
-
-
-/* The next usable entries in the various parser tables being filled in by
-   convert_options.  */
-struct parser_convert_state
-{
-  struct parser *parser;
-  char *short_end;
-  void **child_inputs_end;
-};
-
-/* Initialize GROUP from ARGP. If CVT->SHORT_END is non-NULL, short
-   options are recorded in the short options string. Returns the next
-   unused group entry. CVT holds state used during the conversion. */
-static struct group *
-convert_options (const struct argp *argp,
-		 struct group *parent, unsigned parent_index,
-		 struct group *group, struct parser_convert_state *cvt)
-{
-  const struct argp_option *opt = argp->options;
-  const struct argp_child *children = argp->children;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      if (cvt->short_end)
-	{
-	  /* Record any short options. */
-	  for ( ; !__option_is_end (opt); opt++)
-	    if (__option_is_short(opt))
-	      *cvt->short_end++ = opt->key;
-	}
-      
-      group->parser = argp->parser;
-      group->argp = argp;
-      group->args_processed = 0;
-      group->parent = parent;
-      group->parent_index = parent_index;
-      group->input = 0;
-      group->hook = 0;
-      group->child_inputs = 0;
-      
-      if (children)
-	/* Assign GROUP's CHILD_INPUTS field some space from
-	   CVT->child_inputs_end.*/
-	{
-	  unsigned num_children = 0;
-	  while (children[num_children].argp)
-	    num_children++;
-	  group->child_inputs = cvt->child_inputs_end;
-	  cvt->child_inputs_end += num_children;
-	}
-      parent = group++;
-    }
-  else
-    parent = 0;
-
-  if (children)
-    {
-      unsigned index = 0;
-      while (children->argp)
-	group =
-	  convert_options (children++->argp, parent, index++, group, cvt);
-    }
-
-  return group;
-}
-/* Allocate and initialize the group structures, so that they are
-   ordered as if by traversing the corresponding argp parser tree in
-   pre-order. Also build the list of short options, if that is needed. */
-static void
-parser_convert (struct parser *parser, const struct argp *argp)
-{
-  struct parser_convert_state cvt;
-
-  cvt.parser = parser;
-  cvt.short_end = parser->short_opts;
-  cvt.child_inputs_end = parser->child_inputs;
-
-  parser->argp = argp;
-
-  if (argp)
-    parser->egroup = convert_options (argp, 0, 0, parser->groups, &cvt);
-  else
-    parser->egroup = parser->groups; /* No parsers at all! */
-
-  if (parser->short_opts)
-    *cvt.short_end ='\0';
-}
-
-/* Lengths of various parser fields which we will allocated.  */
-struct parser_sizes
-{
-  /* Needed only ARGP_LONG_ONLY */
-  size_t short_len;		/* Number of short options.  */
-
-  size_t num_groups;		/* Group structures we allocate.  */
-  size_t num_child_inputs;	/* Child input slots.  */
-};
-
-/* For ARGP, increments the NUM_GROUPS field in SZS by the total
-   number of argp structures descended from it, and the SHORT_LEN by
-   the total number of short options. */
-static void
-calc_sizes (const struct argp *argp,  struct parser_sizes *szs)
-{
-  const struct argp_child *child = argp->children;
-  const struct argp_option *opt = argp->options;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      szs->num_groups++;
-      if (opt)
-	{
-	  while (__option_is_short (opt++))
-	    szs->short_len++;
-	}
-    }
-
-  if (child)
-    while (child->argp)
-      {
-	calc_sizes ((child++)->argp, szs);
-	szs->num_child_inputs++;
-      }
-}
-
-/* Initializes PARSER to parse ARGP in a manner described by FLAGS.  */
-static error_t
-parser_init (struct parser *parser, const struct argp *argp,
-	     int argc, char **argv, int flags, void *input)
-{
-  error_t err = 0;
-  struct group *group;
-  struct parser_sizes szs;
-
-  parser->posixly_correct = getenv ("POSIXLY_CORRECT");
-
-  if (flags & ARGP_IN_ORDER)
-    parser->ordering = RETURN_IN_ORDER;
-  else if (flags & ARGP_NO_ARGS)
-    parser->ordering = REQUIRE_ORDER;
-  else if (parser->posixly_correct)
-    parser->ordering = REQUIRE_ORDER;
-  else
-    parser->ordering = PERMUTE;
-  
-  szs.short_len = 0;
-  szs.num_groups = 0;
-  szs.num_child_inputs = 0;
-
-  if (argp)
-    calc_sizes (argp, &szs);
-
-  if (!(flags & ARGP_LONG_ONLY))
-    /* We have no use for the short option array. */
-    szs.short_len = 0;
-  
-  /* Lengths of the various bits of storage used by PARSER.  */
-#define GLEN (szs.num_groups + 1) * sizeof (struct group)
-#define CLEN (szs.num_child_inputs * sizeof (void *))
-#define SLEN (szs.short_len + 1)
-#define STORAGE(offset) ((void *) (((char *) parser->storage) + (offset)))
-
-  parser->storage = malloc (GLEN + CLEN + SLEN);
-  if (! parser->storage)
-    return ENOMEM;
-
-  parser->groups = parser->storage;
-
-  parser->child_inputs = STORAGE(GLEN);
-  memset (parser->child_inputs, 0, szs.num_child_inputs * sizeof (void *));
-
-  if (flags & ARGP_LONG_ONLY)
-    parser->short_opts = STORAGE(GLEN + CLEN);
-  else
-    parser->short_opts = NULL;
-
-  parser_convert (parser, argp);
-
-  memset (&parser->state, 0, sizeof (struct argp_state));
-  
-  parser->state.root_argp = parser->argp;
-  parser->state.argc = argc;
-  parser->state.argv = argv;
-  parser->state.flags = flags;
-  parser->state.err_stream = stderr;
-  parser->state.out_stream = stdout;
-  parser->state.pstate = parser;
-
-  parser->args_only = 0;
-  parser->nextchar = NULL;
-  parser->first_nonopt = parser->last_nonopt = 0;
-    
-  /* Call each parser for the first time, giving it a chance to propagate
-     values to child parsers.  */
-  if (parser->groups < parser->egroup)
-    parser->groups->input = input;
-  for (group = parser->groups;
-       group < parser->egroup && (!err || err == EBADKEY);
-       group++)
-    {
-      if (group->parent)
-	/* If a child parser, get the initial input value from the parent. */
-	group->input = group->parent->child_inputs[group->parent_index];
-
-      if (!group->parser
-	  && group->argp->children && group->argp->children->argp)
-	/* For the special case where no parsing function is supplied for an
-	   argp, propagate its input to its first child, if any (this just
-	   makes very simple wrapper argps more convenient).  */
-	group->child_inputs[0] = group->input;
-
-      err = group_parse (group, &parser->state, ARGP_KEY_INIT, 0);
-    }
-  if (err == EBADKEY)
-    err = 0;			/* Some parser didn't understand.  */
-
-  if (err)
-    return err;
-
-  if (argv[0] && !(parser->state.flags & ARGP_PARSE_ARGV0))
-    /* There's an argv[0]; use it for messages.  */
-    {
-      parser->state.name = __argp_basename(argv[0]);
-
-      /* Don't parse it as an argument. */
-      parser->state.next = 1;
-    }
-  else
-    parser->state.name = __argp_short_program_name(NULL);
-  
-  return 0;
-}
-
-/* Free any storage consumed by PARSER (but not PARSER itself).  */
-static error_t
-parser_finalize (struct parser *parser,
-		 error_t err, int arg_ebadkey, int *end_index)
-{
-  struct group *group;
-
-  if (err == EBADKEY && arg_ebadkey)
-    /* Suppress errors generated by unparsed arguments.  */
-    err = 0;
-
-  if (! err)
-    {
-      if (parser->state.next == parser->state.argc)
-	/* We successfully parsed all arguments!  Call all the parsers again,
-	   just a few more times... */
-	{
-	  for (group = parser->groups;
-	       group < parser->egroup && (!err || err==EBADKEY);
-	       group++)
-	    if (group->args_processed == 0)
-	      err = group_parse (group, &parser->state, ARGP_KEY_NO_ARGS, 0);
-	  for (group = parser->egroup - 1;
-	       group >= parser->groups && (!err || err==EBADKEY);
-	       group--)
-	    err = group_parse (group, &parser->state, ARGP_KEY_END, 0);
-
-	  if (err == EBADKEY)
-	    err = 0;		/* Some parser didn't understand.  */
-
-	  /* Tell the user that all arguments are parsed.  */
-	  if (end_index)
-	    *end_index = parser->state.next;
-	}
-      else if (end_index)
-	/* Return any remaining arguments to the user.  */
-	*end_index = parser->state.next;
-      else
-	/* No way to return the remaining arguments, they must be bogus. */
-	{
-	  if (!(parser->state.flags & ARGP_NO_ERRS)
-	      && parser->state.err_stream)
-	    fprintf (parser->state.err_stream,
-		     dgettext (parser->argp->argp_domain,
-			       "%s: Too many arguments\n"),
-		     parser->state.name);
-	  err = EBADKEY;
-	}
-    }
-
-  /* Okay, we're all done, with either an error or success; call the parsers
-     to indicate which one.  */
-
-  if (err)
-    {
-      /* Maybe print an error message.  */
-      if (err == EBADKEY)
-	/* An appropriate message describing what the error was should have
-	   been printed earlier.  */
-	__argp_state_help (&parser->state, parser->state.err_stream,
-			   ARGP_HELP_STD_ERR);
-
-      /* Since we didn't exit, give each parser an error indication.  */
-      for (group = parser->groups; group < parser->egroup; group++)
-	group_parse (group, &parser->state, ARGP_KEY_ERROR, 0);
-    }
-  else
-    /* Notify parsers of success, and propagate back values from parsers.  */
-    {
-      /* We pass over the groups in reverse order so that child groups are
-	 given a chance to do there processing before passing back a value to
-	 the parent.  */
-      for (group = parser->egroup - 1
-	   ; group >= parser->groups && (!err || err == EBADKEY)
-	   ; group--)
-	err = group_parse (group, &parser->state, ARGP_KEY_SUCCESS, 0);
-      if (err == EBADKEY)
-	err = 0;		/* Some parser didn't understand.  */
-    }
-
-  /* Call parsers once more, to do any final cleanup.  Errors are ignored.  */
-  for (group = parser->egroup - 1; group >= parser->groups; group--)
-    group_parse (group, &parser->state, ARGP_KEY_FINI, 0);
-
-  if (err == EBADKEY)
-    err = EINVAL;
-
-  free (parser->storage);
-
-  return err;
-}
-
-/* Call the user parsers to parse the non-option argument VAL, at the
-   current position, returning any error. The state NEXT pointer
-   should point to the argument; this function will adjust it
-   correctly to reflect however many args actually end up being
-   consumed. */
-static error_t
-parser_parse_arg (struct parser *parser, char *val)
-{
-  /* Save the starting value of NEXT */
-  int index = parser->state.next;
-  error_t err = EBADKEY;
-  struct group *group;
-  int key = 0;			/* Which of ARGP_KEY_ARG[S] we used.  */
-
-  /* Try to parse the argument in each parser.  */
-  for (group = parser->groups
-       ; group < parser->egroup && err == EBADKEY
-       ; group++)
-    {
-      parser->state.next++;	/* For ARGP_KEY_ARG, consume the arg.  */
-      key = ARGP_KEY_ARG;
-      err = group_parse (group, &parser->state, key, val);
-
-      if (err == EBADKEY)
-	/* This parser doesn't like ARGP_KEY_ARG; try ARGP_KEY_ARGS instead. */
-	{
-	  parser->state.next--;	/* For ARGP_KEY_ARGS, put back the arg.  */
-	  key = ARGP_KEY_ARGS;
-	  err = group_parse (group, &parser->state, key, 0);
-	}
-    }
-
-  if (! err)
-    {
-      if (key == ARGP_KEY_ARGS)
-	/* The default for ARGP_KEY_ARGS is to assume that if NEXT isn't
-	   changed by the user, *all* arguments should be considered
-	   consumed.  */
-	parser->state.next = parser->state.argc;
-
-      if (parser->state.next > index)
-	/* Remember that we successfully processed a non-option
-	   argument -- but only if the user hasn't gotten tricky and set
-	   the clock back.  */
-	(--group)->args_processed += (parser->state.next - index);
-      else
-	/* The user wants to reparse some args, so try looking for options again.  */
-	parser->args_only = 0;
-    }
-
-  return err;
-}
-
-/* Exchange two adjacent subsequences of ARGV.
-   One subsequence is elements [first_nonopt,last_nonopt)
-   which contains all the non-options that have been skipped so far.
-   The other is elements [last_nonopt,next), which contains all
-   the options processed since those non-options were skipped.
-
-   `first_nonopt' and `last_nonopt' are relocated so that they describe
-   the new indices of the non-options in ARGV after they are moved.  */
-
-static void
-exchange (struct parser *parser)
-{
-  int bottom = parser->first_nonopt;
-  int middle = parser->last_nonopt;
-  int top = parser->state.next;
-  char **argv = parser->state.argv;
-  
-  char *tem;
-
-  /* Exchange the shorter segment with the far end of the longer segment.
-     That puts the shorter segment into the right place.
-     It leaves the longer segment in the right place overall,
-     but it consists of two parts that need to be swapped next.  */
-
-  while (top > middle && middle > bottom)
-    {
-      if (top - middle > middle - bottom)
-	{
-	  /* Bottom segment is the short one.  */
-	  int len = middle - bottom;
-	  register int i;
-
-	  /* Swap it with the top part of the top segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[top - (middle - bottom) + i];
-	      argv[top - (middle - bottom) + i] = tem;
-	    }
-	  /* Exclude the moved bottom segment from further swapping.  */
-	  top -= len;
-	}
-      else
-	{
-	  /* Top segment is the short one.  */
-	  int len = top - middle;
-	  register int i;
-
-	  /* Swap it with the bottom part of the bottom segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[middle + i];
-	      argv[middle + i] = tem;
-	    }
-	  /* Exclude the moved top segment from further swapping.  */
-	  bottom += len;
-	}
-    }
-
-  /* Update records for the slots the non-options now occupy.  */
-
-  parser->first_nonopt += (parser->state.next - parser->last_nonopt);
-  parser->last_nonopt = parser->state.next;
-}
-
-
-
-enum arg_type { ARG_ARG, ARG_SHORT_OPTION,
-		ARG_LONG_OPTION, ARG_LONG_ONLY_OPTION,
-		ARG_QUOTE };
-
-static enum arg_type
-classify_arg(struct parser *parser, char *arg, char **opt)
-{
-  if (arg[0] == '-')
-    /* Looks like an option... */
-    switch (arg[1])
-      {
-      case '\0':
-	/* "-" is not an option. */
-	return ARG_ARG;
-      case '-':
-	/* Long option, or quote. */
-	if (!arg[2])
-	  return ARG_QUOTE;
-	  
-	/* A long option. */
-	if (opt)
-	  *opt = arg + 2;
-	return ARG_LONG_OPTION;
-
-      default:
-	/* Short option. But if ARGP_LONG_ONLY, it can also be a long option. */
-
-	if (opt)
-	  *opt = arg + 1;
-
-	if (parser->state.flags & ARGP_LONG_ONLY)
-	  {
-	    /* Rules from getopt.c:
-
-	       If long_only and the ARGV-element has the form "-f",
-	       where f is a valid short option, don't consider it an
-	       abbreviated form of a long option that starts with f.
-	       Otherwise there would be no way to give the -f short
-	       option.
-
-	       On the other hand, if there's a long option "fubar" and
-	       the ARGV-element is "-fu", do consider that an
-	       abbreviation of the long option, just like "--fu", and
-	       not "-f" with arg "u".
-
-	       This distinction seems to be the most useful approach. */
-
-	    assert(parser->short_opts);
-	    
-	    if (arg[2] || !strchr(parser->short_opts, arg[1]))
-	      return ARG_LONG_ONLY_OPTION;
-	  }
-
-	return ARG_SHORT_OPTION;
-      }
-  
-  else
-    return ARG_ARG;
-}
-
-/* Parse the next argument in PARSER (as indicated by PARSER->state.next).
-   Any error from the parsers is returned, and *ARGP_EBADKEY indicates
-   whether a value of EBADKEY is due to an unrecognized argument (which is
-   generally not fatal).  */
-static error_t
-parser_parse_next (struct parser *parser, int *arg_ebadkey)
-{
-  if (parser->state.quoted && parser->state.next < parser->state.quoted)
-    /* The next argument pointer has been moved to before the quoted
-       region, so pretend we never saw the quoting `--', and start
-       looking for options again. If the `--' is still there we'll just
-       process it one more time. */
-    parser->state.quoted = parser->args_only = 0;
-
-  /* Give FIRST_NONOPT & LAST_NONOPT rational values if NEXT has been
-     moved back by the user (who may also have changed the arguments).  */
-  if (parser->last_nonopt > parser->state.next)
-    parser->last_nonopt = parser->state.next;
-  if (parser->first_nonopt > parser->state.next)
-    parser->first_nonopt = parser->state.next;
-
-  if (parser->nextchar)
-    /* Deal with short options. */
-    {
-      struct group *group;
-      char c;
-      const struct argp_option *option;
-      char *value = NULL;;
-
-      assert(!parser->args_only);
-
-      c = *parser->nextchar++;
-      
-      option = find_short_option(parser, c, &group);
-      if (!option)
-	{
-	  if (parser->posixly_correct)
-	    /* 1003.2 specifies the format of this message.  */
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: illegal option -- %c\n"),
-		     parser->state.name, c);
-	  else
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: invalid option -- %c\n"),
-		     parser->state.name, c);
-
-	  *arg_ebadkey = 0;
-	  return EBADKEY;
-	}
-
-      if (!*parser->nextchar)
-	parser->nextchar = NULL;
-
-      if (option->arg)
-	{
-	  value = parser->nextchar;
-	  parser->nextchar = NULL;
-	      
-	  if (!value
-	      && !(option->flags & OPTION_ARG_OPTIONAL))
-	    /* We need an mandatory argument. */
-	    {
-	      if (parser->state.next == parser->state.argc)
-		/* Missing argument */
-		{
-		  /* 1003.2 specifies the format of this message.  */
-		  fprintf (parser->state.err_stream,
-			   dgettext(parser->state.root_argp->argp_domain,
-				    "%s: option requires an argument -- %c\n"),
-			   parser->state.name, c);
-
-		  *arg_ebadkey = 0;
-		  return EBADKEY;
-		}
-	      value = parser->state.argv[parser->state.next++];
-	    }
-	}
-      return group_parse(group, &parser->state,
-			 option->key, value);
-    }
-  else
-    /* Advance to the next ARGV-element.  */
-    {
-      if (parser->args_only)
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->state.next >= parser->state.argc)
-	    /* We're done. */
-	    return EBADKEY;
-	  else
-	    return parser_parse_arg(parser,
-				    parser->state.argv[parser->state.next]);
-	}
-      
-      if (parser->state.next >= parser->state.argc)
-	/* Almost done. If there are non-options that we skipped
-	   previously, we should process them now. */
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->first_nonopt != parser->last_nonopt)
-	    {
-	      exchange(parser);
-	      
-	      /* Start processing the arguments we skipped previously. */
-	      parser->state.next = parser->first_nonopt;
-	      
-	      parser->first_nonopt = parser->last_nonopt = 0;
-
-	      parser->args_only = 1;
-	      return 0;
-	    }
-	  else
-	    /* Indicate that we're really done. */
-	    return EBADKEY;
-	}
-      else
-	/* Look for options. */
-	{
-	  char *arg = parser->state.argv[parser->state.next];
-
-	  char *optstart;
-	  enum arg_type token = classify_arg(parser, arg, &optstart);
-	  
-	  switch (token)
-	    {
-	    case ARG_ARG:
-	      switch (parser->ordering)
-		{
-		case PERMUTE:
-		  if (parser->first_nonopt == parser->last_nonopt)
-		    /* Skipped sequence is empty; start a new one. */
-		    parser->first_nonopt = parser->last_nonopt = parser->state.next;
-
-		  else if (parser->last_nonopt != parser->state.next)
-		    /* We have a non-empty skipped sequence, and
-		       we're not at the end-point, so move it. */
-		    exchange(parser);
-
-		  assert(parser->last_nonopt == parser->state.next);
-		  
-		  /* Skip this argument for now. */
-		  parser->state.next++;
-		  parser->last_nonopt = parser->state.next; 
-		  
-		  return 0;
-
-		case REQUIRE_ORDER:
-		  /* Implicit quote before the first argument. */
-		   parser->args_only = 1;
-		   return 0;
-		   
-		case RETURN_IN_ORDER:
-		  *arg_ebadkey = 1;
-		  return parser_parse_arg(parser, arg);
-
-		default:
-		  abort();
-		}
-	    case ARG_QUOTE:
-	      /* Skip it, then exchange with any previous non-options. */
-	      parser->state.next++;
-	      assert (parser->last_nonopt != parser->state.next);
-
-	      if (parser->first_nonopt != parser->last_nonopt)
-		{
-		  exchange(parser);
-		  
-		  /* Start processing the skipped and the quoted
-		     arguments. */
-
-		  parser->state.quoted = parser->state.next = parser->first_nonopt;
-
-		  /* Also empty the skipped-list, to avoid confusion
-		     if the user resets the next pointer. */
-		  parser->first_nonopt = parser->last_nonopt = 0;
-		}
-	      else
-		parser->state.quoted = parser->state.next;
-
-	      parser->args_only = 1;	      
-	      return 0;
-
-	    case ARG_LONG_ONLY_OPTION:
-	    case ARG_LONG_OPTION:
-	      {
-		struct group *group;
-		const struct argp_option *option;
-		char *value;
-
-		parser->state.next++;
-		option = find_long_option(parser, optstart, &group);
-		
-		if (!option)
-		  {
-		    /* NOTE: This includes any "=something" in the output. */
-		    fprintf (parser->state.err_stream,
-			     dgettext(parser->state.root_argp->argp_domain,
-				      "%s: unrecognized option `%s'\n"),
-			     parser->state.name, arg);
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-
-		value = strchr(optstart, '=');
-		if (value)
-		  value++;
-		
-		if (value && !option->arg)
-		  /* Unexpected argument. */
-		  {
-		    if (token == ARG_LONG_OPTION)
-		      /* --option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `--%s' doesn't allow an argument\n"),
-			       parser->state.name, option->name);
-		    else
-		      /* +option or -option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `%c%s' doesn't allow an argument\n"),
-			       parser->state.name, arg[0], option->name);
-
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-		
-		if (option->arg && !value
-		    && !(option->flags & OPTION_ARG_OPTIONAL))
-		  /* We need an mandatory argument. */
-		  {
-		    if (parser->state.next == parser->state.argc)
-		      /* Missing argument */
-		      {
-			if (token == ARG_LONG_OPTION)
-			  /* --option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `--%s' requires an argument\n"),
-				 parser->state.name, option->name);
-			else
-			  /* +option or -option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `%c%s' requires an argument\n"),
-				   parser->state.name, arg[0], option->name);
-
-			*arg_ebadkey = 0;
-			return EBADKEY;
-		      }
-
-		    value = parser->state.argv[parser->state.next++];
-		  }
-		*arg_ebadkey = 0;
-		return group_parse(group, &parser->state,
-				   option->key, value);
-	      }
-	    case ARG_SHORT_OPTION:
-	      parser->state.next++;
-	      parser->nextchar = optstart;
-	      return 0;
-
-	    default:
-	      abort();
-	    }
-	}
-    }
-}
-
-/* Parse the options strings in ARGC & ARGV according to the argp in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If END_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, EINVAL is returned; if some parser routine
-   returned a non-zero value, it is returned; otherwise 0 is returned.  */
-error_t
-__argp_parse (const struct argp *argp, int argc, char **argv, unsigned flags,
-	      int *end_index, void *input)
-{
-  error_t err;
-  struct parser parser;
-
-  /* If true, then err == EBADKEY is a result of a non-option argument failing
-     to be parsed (which in some cases isn't actually an error).  */
-  int arg_ebadkey = 0;
-
-  if (! (flags & ARGP_NO_HELP))
-    /* Add our own options.  */
-    {
-      struct argp_child *child = alloca (4 * sizeof (struct argp_child));
-      struct argp *top_argp = alloca (sizeof (struct argp));
-
-      /* TOP_ARGP has no options, it just serves to group the user & default
-	 argps.  */
-      memset (top_argp, 0, sizeof (*top_argp));
-      top_argp->children = child;
-
-      memset (child, 0, 4 * sizeof (struct argp_child));
-
-      if (argp)
-	(child++)->argp = argp;
-      (child++)->argp = &argp_default_argp;
-      if (argp_program_version || argp_program_version_hook)
-	(child++)->argp = &argp_version_argp;
-      child->argp = 0;
-
-      argp = top_argp;
-    }
-
-  /* Construct a parser for these arguments.  */
-  err = parser_init (&parser, argp, argc, argv, flags, input);
-
-  if (! err)
-    /* Parse! */
-    {
-      while (! err)
-	err = parser_parse_next (&parser, &arg_ebadkey);
-      err = parser_finalize (&parser, err, arg_ebadkey, end_index);
-    }
-
-  return err;
-}
-#ifdef weak_alias
-weak_alias (__argp_parse, argp_parse)
-#endif
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-void *
-__argp_input (const struct argp *argp, const struct argp_state *state)
-{
-  if (state)
-    {
-      struct group *group;
-      struct parser *parser = state->pstate;
-
-      for (group = parser->groups; group < parser->egroup; group++)
-	if (group->argp == argp)
-	  return group->input;
-    }
-
-  return 0;
-}
-#ifdef weak_alias
-weak_alias (__argp_input, _argp_input)
-#endif
-
-/* Defined here, in case a user is not inlining the definitions in
- * argp.h */
-void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      /* FIXME: whether or not a particular key implies a short option
-       * ought not to be locale dependent. */
-      return __key > 0 && isprint (__key);
-    }
-}
-
-int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
diff --git a/argp-standalone/argp-pv.c b/argp-standalone/argp-pv.c
deleted file mode 100644
index d7d374a66bd..00000000000
--- a/argp-standalone/argp-pv.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which will
-   print this this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-const char *argp_program_version = 0;
diff --git a/argp-standalone/argp-pvh.c b/argp-standalone/argp-pvh.c
deleted file mode 100644
index 829a1cda80d..00000000000
--- a/argp-standalone/argp-pvh.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION_HOOK.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "argp.h"
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which calls
-   this function with a stream to print the version to and a pointer to the
-   current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-void (*argp_program_version_hook) (FILE *stream, struct argp_state *state) = 0;
diff --git a/argp-standalone/argp.h b/argp-standalone/argp.h
deleted file mode 100644
index 29d3dfe9720..00000000000
--- a/argp-standalone/argp.h
+++ /dev/null
@@ -1,602 +0,0 @@
-/* Hierarchial argument parsing.
-   Copyright (C) 1995, 96, 97, 98, 99, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _ARGP_H
-#define _ARGP_H
-
-#include <stdio.h>
-#include <ctype.h>
-
-#define __need_error_t
-#include <errno.h>
-
-#ifndef __THROW
-# define __THROW
-#endif
-
-#ifndef __const
-# define __const const
-#endif
-
-#ifndef __error_t_defined
-typedef int error_t;
-# define __error_t_defined
-#endif
-
-/* FIXME: What's the right way to check for __restrict? Sun's cc seems
-   not to have it. Perhaps it's easiest to just delete the use of
-   __restrict from the prototypes. */
-#ifndef __restrict
-# ifndef __GNUC___
-#  define __restrict
-# endif
-#endif
-
-/* NOTE: We can't use the autoconf tests, since this is supposed to be
-   an installed header file and argp's config.h is of course not
-   installed. */
-#ifndef PRINTF_STYLE
-# if __GNUC__ >= 2
-#  define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# else
-#  define PRINTF_STYLE(f, a)
-# endif
-#endif
-
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-/* A description of a particular option.  A pointer to an array of
-   these is passed in the OPTIONS field of an argp structure.  Each option
-   entry can correspond to one long option and/or one short option; more
-   names for the same option can be added by following an entry in an option
-   array with options having the OPTION_ALIAS flag set.  */
-struct argp_option
-{
-  /* The long option name.  For more than one name for the same option, you
-     can use following options with the OPTION_ALIAS flag set.  */
-  __const char *name;
-
-  /* What key is returned for this option.  If > 0 and printable, then it's
-     also accepted as a short option.  */
-  int key;
-
-  /* If non-NULL, this is the name of the argument associated with this
-     option, which is required unless the OPTION_ARG_OPTIONAL flag is set. */
-  __const char *arg;
-
-  /* OPTION_ flags.  */
-  int flags;
-
-  /* The doc string for this option.  If both NAME and KEY are 0, This string
-     will be printed outdented from the normal option column, making it
-     useful as a group header (it will be the first thing printed in its
-     group); in this usage, it's conventional to end the string with a `:'.  */
-  __const char *doc;
-
-  /* The group this option is in.  In a long help message, options are sorted
-     alphabetically within each group, and the groups presented in the order
-     0, 1, 2, ..., n, -m, ..., -2, -1.  Every entry in an options array with
-     if this field 0 will inherit the group number of the previous entry, or
-     zero if it's the first one, unless its a group header (NAME and KEY both
-     0), in which case, the previous entry + 1 is the default.  Automagic
-     options such as --help are put into group -1.  */
-  int group;
-};
-
-/* The argument associated with this option is optional.  */
-#define OPTION_ARG_OPTIONAL	0x1
-
-/* This option isn't displayed in any help messages.  */
-#define OPTION_HIDDEN	       	0x2
-
-/* This option is an alias for the closest previous non-alias option.  This
-   means that it will be displayed in the same help entry, and will inherit
-   fields other than NAME and KEY from the aliased option.  */
-#define OPTION_ALIAS		0x4
-
-/* This option isn't actually an option (and so should be ignored by the
-   actual option parser), but rather an arbitrary piece of documentation that
-   should be displayed in much the same manner as the options.  If this flag
-   is set, then the option NAME field is displayed unmodified (e.g., no `--'
-   prefix is added) at the left-margin (where a *short* option would normally
-   be displayed), and the documentation string in the normal place.  For
-   purposes of sorting, any leading whitespace and puncuation is ignored,
-   except that if the first non-whitespace character is not `-', this entry
-   is displayed after all options (and OPTION_DOC entries with a leading `-')
-   in the same group.  */
-#define OPTION_DOC		0x8
-
-/* This option shouldn't be included in `long' usage messages (but is still
-   included in help messages).  This is mainly intended for options that are
-   completely documented in an argp's ARGS_DOC field, in which case including
-   the option in the generic usage list would be redundant.  For instance,
-   if ARGS_DOC is "FOO BAR\n-x BLAH", and the `-x' option's purpose is to
-   distinguish these two cases, -x should probably be marked
-   OPTION_NO_USAGE.  */
-#define OPTION_NO_USAGE		0x10
-
-struct argp;			/* fwd declare this type */
-struct argp_state;		/* " */
-struct argp_child;		/* " */
-
-/* The type of a pointer to an argp parsing function.  */
-typedef error_t (*argp_parser_t) (int key, char *arg,
-				  struct argp_state *state);
-
-/* What to return for unrecognized keys.  For special ARGP_KEY_ keys, such
-   returns will simply be ignored.  For user keys, this error will be turned
-   into EINVAL (if the call to argp_parse is such that errors are propagated
-   back to the user instead of exiting); returning EINVAL itself would result
-   in an immediate stop to parsing in *all* cases.  */
-#define ARGP_ERR_UNKNOWN	E2BIG /* Hurd should never need E2BIG.  XXX */
-
-/* Special values for the KEY argument to an argument parsing function.
-   ARGP_ERR_UNKNOWN should be returned if they aren't understood.
-
-   The sequence of keys to a parsing function is either (where each
-   uppercased word should be prefixed by `ARGP_KEY_' and opt is a user key):
-
-       INIT opt... NO_ARGS END SUCCESS  -- No non-option arguments at all
-   or  INIT (opt | ARG)... END SUCCESS  -- All non-option args parsed
-   or  INIT (opt | ARG)... SUCCESS      -- Some non-option arg unrecognized
-
-   The third case is where every parser returned ARGP_KEY_UNKNOWN for an
-   argument, in which case parsing stops at that argument (returning the
-   unparsed arguments to the caller of argp_parse if requested, or stopping
-   with an error message if not).
-
-   If an error occurs (either detected by argp, or because the parsing
-   function returned an error value), then the parser is called with
-   ARGP_KEY_ERROR, and no further calls are made.  */
-
-/* This is not an option at all, but rather a command line argument.  If a
-   parser receiving this key returns success, the fact is recorded, and the
-   ARGP_KEY_NO_ARGS case won't be used.  HOWEVER, if while processing the
-   argument, a parser function decrements the NEXT field of the state it's
-   passed, the option won't be considered processed; this is to allow you to
-   actually modify the argument (perhaps into an option), and have it
-   processed again.  */
-#define ARGP_KEY_ARG		0
-/* There are remaining arguments not parsed by any parser, which may be found
-   starting at (STATE->argv + STATE->next).  If success is returned, but
-   STATE->next left untouched, it's assumed that all arguments were consume,
-   otherwise, the parser should adjust STATE->next to reflect any arguments
-   consumed.  */
-#define ARGP_KEY_ARGS		0x1000006
-/* There are no more command line arguments at all.  */
-#define ARGP_KEY_END		0x1000001
-/* Because it's common to want to do some special processing if there aren't
-   any non-option args, user parsers are called with this key if they didn't
-   successfully process any non-option arguments.  Called just before
-   ARGP_KEY_END (where more general validity checks on previously parsed
-   arguments can take place).  */
-#define ARGP_KEY_NO_ARGS	0x1000002
-/* Passed in before any parsing is done.  Afterwards, the values of each
-   element of the CHILD_INPUT field, if any, in the state structure is
-   copied to each child's state to be the initial value of the INPUT field.  */
-#define ARGP_KEY_INIT		0x1000003
-/* Use after all other keys, including SUCCESS & END.  */
-#define ARGP_KEY_FINI		0x1000007
-/* Passed in when parsing has successfully been completed (even if there are
-   still arguments remaining).  */
-#define ARGP_KEY_SUCCESS	0x1000004
-/* Passed in if an error occurs.  */
-#define ARGP_KEY_ERROR		0x1000005
-
-/* An argp structure contains a set of options declarations, a function to
-   deal with parsing one, documentation string, a possible vector of child
-   argp's, and perhaps a function to filter help output.  When actually
-   parsing options, getopt is called with the union of all the argp
-   structures chained together through their CHILD pointers, with conflicts
-   being resolved in favor of the first occurrence in the chain.  */
-struct argp
-{
-  /* An array of argp_option structures, terminated by an entry with both
-     NAME and KEY having a value of 0.  */
-  __const struct argp_option *options;
-
-  /* What to do with an option from this structure.  KEY is the key
-     associated with the option, and ARG is any associated argument (NULL if
-     none was supplied).  If KEY isn't understood, ARGP_ERR_UNKNOWN should be
-     returned.  If a non-zero, non-ARGP_ERR_UNKNOWN value is returned, then
-     parsing is stopped immediately, and that value is returned from
-     argp_parse().  For special (non-user-supplied) values of KEY, see the
-     ARGP_KEY_ definitions below.  */
-  argp_parser_t parser;
-
-  /* A string describing what other arguments are wanted by this program.  It
-     is only used by argp_usage to print the `Usage:' message.  If it
-     contains newlines, the strings separated by them are considered
-     alternative usage patterns, and printed on separate lines (lines after
-     the first are prefix by `  or: ' instead of `Usage:').  */
-  __const char *args_doc;
-
-  /* If non-NULL, a string containing extra text to be printed before and
-     after the options in a long help message (separated by a vertical tab
-     `\v' character).  */
-  __const char *doc;
-
-  /* A vector of argp_children structures, terminated by a member with a 0
-     argp field, pointing to child argps should be parsed with this one.  Any
-     conflicts are resolved in favor of this argp, or early argps in the
-     CHILDREN list.  This field is useful if you use libraries that supply
-     their own argp structure, which you want to use in conjunction with your
-     own.  */
-  __const struct argp_child *children;
-
-  /* If non-zero, this should be a function to filter the output of help
-     messages.  KEY is either a key from an option, in which case TEXT is
-     that option's help text, or a special key from the ARGP_KEY_HELP_
-     defines, below, describing which other help text TEXT is.  The function
-     should return either TEXT, if it should be used as-is, a replacement
-     string, which should be malloced, and will be freed by argp, or NULL,
-     meaning `print nothing'.  The value for TEXT is *after* any translation
-     has been done, so if any of the replacement text also needs translation,
-     that should be done by the filter function.  INPUT is either the input
-     supplied to argp_parse, or NULL, if argp_help was called directly.  */
-  char *(*help_filter) (int __key, __const char *__text, void *__input);
-
-  /* If non-zero the strings used in the argp library are translated using
-     the domain described by this string.  Otherwise the currently installed
-     default domain is used.  */
-  const char *argp_domain;
-};
-
-/* Possible KEY arguments to a help filter function.  */
-#define ARGP_KEY_HELP_PRE_DOC	0x2000001 /* Help text preceeding options. */
-#define ARGP_KEY_HELP_POST_DOC	0x2000002 /* Help text following options. */
-#define ARGP_KEY_HELP_HEADER	0x2000003 /* Option header string. */
-#define ARGP_KEY_HELP_EXTRA	0x2000004 /* After all other documentation;
-					     TEXT is NULL for this key.  */
-/* Explanatory note emitted when duplicate option arguments have been
-   suppressed.  */
-#define ARGP_KEY_HELP_DUP_ARGS_NOTE 0x2000005
-#define ARGP_KEY_HELP_ARGS_DOC	0x2000006 /* Argument doc string.  */
-
-/* When an argp has a non-zero CHILDREN field, it should point to a vector of
-   argp_child structures, each of which describes a subsidiary argp.  */
-struct argp_child
-{
-  /* The child parser.  */
-  __const struct argp *argp;
-
-  /* Flags for this child.  */
-  int flags;
-
-  /* If non-zero, an optional header to be printed in help output before the
-     child options.  As a side-effect, a non-zero value forces the child
-     options to be grouped together; to achieve this effect without actually
-     printing a header string, use a value of "".  */
-  __const char *header;
-
-  /* Where to group the child options relative to the other (`consolidated')
-     options in the parent argp; the values are the same as the GROUP field
-     in argp_option structs, but all child-groupings follow parent options at
-     a particular group level.  If both this field and HEADER are zero, then
-     they aren't grouped at all, but rather merged with the parent options
-     (merging the child's grouping levels with the parents).  */
-  int group;
-};
-
-/* Parsing state.  This is provided to parsing functions called by argp,
-   which may examine and, as noted, modify fields.  */
-struct argp_state
-{
-  /* The top level ARGP being parsed.  */
-  __const struct argp *root_argp;
-
-  /* The argument vector being parsed.  May be modified.  */
-  int argc;
-  char **argv;
-
-  /* The index in ARGV of the next arg that to be parsed.  May be modified. */
-  int next;
-
-  /* The flags supplied to argp_parse.  May be modified.  */
-  unsigned flags;
-
-  /* While calling a parsing function with a key of ARGP_KEY_ARG, this is the
-     number of the current arg, starting at zero, and incremented after each
-     such call returns.  At all other times, this is the number of such
-     arguments that have been processed.  */
-  unsigned arg_num;
-
-  /* If non-zero, the index in ARGV of the first argument following a special
-     `--' argument (which prevents anything following being interpreted as an
-     option).  Only set once argument parsing has proceeded past this point. */
-  int quoted;
-
-  /* An arbitrary pointer passed in from the user.  */
-  void *input;
-  /* Values to pass to child parsers.  This vector will be the same length as
-     the number of children for the current parser.  */
-  void **child_inputs;
-
-  /* For the parser's use.  Initialized to 0.  */
-  void *hook;
-
-  /* The name used when printing messages.  This is initialized to ARGV[0],
-     or PROGRAM_INVOCATION_NAME if that is unavailable.  */
-  char *name;
-
-  /* Streams used when argp prints something.  */
-  FILE *err_stream;		/* For errors; initialized to stderr. */
-  FILE *out_stream;		/* For information; initialized to stdout. */
-
-  void *pstate;			/* Private, for use by argp.  */
-};
-
-/* Flags for argp_parse (note that the defaults are those that are
-   convenient for program command line parsing): */
-
-/* Don't ignore the first element of ARGV.  Normally (and always unless
-   ARGP_NO_ERRS is set) the first element of the argument vector is
-   skipped for option parsing purposes, as it corresponds to the program name
-   in a command line.  */
-#define ARGP_PARSE_ARGV0  0x01
-
-/* Don't print error messages for unknown options to stderr; unless this flag
-   is set, ARGP_PARSE_ARGV0 is ignored, as ARGV[0] is used as the program
-   name in the error messages.  This flag implies ARGP_NO_EXIT (on the
-   assumption that silent exiting upon errors is bad behaviour).  */
-#define ARGP_NO_ERRS	0x02
-
-/* Don't parse any non-option args.  Normally non-option args are parsed by
-   calling the parse functions with a key of ARGP_KEY_ARG, and the actual arg
-   as the value.  Since it's impossible to know which parse function wants to
-   handle it, each one is called in turn, until one returns 0 or an error
-   other than ARGP_ERR_UNKNOWN; if an argument is handled by no one, the
-   argp_parse returns prematurely (but with a return value of 0).  If all
-   args have been parsed without error, all parsing functions are called one
-   last time with a key of ARGP_KEY_END.  This flag needn't normally be set,
-   as the normal behavior is to stop parsing as soon as some argument can't
-   be handled.  */
-#define ARGP_NO_ARGS	0x04
-
-/* Parse options and arguments in the same order they occur on the command
-   line -- normally they're rearranged so that all options come first. */
-#define ARGP_IN_ORDER	0x08
-
-/* Don't provide the standard long option --help, which causes usage and
-      option help information to be output to stdout, and exit (0) called. */
-#define ARGP_NO_HELP	0x10
-
-/* Don't exit on errors (they may still result in error messages).  */
-#define ARGP_NO_EXIT	0x20
-
-/* Use the gnu getopt `long-only' rules for parsing arguments.  */
-#define ARGP_LONG_ONLY	0x40
-
-/* Turns off any message-printing/exiting options.  */
-#define ARGP_SILENT    (ARGP_NO_EXIT | ARGP_NO_ERRS | ARGP_NO_HELP)
-
-/* Parse the options strings in ARGC & ARGV according to the options in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If ARG_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, ARGP_ERR_UNKNOWN is returned; if some parser
-   routine returned a non-zero value, it is returned; otherwise 0 is
-   returned.  This function may also call exit unless the ARGP_NO_HELP flag
-   is set.  INPUT is a pointer to a value to be passed in to the parser.  */
-extern error_t argp_parse (__const struct argp *__restrict argp,
-			   int argc, char **__restrict argv,
-			   unsigned flags, int *__restrict arg_index,
-			   void *__restrict input) __THROW;
-extern error_t __argp_parse (__const struct argp *__restrict argp,
-			     int argc, char **__restrict argv,
-			     unsigned flags, int *__restrict arg_index,
-			     void *__restrict input) __THROW;
-
-/* Global variables.  */
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   will print this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-extern __const char *argp_program_version;
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   calls this function with a stream to print the version to and a pointer to
-   the current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-extern void (*argp_program_version_hook) (FILE *__restrict __stream,
-					  struct argp_state *__restrict
-					  __state);
-
-/* If defined or set by the user program, it should point to string that is
-   the bug-reporting address for the program.  It will be printed by
-   argp_help if the ARGP_HELP_BUG_ADDR flag is set (as it is by various
-   standard help messages), embedded in a sentence that says something like
-   `Report bugs to ADDR.'.  */
-extern __const char *argp_program_bug_address;
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-extern error_t argp_err_exit_status;
-
-/* Flags for argp_help.  */
-#define ARGP_HELP_USAGE		0x01 /* a Usage: message. */
-#define ARGP_HELP_SHORT_USAGE	0x02 /*  " but don't actually print options. */
-#define ARGP_HELP_SEE		0x04 /* a `Try ... for more help' message. */
-#define ARGP_HELP_LONG		0x08 /* a long help message. */
-#define ARGP_HELP_PRE_DOC	0x10 /* doc string preceding long help.  */
-#define ARGP_HELP_POST_DOC	0x20 /* doc string following long help.  */
-#define ARGP_HELP_DOC		(ARGP_HELP_PRE_DOC | ARGP_HELP_POST_DOC)
-#define ARGP_HELP_BUG_ADDR	0x40 /* bug report address */
-#define ARGP_HELP_LONG_ONLY	0x80 /* modify output appropriately to
-					reflect ARGP_LONG_ONLY mode.  */
-
-/* These ARGP_HELP flags are only understood by argp_state_help.  */
-#define ARGP_HELP_EXIT_ERR	0x100 /* Call exit(1) instead of returning.  */
-#define ARGP_HELP_EXIT_OK	0x200 /* Call exit(0) instead of returning.  */
-
-/* The standard thing to do after a program command line parsing error, if an
-   error message has already been printed.  */
-#define ARGP_HELP_STD_ERR \
-  (ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do after a program command line parsing error, if no
-   more specific error message has been printed.  */
-#define ARGP_HELP_STD_USAGE \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do in response to a --help option.  */
-#define ARGP_HELP_STD_HELP \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG | ARGP_HELP_EXIT_OK \
-   | ARGP_HELP_DOC | ARGP_HELP_BUG_ADDR)
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  */
-extern void argp_help (__const struct argp *__restrict __argp,
-		       FILE *__restrict __stream,
-		       unsigned __flags, char *__restrict __name) __THROW;
-extern void __argp_help (__const struct argp *__restrict __argp,
-			 FILE *__restrict __stream, unsigned __flags,
-			 char *__name) __THROW;
-
-/* The following routines are intended to be called from within an argp
-   parsing routine (thus taking an argp_state structure as the first
-   argument).  They may or may not print an error message and exit, depending
-   on the flags in STATE -- in any case, the caller should be prepared for
-   them *not* to exit, and should return an appropiate error after calling
-   them.  [argp_usage & argp_error should probably be called argp_state_...,
-   but they're used often enough that they should be short]  */
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-extern void argp_state_help (__const struct argp_state *__restrict __state,
-			     FILE *__restrict __stream,
-			     unsigned int __flags) __THROW;
-extern void __argp_state_help (__const struct argp_state *__restrict __state,
-			       FILE *__restrict __stream,
-			       unsigned int __flags) __THROW;
-
-/* Possibly output the standard usage message for ARGP to stderr and exit.  */
-extern void argp_usage (__const struct argp_state *__state) __THROW;
-extern void __argp_usage (__const struct argp_state *__state) __THROW;
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-extern void argp_error (__const struct argp_state *__restrict __state,
-			__const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-extern void __argp_error (__const struct argp_state *__restrict __state,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-extern void argp_failure (__const struct argp_state *__restrict __state,
-			  int __status, int __errnum,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-extern void __argp_failure (__const struct argp_state *__restrict __state,
-			    int __status, int __errnum,
-			    __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-
-/* Returns true if the option OPT is a valid short option.  */
-extern int _option_is_short (__const struct argp_option *__opt) __THROW;
-extern int __option_is_short (__const struct argp_option *__opt) __THROW;
-
-/* Returns true if the option OPT is in fact the last (unused) entry in an
-   options array.  */
-extern int _option_is_end (__const struct argp_option *__opt) __THROW;
-extern int __option_is_end (__const struct argp_option *__opt) __THROW;
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-extern void *_argp_input (__const struct argp *__restrict __argp,
-			  __const struct argp_state *__restrict __state)
-     __THROW;
-extern void *__argp_input (__const struct argp *__restrict __argp,
-			   __const struct argp_state *__restrict __state)
-     __THROW;
-
-/* Used for extracting the program name from argv[0] */
-extern char *_argp_basename(char *name) __THROW;
-extern char *__argp_basename(char *name) __THROW;
-
-/* Getting the program name given an argp state */
-extern char *
-_argp_short_program_name(const struct argp_state *state) __THROW;
-extern char *
-__argp_short_program_name(const struct argp_state *state) __THROW;
-
-
-#ifdef __USE_EXTERN_INLINES
-
-# if !_LIBC
-#  define __argp_usage argp_usage
-#  define __argp_state_help argp_state_help
-#  define __option_is_short _option_is_short
-#  define __option_is_end _option_is_end
-# endif
-
-# ifndef ARGP_EI
-#  define ARGP_EI extern __inline__
-# endif
-
-ARGP_EI void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-ARGP_EI int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      return __key > 0 && isprint (__key);
-    }
-}
-
-ARGP_EI int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
-
-# if !_LIBC
-#  undef __argp_usage
-#  undef __argp_state_help
-#  undef __option_is_short
-#  undef __option_is_end
-# endif
-#endif /* Use extern inlines.  */
-
-#ifdef  __cplusplus
-}
-#endif
-
-#endif /* argp.h */
diff --git a/argp-standalone/autogen.sh b/argp-standalone/autogen.sh
deleted file mode 100755
index 8337353b5ae..00000000000
--- a/argp-standalone/autogen.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-aclocal -I .
-autoheader
-autoconf
-automake --add-missing --copy --foreign
diff --git a/argp-standalone/configure.ac b/argp-standalone/configure.ac
deleted file mode 100644
index fe54d5ac991..00000000000
--- a/argp-standalone/configure.ac
+++ /dev/null
@@ -1,100 +0,0 @@
-dnl Process this file with autoconf to produce a configure script.
-
-dnl This configure.ac is only for building a standalone argp library.
-AC_INIT([argp], [standalone-1.3])
-AC_PREREQ(2.54)
-AC_CONFIG_SRCDIR([argp-ba.c])
-# Needed to stop autoconf from looking for files in parent directories.
-AC_CONFIG_AUX_DIR([.])
-
-AM_INIT_AUTOMAKE
-AM_CONFIG_HEADER(config.h)
-
-# GNU libc defaults to supplying the ISO C library functions only. The
-# _GNU_SOURCE define enables these extensions, in particular we want
-# errno.h to declare program_invocation_name. Enable it on all
-# systems; no problems have been reported with it so far.
-AC_GNU_SOURCE
-	
-# Checks for programs.
-AC_PROG_CC
-AC_PROG_MAKE_SET
-AC_PROG_RANLIB
-AM_PROG_CC_STDC
-
-if test "x$am_cv_prog_cc_stdc" = xno ; then
-  AC_ERROR([the C compiler doesn't handle ANSI-C])
-fi
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_HEADER_STDC
-AC_CHECK_HEADERS(limits.h malloc.h unistd.h sysexits.h stdarg.h)
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_C_CONST
-AC_C_INLINE
-AC_TYPE_SIZE_T
-
-LSH_GCC_ATTRIBUTES
-
-# Checks for library functions.
-AC_FUNC_ALLOCA
-AC_FUNC_VPRINTF
-AC_CHECK_FUNCS(strerror sleep getpid snprintf)
-
-AC_REPLACE_FUNCS(mempcpy strndup strchrnul strcasecmp vsnprintf)
-
-dnl ARGP_CHECK_FUNC(includes, function-call [, if-found [, if-not-found]])
-AC_DEFUN([ARGP_CHECK_FUNC],
-  [AS_VAR_PUSHDEF([ac_func], m4_substr([$2], 0, m4_index([$2], [(])))
-   AS_VAR_PUSHDEF([ac_var], [ac_cv_func_call_]ac_func)
-   AH_TEMPLATE(AS_TR_CPP(HAVE_[]ac_func),
-               [Define to 1 if you have the `]ac_func[' function.])
-   AC_CACHE_CHECK([for $2], ac_var,
-     [AC_TRY_LINK([$1], [$2],
-		  [AS_VAR_SET(ac_var, yes)],
-		  [AS_VAR_SET(ac_var, no)])])
-   if test AS_VAR_GET(ac_var) = yes ; then
-     ifelse([$3],,
-	    [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_[]ac_func))],
-	    [$3
-])
-   else
-     ifelse([$4],, true, [$4])
-   fi
-   AS_VAR_POPDEF([ac_var])
-   AS_VAR_POPDEF([ac_func])
-   ])
-
-# At least on freebsd, putc_unlocked is a macro, so the standard
-# AC_CHECK_FUNCS doesn't work well.
-ARGP_CHECK_FUNC([#include <stdio.h>], [putc_unlocked('x', stdout)])
-
-AC_CHECK_FUNCS(flockfile)
-AC_CHECK_FUNCS(fputs_unlocked fwrite_unlocked)
-
-# Used only by argp-test.c, so don't use AC_REPLACE_FUNCS.
-AC_CHECK_FUNCS(strdup asprintf)
-
-AC_CHECK_DECLS([program_invocation_name, program_invocation_short_name],
-               [], [], [[#include <errno.h>]])
-
-# Set these flags *last*, or else the test programs won't compile
-if test x$GCC = xyes ; then
-  # Using -ggdb3 makes (some versions of) Redhat's gcc-2.96 dump core
-  if "$CC" --version | grep '^2\.96$' 1>/dev/null 2>&1; then
-    true
-  else
-    CFLAGS="$CFLAGS -ggdb3"
-  fi
-  CFLAGS="$CFLAGS -Wall -W \
- -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes \
- -Waggregate-return \
- -Wpointer-arith -Wbad-function-cast -Wnested-externs"
-fi
-
-CPPFLAGS="$CPPFLAGS -I$srcdir"
-
-AC_OUTPUT(Makefile)
diff --git a/argp-standalone/mempcpy.c b/argp-standalone/mempcpy.c
deleted file mode 100644
index 21d8bd2ed94..00000000000
--- a/argp-standalone/mempcpy.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <string.h>
-
-void *
-mempcpy (void *, const void *, size_t) ;
-
-void *
-mempcpy (void *to, const void *from, size_t size)
-{
-  memcpy(to, from, size);
-  return (char *) to + size;
-}
-
diff --git a/argp-standalone/strcasecmp.c b/argp-standalone/strcasecmp.c
deleted file mode 100644
index 9c1637232fd..00000000000
--- a/argp-standalone/strcasecmp.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* strcasecmp.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <ctype.h>
-int strcasecmp(const char *, const char *);
-
-int strcasecmp(const char *s1, const char *s2)
-{
-  unsigned i;
-  
-  for (i = 0; s1[i] && s2[i]; i++)
-    {
-      unsigned char c1 = tolower( (unsigned char) s1[i]);
-      unsigned char c2 = tolower( (unsigned char) s2[i]);
-
-      if (c1 < c2)
-	return -1;
-      else if (c1 > c2)
-	return 1;
-    }
-
-  return !s2[i] - !s1[i];	
-}
diff --git a/argp-standalone/strchrnul.c b/argp-standalone/strchrnul.c
deleted file mode 100644
index ee4145e4eda..00000000000
--- a/argp-standalone/strchrnul.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* strchrnul.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-/* FIXME: What is this function supposed to do? My guess is that it is
- * like strchr, but returns a pointer to the NUL character, not a NULL
- * pointer, if the character isn't found. */
-
-char *strchrnul(const char *, int );
-
-char *strchrnul(const char *s, int c)
-{
-  const char *p = s;
-  while (*p && (*p != c))
-    p++;
-
-  return (char *) p;
-}
diff --git a/argp-standalone/strndup.c b/argp-standalone/strndup.c
deleted file mode 100644
index 4147b7a2051..00000000000
--- a/argp-standalone/strndup.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels M�ller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <stdlib.h>
-#include <string.h>
-
-char *
-strndup (const char *, size_t);
-
-char *
-strndup (const char *s, size_t size)
-{
-  char *r;
-  char *end = memchr(s, 0, size);
-  
-  if (end)
-    /* Length + 1 */
-    size = end - s + 1;
-  
-  r = malloc(size);
-
-  if (size)
-    {
-      memcpy(r, s, size-1);
-      r[size-1] = '\0';
-    }
-  return r;
-}
diff --git a/argp-standalone/vsnprintf.c b/argp-standalone/vsnprintf.c
deleted file mode 100644
index 33c9a5d0042..00000000000
--- a/argp-standalone/vsnprintf.c
+++ /dev/null
@@ -1,839 +0,0 @@
-/* Copied from http://www.fiction.net/blong/programs/snprintf.c */
-
-/*
- * Copyright Patrick Powell 1995
- * This code is based on code written by Patrick Powell (papowell@astart.com)
- * It may be used for any purpose as long as this notice remains intact
- * on all source code distributions
- */
-
-/**************************************************************
- * Original:
- * Patrick Powell Tue Apr 11 09:48:21 PDT 1995
- * A bombproof version of doprnt (dopr) included.
- * Sigh.  This sort of thing is always nasty do deal with.  Note that
- * the version here does not include floating point...
- *
- * snprintf() is used instead of sprintf() as it does limit checks
- * for string length.  This covers a nasty loophole.
- *
- * The other functions are there to prevent NULL pointers from
- * causing nast effects.
- *
- * More Recently:
- *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43
- *  This was ugly.  It is still ugly.  I opted out of floating point
- *  numbers, but the formatter understands just about everything
- *  from the normal C string format, at least as far as I can tell from
- *  the Solaris 2.5 printf(3S) man page.
- *
- *  Brandon Long <blong@fiction.net> 10/22/97 for mutt 0.87.1
- *    Ok, added some minimal floating point support, which means this
- *    probably requires libm on most operating systems.  Don't yet
- *    support the exponent (e,E) and sigfig (g,G).  Also, fmtint()
- *    was pretty badly broken, it just wasn't being exercised in ways
- *    which showed it, so that's been fixed.  Also, formated the code
- *    to mutt conventions, and removed dead code left over from the
- *    original.  Also, there is now a builtin-test, just compile with:
- *           gcc -DTEST_SNPRINTF -o snprintf snprintf.c -lm
- *    and run snprintf for results.
- * 
- *  Thomas Roessler <roessler@guug.de> 01/27/98 for mutt 0.89i
- *    The PGP code was using unsigned hexadecimal formats. 
- *    Unfortunately, unsigned formats simply didn't work.
- *
- *  Michael Elkins <me@cs.hmc.edu> 03/05/98 for mutt 0.90.8
- *    The original code assumed that both snprintf() and vsnprintf() were
- *    missing.  Some systems only have snprintf() but not vsnprintf(), so
- *    the code is now broken down under HAVE_SNPRINTF and HAVE_VSNPRINTF.
- *
- *  Andrew Tridgell (tridge@samba.org) Oct 1998
- *    fixed handling of %.0f
- *    added test for HAVE_LONG_DOUBLE
- *
- *  Russ Allbery <rra@stanford.edu> 2000-08-26
- *    fixed return value to comply with C99
- *    fixed handling of snprintf(NULL, ...)
- *
- * Niels M�ller <nisse@lysator.liu.se> 2004-03-05
- *    fixed calls to isdigit to use unsigned char.
- *    fixed calls to va_arg; short arguments are always passed as int.
- *
- **************************************************************/
-
-#if HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#if !defined(HAVE_SNPRINTF) || !defined(HAVE_VSNPRINTF)
-
-#include <string.h>
-#include <ctype.h>
-#include <sys/types.h>
-
-/* Define this as a fall through, HAVE_STDARG_H is probably already set */
-
-#define HAVE_VARARGS_H
-
-
-/* varargs declarations: */
-
-#if defined(HAVE_STDARG_H)
-# include <stdarg.h>
-# define HAVE_STDARGS    /* let's hope that works everywhere (mj) */
-# define VA_LOCAL_DECL   va_list ap
-# define VA_START(f)     va_start(ap, f)
-# define VA_SHIFT(v,t)  ;   /* no-op for ANSI */
-# define VA_END          va_end(ap)
-#else
-# if defined(HAVE_VARARGS_H)
-#  include <varargs.h>
-#  undef HAVE_STDARGS
-#  define VA_LOCAL_DECL   va_list ap
-#  define VA_START(f)     va_start(ap)      /* f is ignored! */
-#  define VA_SHIFT(v,t) v = va_arg(ap,t)
-#  define VA_END        va_end(ap)
-# else
-/*XX ** NO VARARGS ** XX*/
-# endif
-#endif
-
-#ifdef HAVE_LONG_DOUBLE
-#define LDOUBLE long double
-#else
-#define LDOUBLE double
-#endif
-
-int snprintf (char *str, size_t count, const char *fmt, ...);
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-
-static int dopr (char *buffer, size_t maxlen, const char *format, 
-                 va_list args);
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-		   char *value, int flags, int min, int max);
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags);
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags);
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c );
-
-/*
- * dopr(): poor man's version of doprintf
- */
-
-/* format read states */
-#define DP_S_DEFAULT 0
-#define DP_S_FLAGS   1
-#define DP_S_MIN     2
-#define DP_S_DOT     3
-#define DP_S_MAX     4
-#define DP_S_MOD     5
-#define DP_S_CONV    6
-#define DP_S_DONE    7
-
-/* format flags - Bits */
-#define DP_F_MINUS 	(1 << 0)
-#define DP_F_PLUS  	(1 << 1)
-#define DP_F_SPACE 	(1 << 2)
-#define DP_F_NUM   	(1 << 3)
-#define DP_F_ZERO  	(1 << 4)
-#define DP_F_UP    	(1 << 5)
-#define DP_F_UNSIGNED 	(1 << 6)
-
-/* Conversion Flags */
-#define DP_C_SHORT   1
-#define DP_C_LONG    2
-#define DP_C_LDOUBLE 3
-
-#define char_to_int(p) (p - '0')
-#define MAX(p,q) ((p >= q) ? p : q)
-#define MIN(p,q) ((p <= q) ? p : q)
-
-static int dopr (char *buffer, size_t maxlen, const char *format, va_list args)
-{
-  unsigned char ch;
-  long value;
-  LDOUBLE fvalue;
-  char *strvalue;
-  int min;
-  int max;
-  int state;
-  int flags;
-  int cflags;
-  int total;
-  size_t currlen;
-  
-  state = DP_S_DEFAULT;
-  currlen = flags = cflags = min = 0;
-  max = -1;
-  ch = *format++;
-  total = 0;
-
-  while (state != DP_S_DONE)
-  {
-    if (ch == '\0')
-      state = DP_S_DONE;
-
-    switch(state) 
-    {
-    case DP_S_DEFAULT:
-      if (ch == '%') 
-	state = DP_S_FLAGS;
-      else 
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-      ch = *format++;
-      break;
-    case DP_S_FLAGS:
-      switch (ch) 
-      {
-      case '-':
-	flags |= DP_F_MINUS;
-        ch = *format++;
-	break;
-      case '+':
-	flags |= DP_F_PLUS;
-        ch = *format++;
-	break;
-      case ' ':
-	flags |= DP_F_SPACE;
-        ch = *format++;
-	break;
-      case '#':
-	flags |= DP_F_NUM;
-        ch = *format++;
-	break;
-      case '0':
-	flags |= DP_F_ZERO;
-        ch = *format++;
-	break;
-      default:
-	state = DP_S_MIN;
-	break;
-      }
-      break;
-    case DP_S_MIN:
-      if (isdigit(ch)) 
-      {
-	min = 10*min + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	min = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_DOT;
-      } 
-      else 
-	state = DP_S_DOT;
-      break;
-    case DP_S_DOT:
-      if (ch == '.') 
-      {
-	state = DP_S_MAX;
-	ch = *format++;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MAX:
-      if (isdigit(ch)) 
-      {
-	if (max < 0)
-	  max = 0;
-	max = 10*max + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	max = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_MOD;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MOD:
-      /* Currently, we don't support Long Long, bummer */
-      switch (ch) 
-      {
-      case 'h':
-	cflags = DP_C_SHORT;
-	ch = *format++;
-	break;
-      case 'l':
-	cflags = DP_C_LONG;
-	ch = *format++;
-	break;
-      case 'L':
-	cflags = DP_C_LDOUBLE;
-	ch = *format++;
-	break;
-      default:
-	break;
-      }
-      state = DP_S_CONV;
-      break;
-    case DP_S_CONV:
-      switch (ch) 
-      {
-      case 'd':
-      case 'i':
-	if (cflags == DP_C_SHORT) 
-	  value = (short) va_arg (args, int);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, long int);
-	else
-	  value = va_arg (args, int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'o':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 8, min, max, flags);
-	break;
-      case 'u':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'X':
-	flags |= DP_F_UP;
-      case 'x':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 16, min, max, flags);
-	break;
-      case 'f':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	/* um, floating point? */
-	total += fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
-	break;
-      case 'E':
-	flags |= DP_F_UP;
-      case 'e':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'G':
-	flags |= DP_F_UP;
-      case 'g':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'c':
-	total += dopr_outch (buffer, &currlen, maxlen, va_arg (args, int));
-	break;
-      case 's':
-	strvalue = va_arg (args, char *);
-	total += fmtstr (buffer, &currlen, maxlen, strvalue, flags, min, max);
-	break;
-      case 'p':
-	strvalue = va_arg (args, void *);
-	total += fmtint (buffer, &currlen, maxlen, (long) strvalue, 16, min,
-                         max, flags);
-	break;
-      case 'n':
-	if (cflags == DP_C_SHORT) 
-	{
-	  short int *num;
-	  num = va_arg (args, short int *);
-	  *num = currlen;
-        } 
-	else if (cflags == DP_C_LONG) 
-	{
-	  long int *num;
-	  num = va_arg (args, long int *);
-	  *num = currlen;
-        } 
-	else 
-	{
-	  int *num;
-	  num = va_arg (args, int *);
-	  *num = currlen;
-        }
-	break;
-      case '%':
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-	break;
-      case 'w':
-	/* not supported yet, treat as next char */
-	ch = *format++;
-	break;
-      default:
-	/* Unknown, skip */
-	break;
-      }
-      ch = *format++;
-      state = DP_S_DEFAULT;
-      flags = cflags = min = 0;
-      max = -1;
-      break;
-    case DP_S_DONE:
-      break;
-    default:
-      /* hmm? */
-      break; /* some picky compilers need this */
-    }
-  }
-  if (buffer != NULL)
-  {
-    if (currlen < maxlen - 1) 
-      buffer[currlen] = '\0';
-    else 
-      buffer[maxlen - 1] = '\0';
-  }
-  return total;
-}
-
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-                   char *value, int flags, int min, int max)
-{
-  int padlen, strln;     /* amount to pad */
-  int cnt = 0;
-  int total = 0;
-  
-  if (value == 0)
-  {
-    value = "<NULL>";
-  }
-
-  for (strln = 0; value[strln]; ++strln); /* strlen */
-  if (max >= 0 && max < strln)
-    strln = max;
-  padlen = min - strln;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justify */
-
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  while (*value && ((max < 0) || (cnt < max)))
-  {
-    total += dopr_outch (buffer, currlen, maxlen, *value++);
-    ++cnt;
-  }
-  while (padlen < 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-  return total;
-}
-
-/* Have to handle DP_F_NUM (ie 0x and 0 alternates) */
-
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags)
-{
-  int signvalue = 0;
-  unsigned long uvalue;
-  char convert[20];
-  int place = 0;
-  int spadlen = 0; /* amount to space pad */
-  int zpadlen = 0; /* amount to zero pad */
-  int caps = 0;
-  int total = 0;
-  
-  if (max < 0)
-    max = 0;
-
-  uvalue = value;
-
-  if(!(flags & DP_F_UNSIGNED))
-  {
-    if( value < 0 ) {
-      signvalue = '-';
-      uvalue = -value;
-    }
-    else
-      if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-	signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-  }
-  
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-
-  do {
-    convert[place++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")
-      [uvalue % (unsigned)base  ];
-    uvalue = (uvalue / (unsigned)base );
-  } while(uvalue && (place < 20));
-  if (place == 20) place--;
-  convert[place] = 0;
-
-  zpadlen = max - place;
-  spadlen = min - MAX (max, place) - (signvalue ? 1 : 0);
-  if (zpadlen < 0) zpadlen = 0;
-  if (spadlen < 0) spadlen = 0;
-  if (flags & DP_F_ZERO)
-  {
-    zpadlen = MAX(zpadlen, spadlen);
-    spadlen = 0;
-  }
-  if (flags & DP_F_MINUS) 
-    spadlen = -spadlen; /* Left Justifty */
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "zpad: %d, spad: %d, min: %d, max: %d, place: %d\n",
-      zpadlen, spadlen, min, max, place));
-#endif
-
-  /* Spaces */
-  while (spadlen > 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --spadlen;
-  }
-
-  /* Sign */
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  /* Zeros */
-  if (zpadlen > 0) 
-  {
-    while (zpadlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --zpadlen;
-    }
-  }
-
-  /* Digits */
-  while (place > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, convert[--place]);
-  
-  /* Left Justified spaces */
-  while (spadlen < 0) {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++spadlen;
-  }
-
-  return total;
-}
-
-static LDOUBLE abs_val (LDOUBLE value)
-{
-  LDOUBLE result = value;
-
-  if (value < 0)
-    result = -value;
-
-  return result;
-}
-
-static LDOUBLE pow10_argp (int exp)
-{
-  LDOUBLE result = 1;
-
-  while (exp)
-  {
-    result *= 10;
-    exp--;
-  }
-  
-  return result;
-}
-
-static long round_argp (LDOUBLE value)
-{
-  long intpart;
-
-  intpart = value;
-  value = value - intpart;
-  if (value >= 0.5)
-    intpart++;
-
-  return intpart;
-}
-
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags)
-{
-  int signvalue = 0;
-  LDOUBLE ufvalue;
-  char iconvert[20];
-  char fconvert[20];
-  int iplace = 0;
-  int fplace = 0;
-  int padlen = 0; /* amount to pad */
-  int zpadlen = 0; 
-  int caps = 0;
-  int total = 0;
-  long intpart;
-  long fracpart;
-  
-  /* 
-   * AIX manpage says the default is 0, but Solaris says the default
-   * is 6, and sprintf on AIX defaults to 6
-   */
-  if (max < 0)
-    max = 6;
-
-  ufvalue = abs_val (fvalue);
-
-  if (fvalue < 0)
-    signvalue = '-';
-  else
-    if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-      signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-
-#if 0
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-#endif
-
-  intpart = ufvalue;
-
-  /* 
-   * Sorry, we only support 9 digits past the decimal because of our 
-   * conversion method
-   */
-  if (max > 9)
-    max = 9;
-
-  /* We "cheat" by converting the fractional part to integer by
-   * multiplying by a factor of 10
-   */
-  fracpart = round_argp ((pow10_argp (max)) * (ufvalue - intpart));
-
-  if (fracpart >= pow10_argp (max))
-  {
-    intpart++;
-    fracpart -= pow10_argp (max);
-  }
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "fmtfp: %f =? %d.%d\n", fvalue, intpart, fracpart));
-#endif
-
-  /* Convert integer part */
-  do {
-    iconvert[iplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[intpart % 10];
-    intpart = (intpart / 10);
-  } while(intpart && (iplace < 20));
-  if (iplace == 20) iplace--;
-  iconvert[iplace] = 0;
-
-  /* Convert fractional part */
-  do {
-    fconvert[fplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[fracpart % 10];
-    fracpart = (fracpart / 10);
-  } while(fracpart && (fplace < 20));
-  if (fplace == 20) fplace--;
-  fconvert[fplace] = 0;
-
-  /* -1 for decimal point, another -1 if we are printing a sign */
-  padlen = min - iplace - max - 1 - ((signvalue) ? 1 : 0); 
-  zpadlen = max - fplace;
-  if (zpadlen < 0)
-    zpadlen = 0;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justifty */
-
-  if ((flags & DP_F_ZERO) && (padlen > 0)) 
-  {
-    if (signvalue) 
-    {
-      total += dopr_outch (buffer, currlen, maxlen, signvalue);
-      --padlen;
-      signvalue = 0;
-    }
-    while (padlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --padlen;
-    }
-  }
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  while (iplace > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, iconvert[--iplace]);
-
-  /*
-   * Decimal point.  This should probably use locale to find the correct
-   * char to print out.
-   */
-  if (max > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '.');
-
-    while (fplace > 0) 
-      total += dopr_outch (buffer, currlen, maxlen, fconvert[--fplace]);
-  }
-
-  while (zpadlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '0');
-    --zpadlen;
-  }
-
-  while (padlen < 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-
-  return total;
-}
-
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c)
-{
-  if (*currlen + 1 < maxlen)
-    buffer[(*currlen)++] = c;
-  return 1;
-}
-
-#ifndef HAVE_VSNPRINTF
-int vsnprintf (char *str, size_t count, const char *fmt, va_list args)
-{
-  if (str != NULL)
-    str[0] = 0;
-  return dopr(str, count, fmt, args);
-}
-#endif /* !HAVE_VSNPRINTF */
-
-#ifndef HAVE_SNPRINTF
-/* VARARGS3 */
-#ifdef HAVE_STDARGS
-int snprintf (char *str,size_t count,const char *fmt,...)
-#else
-int snprintf (va_alist) va_dcl
-#endif
-{
-#ifndef HAVE_STDARGS
-  char *str;
-  size_t count;
-  char *fmt;
-#endif
-  VA_LOCAL_DECL;
-  int total;
-    
-  VA_START (fmt);
-  VA_SHIFT (str, char *);
-  VA_SHIFT (count, size_t );
-  VA_SHIFT (fmt, char *);
-  total = vsnprintf(str, count, fmt, ap);
-  VA_END;
-  return total;
-}
-#endif /* !HAVE_SNPRINTF */
-
-#ifdef TEST_SNPRINTF
-#ifndef LONG_STRING
-#define LONG_STRING 1024
-#endif
-int main (void)
-{
-  char buf1[LONG_STRING];
-  char buf2[LONG_STRING];
-  char *fp_fmt[] = {
-    "%-1.5f",
-    "%1.5f",
-    "%123.9f",
-    "%10.5f",
-    "% 10.5f",
-    "%+22.9f",
-    "%+4.9f",
-    "%01.3f",
-    "%4f",
-    "%3.1f",
-    "%3.2f",
-    "%.0f",
-    "%.1f",
-    NULL
-  };
-  double fp_nums[] = { -1.5, 134.21, 91340.2, 341.1234, 0203.9, 0.96, 0.996, 
-    0.9996, 1.996, 4.136, 0};
-  char *int_fmt[] = {
-    "%-1.5d",
-    "%1.5d",
-    "%123.9d",
-    "%5.5d",
-    "%10.5d",
-    "% 10.5d",
-    "%+22.33d",
-    "%01.3d",
-    "%4d",
-    NULL
-  };
-  long int_nums[] = { -1, 134, 91340, 341, 0203, 0};
-  int x, y;
-  int fail = 0;
-  int num = 0;
-
-  printf ("Testing snprintf format codes against system sprintf...\n");
-
-  for (x = 0; fp_fmt[x] != NULL ; x++)
-    for (y = 0; fp_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), fp_fmt[x], fp_nums[y]);
-      sprintf (buf2, fp_fmt[x], fp_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    fp_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-
-  for (x = 0; int_fmt[x] != NULL ; x++)
-    for (y = 0; int_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), int_fmt[x], int_nums[y]);
-      sprintf (buf2, int_fmt[x], int_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    int_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-  printf ("%d tests failed out of %d.\n", fail, num);
-}
-#endif /* SNPRINTF_TEST */
-
-#endif /* !HAVE_SNPRINTF */
diff --git a/autogen.sh b/autogen.sh
index e20408bf2ea..c8cdc3f89fa 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,8 +1,92 @@
 #!/bin/sh
 
-aclocal
-autoheader
-(libtoolize --automake --copy --force || glibtoolize --automake --copy --force)
-autoconf
-automake --add-missing --copy --foreign
-cd argp-standalone;./autogen.sh
+echo
+echo ... GlusterFS autogen ...
+echo
+
+## Check all dependencies are present
+MISSING=""
+
+# Check for aclocal
+env aclocal --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  ACLOCAL=aclocal
+else
+  MISSING="$MISSING aclocal"
+fi
+
+# Check for autoconf
+env autoconf --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOCONF=autoconf
+else
+  MISSING="$MISSING autoconf"
+fi
+
+# Check for autoheader
+env autoheader --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOHEADER=autoheader
+else
+  MISSING="$MISSING autoheader"
+fi
+
+# Check for automake
+env automake --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOMAKE=automake
+else
+  MISSING="$MISSING automake"
+fi
+
+# Check for libtoolize or glibtoolize
+env libtoolize --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  # libtoolize was found, so use it
+  TOOL=libtoolize
+else
+  # libtoolize wasn't found, so check for glibtoolize
+  env glibtoolize --version > /dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    TOOL=glibtoolize
+  else
+    MISSING="$MISSING libtoolize/glibtoolize"
+  fi
+fi
+
+# Check for tar
+env tar -cf /dev/null /dev/null > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  MISSING="$MISSING tar"
+fi
+
+## If dependencies are missing, warn the user and abort
+if [ "x$MISSING" != "x" ]; then
+  echo "Aborting."
+  echo
+  echo "The following build tools are missing:"
+  echo
+  for pkg in $MISSING; do
+    echo "  * $pkg"
+  done
+  echo
+  echo "Please install them and try again."
+  echo
+  exit 1
+fi
+
+## Do the autogeneration
+echo Running ${ACLOCAL}...
+$ACLOCAL -I ./contrib/aclocal
+echo Running ${AUTOHEADER}...
+$AUTOHEADER
+echo Running ${TOOL}...
+$TOOL --automake --copy --force
+echo Running ${AUTOCONF}...
+$AUTOCONF
+echo Running ${AUTOMAKE}...
+$AUTOMAKE --add-missing --force-missing --copy
+
+# Instruct user on next steps
+echo
+echo "Please proceed with configuring, compiling, and installing."
diff --git a/booster/Makefile.am b/booster/Makefile.am
deleted file mode 100644
index e1c45f3051c..00000000000
--- a/booster/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS=src
-\ No newline at end of file
diff --git a/booster/src/Makefile.am b/booster/src/Makefile.am
deleted file mode 100644
index d7d83abf5ee..00000000000
--- a/booster/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-ldpreload_LTLIBRARIES = libglusterfs-booster.la
-ldpreloaddir = $(libdir)/glusterfs
-noinst_HEADERS = booster_fstab.h booster-fd.h
-libglusterfs_booster_la_SOURCES = booster.c booster_stat.c booster_fstab.c booster-fd.c
-libglusterfs_booster_la_CFLAGS = -I$(top_srcdir)/libglusterfsclient/src/ -D_GNU_SOURCE -D$(GF_HOST_OS) -fPIC -Wall \
-	-pthread $(GF_BOOSTER_CFLAGS) -shared -nostartfiles 
-libglusterfs_booster_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE  \
-	-I$(top_srcdir)/libglusterfsclient/src \
-	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(ARGP_STANDALONE_CPPFLAGS)
-
-libglusterfs_booster_la_LDFLAGS = -module -avoidversion
-libglusterfs_booster_la_LIBADD =  $(top_builddir)/libglusterfs/src/libglusterfs.la $(top_builddir)/libglusterfsclient/src/libglusterfsclient.la
-
-CLEANFILES =
-
-uninstall-local:
-	rm -f $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
-
-install-data-hook:
-	ln -sf libglusterfs-booster.so $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
diff --git a/booster/src/booster-fd.c b/booster/src/booster-fd.c
deleted file mode 100644
index c324b25a4e0..00000000000
--- a/booster/src/booster-fd.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-
-#include "booster-fd.h"
-#include <logging.h>
-#include <mem-pool.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <common-utils.h>
-#include <string.h>
-
-#include <assert.h>
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-/*
-   Allocate in memory chunks of power of 2 starting from 1024B
-   Assumes fdtable->lock is held
-   */
-static inline uint
-gf_roundup_power_of_two (uint nr)
-{
-        uint result = 1;
-
-        if (nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Negative number passed");
-                return -1;
-        }
-        
-        while (result <= nr)
-                result *= 2;
-
-        return result;
-}
-
-#define BOOSTER_NFDBITS (sizeof (unsigned long))
-
-#define BOOSTER_FDMASK(d)             (1UL << ((d) % BOOSTER_NFDBITS))
-#define BOOSTER_FDELT(d)              (d / BOOSTER_NFDBITS)
-#define BOOSTER_FD_SET(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] |= BOOSTER_FDMASK(d))
-#define BOOSTER_FD_CLR(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] &= ~BOOSTER_FDMASK(d))
-#define BOOSTER_FD_ISSET(set, d)      (set->fd_bits[BOOSTER_FDELT(d)] & BOOSTER_FDMASK(d))
-
-inline int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        return BOOSTER_FD_ISSET(fdtable->close_on_exec, fd);
-}
-
-inline void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        BOOSTER_FD_SET(fdtable->close_on_exec, fd);
-}
-
-int
-booster_fdtable_expand (booster_fdtable_t *fdtable, uint nr)
-{
-        fd_t    **oldfds = NULL, **tmp = NULL;
-        uint    oldmax_fds = -1;
-        uint    cpy = 0;
-        int32_t ret = -1, bytes = 0;
-        booster_fd_set_t *oldclose_on_exec = NULL;
-
-        if (fdtable == NULL || nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Invalid argument");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        nr /= (1024 / sizeof (fd_t *));
-        nr = gf_roundup_power_of_two (nr + 1);
-        nr *= (1024 / sizeof (fd_t *));
-
-        oldfds = fdtable->fds;
-        oldmax_fds = fdtable->max_fds;
-        oldclose_on_exec = fdtable->close_on_exec;
-
-        fdtable->fds = CALLOC (nr, sizeof (fd_t *));
-        if (fdtable->fds == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation failed");
-                fdtable->fds = oldfds;
-                oldfds = NULL;
-                ret = -1;
-                goto out;
-        }
-
-        fdtable->max_fds = nr;
-
-        if (oldfds) {
-                cpy = oldmax_fds * sizeof (fd_t *);
-                memcpy (fdtable->fds, oldfds, cpy);
-        }
-
-        /* nr will be either less than 8 or a multiple of 8 */
-        bytes = nr/8;
-        bytes = bytes ? bytes : 1;
-        fdtable->close_on_exec = CALLOC (bytes, 1);
-        if (fdtable->close_on_exec == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation "
-                        "failed");
-                tmp = fdtable->fds;
-                fdtable->fds = oldfds;
-                oldfds = tmp;
-                ret = -1;
-                goto out;
-        }
-
-        if (oldclose_on_exec != NULL) {
-                bytes = oldmax_fds/8;
-                cpy = bytes ? bytes : 1;
-                memcpy (fdtable->close_on_exec, oldclose_on_exec, cpy);
-        }
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD-table expanded: Old: %d,New: %d"
-                , oldmax_fds, nr);
-        ret = 0;
-
-out:
-        FREE (oldfds);
-        FREE (oldclose_on_exec);
-
-        return ret;
-}
-
-booster_fdtable_t *
-booster_fdtable_alloc (void)
-{
-        booster_fdtable_t *fdtable = NULL;
-        int32_t            ret = -1;
-
-        fdtable = CALLOC (1, sizeof (*fdtable));
-        GF_VALIDATE_OR_GOTO ("booster-fd", fdtable, out);
-
-        LOCK_INIT (&fdtable->lock);
-
-        LOCK (&fdtable->lock);
-        {
-                ret = booster_fdtable_expand (fdtable, 0);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (ret == -1) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD-table allocation "
-                        "failed");
-                FREE (fdtable);
-                fdtable = NULL;
-        }
-
-out:
-        return fdtable;
-}
-
-fd_t **
-__booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-
-        if (count == NULL)
-                goto out;
-
-        fds = fdtable->fds;
-        fdtable->fds = calloc (fdtable->max_fds, sizeof (fd_t *));
-        *count = fdtable->max_fds;
-
-out:
-        return fds;
-}
-
-fd_t **
-booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-        if (!fdtable)
-                return NULL;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, count);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fds;
-}
-
-void
-booster_fdtable_destroy (booster_fdtable_t *fdtable)
-{
-        fd_t                    *fd = NULL;
-        fd_t                    **fds = NULL;
-        uint                    fd_count = 0;
-        int                     i = 0;
-
-        if (!fdtable)
-                return;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, &fd_count);
-                FREE (fdtable->fds);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (!fds)
-                goto free_table;
-        
-        for (i = 0; i < fd_count; i++) {
-                fd = fds[i];
-                if (fd != NULL)
-                        fd_unref (fd);
-        }
-        FREE (fds);
-free_table:
-        LOCK_DESTROY (&fdtable->lock);
-        FREE (fdtable);
-}
-
-int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd)
-{
-        int ret = -1;
-        int error = 0;
-
-        if (fdtable == NULL || fdptr == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return -1;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "Requested fd: %d", fd);
-        LOCK (&fdtable->lock);
-        {
-                while (fdtable->max_fds < fd) {
-                        error = 0;
-                        error = booster_fdtable_expand (fdtable,
-                                                        fdtable->max_fds + 1);
-                        if (error) {
-                                gf_log ("booster-fd", GF_LOG_ERROR,
-                                        "Cannot expand fdtable:%s",
-                                        strerror (error));
-                                goto err;
-                        }
-                }
-
-                if (!fdtable->fds[fd]) {
-                        fdtable->fds[fd] = fdptr;
-                        fd_ref (fdptr);
-                        ret = fd;
-                } else
-                        gf_log ("booster-fd", GF_LOG_ERROR, "Cannot allocate fd"
-                                " %d (slot not empty in fdtable)", fd);
-        }
-err:
-        UNLOCK (&fdtable->lock);
-
-        return ret;
-}
-
-void
-booster_fd_put (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                return;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD put: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                return;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                fdtable->fds[fd] = NULL;
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (fdptr)
-                fd_unref (fdptr);
-}
-
-fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD ptr request: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                if (fdptr)
-                        fd_ref (fdptr);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fdptr;
-}
-
-void
-booster_fdptr_put (fd_t *booster_fd)
-{
-        if (booster_fd)
-                fd_unref (booster_fd);
-}
diff --git a/booster/src/booster-fd.h b/booster/src/booster-fd.h
deleted file mode 100644
index 19e249fe318..00000000000
--- a/booster/src/booster-fd.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _BOOSTER_FD_H
-#define _BOOSTER_FD_H
-
-#include <libglusterfsclient.h>
-#include <locking.h>
-#include <list.h>
-
-/* This struct must be updated if the fd_t in fd.h changes.
- * We cannot include those headers here because unistd.h, included
- * by glusterfs headers, conflicts with the syscall prototypes we
- * define for booster.
- */
-struct _fd {
-        pid_t             pid;
-	int32_t           flags;
-        int32_t           refcount;
-        struct list_head  inode_list;
-        struct _inode    *inode;
-        struct _dict     *ctx;
-        gf_lock_t         lock; /* used ONLY for manipulating
-                                   'struct _fd_ctx' array (_ctx).*/
-	struct _fd_ctx   *_ctx;
-};
-typedef struct _fd fd_t;
-
-struct _booster_fd_set {
-        unsigned long fd_bits[0];
-};
-typedef struct _booster_fd_set booster_fd_set_t;
-
-struct _booster_fdtable {
-        booster_fd_set_t *close_on_exec;
-        int               refcount;
-        unsigned int      max_fds;
-        gf_lock_t         lock;
-        fd_t            **fds;
-};
-typedef struct _booster_fdtable booster_fdtable_t;
-
-void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-extern int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd);
-
-extern void
-booster_fd_put (booster_fdtable_t *fdtable, int fd);
-
-extern fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd);
-
-extern void
-booster_fdptr_put (fd_t *fdptr);
-
-extern void
-booster_fdtable_destroy (booster_fdtable_t *fdtable);
-
-booster_fdtable_t *
-booster_fdtable_alloc (void);
-
-#endif /* #ifndef _BOOSTER_FD_H */
diff --git a/booster/src/booster.c b/booster/src/booster.c
deleted file mode 100644
index 41f71e96fd7..00000000000
--- a/booster/src/booster.c
+++ /dev/null
@@ -1,3172 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <dlfcn.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdlib.h>
-#include <inttypes.h>
-#include <libglusterfsclient.h>
-#include <list.h>
-#include <pthread.h>
-#include <sys/xattr.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <ctype.h>
-#include <logging.h>
-#include <utime.h>
-#include <dirent.h>
-#include <sys/statfs.h>
-#include <sys/statvfs.h>
-#include <fcntl.h>
-#include "booster-fd.h"
-
-#ifndef GF_UNIT_KB
-#define GF_UNIT_KB 1024
-#endif
-
-static pthread_mutex_t cwdlock   = PTHREAD_MUTEX_INITIALIZER;
-
-/* attr constructor registers this function with libc's
- * _init function as a function that must be called before
- * the main() of the program.
- */
-static void booster_lib_init (void) __attribute__((constructor));
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-
-extern int pipe (int filedes[2]);
-/* We define these flags so that we can remove fcntl.h from the include path.
- * fcntl.h has certain defines and other lines of code that redirect the
- * application's open and open64 calls to the syscalls defined by
- * libc, for us, thats not a Good Thing (TM).
- */
-#ifndef GF_O_CREAT
-#define GF_O_CREAT      0x40
-#endif
-
-#ifndef GF_O_TRUNC
-#define GF_O_TRUNC      0x200
-#endif
-
-#ifndef GF_O_RDWR
-#define GF_O_RDWR       0x2
-#endif
-
-#ifndef GF_O_WRONLY
-#define GF_O_WRONLY     0x1
-#endif
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-typedef enum {
-        BOOSTER_OPEN,
-        BOOSTER_CREAT
-} booster_op_t;
-
-struct _inode;
-struct _dict;
-
-ssize_t
-write (int fd, const void *buf, size_t count);
-
-/* open, open64, creat */
-static int (*real_open) (const char *pathname, int flags, ...);
-static int (*real_open64) (const char *pathname, int flags, ...);
-static int (*real_creat) (const char *pathname, mode_t mode);
-static int (*real_creat64) (const char *pathname, mode_t mode);
-
-/* read, readv, pread, pread64 */
-static ssize_t (*real_read) (int fd, void *buf, size_t count);
-static ssize_t (*real_readv) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pread) (int fd, void *buf, size_t count,
-                              unsigned long offset);
-static ssize_t (*real_pread64) (int fd, void *buf, size_t count,
-                                uint64_t offset);
-
-/* write, writev, pwrite, pwrite64 */
-static ssize_t (*real_write) (int fd, const void *buf, size_t count);
-static ssize_t (*real_writev) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pwrite) (int fd, const void *buf, size_t count,
-                               unsigned long offset);
-static ssize_t (*real_pwrite64) (int fd, const void *buf, size_t count,
-                                 uint64_t offset);
-
-/* lseek, llseek, lseek64 */
-static off_t (*real_lseek) (int fildes, unsigned long offset, int whence);
-static off_t (*real_lseek64) (int fildes, uint64_t offset, int whence);
-
-/* close */
-static int (*real_close) (int fd);
-
-/* dup dup2 */
-static int (*real_dup) (int fd);
-static int (*real_dup2) (int oldfd, int newfd);
-
-static pid_t (*real_fork) (void);
-static int (*real_mkdir) (const char *pathname, mode_t mode);
-static int (*real_rmdir) (const char *pathname);
-static int (*real_chmod) (const char *pathname, mode_t mode);
-static int (*real_chown) (const char *pathname, uid_t owner, gid_t group);
-static int (*real_fchmod) (int fd, mode_t mode);
-static int (*real_fchown) (int fd, uid_t, gid_t gid);
-static int (*real_fsync) (int fd);
-static int (*real_ftruncate) (int fd, off_t length);
-static int (*real_ftruncate64) (int fd, loff_t length);
-static int (*real_link) (const char *oldpath, const char *newname);
-static int (*real_rename) (const char *oldpath, const char *newpath);
-static int (*real_utimes) (const char *path, const struct timeval times[2]);
-static int (*real_utime) (const char *path, const struct utimbuf *buf);
-static int (*real_mknod) (const char *path, mode_t mode, dev_t dev);
-static int (*real_mkfifo) (const char *path, mode_t mode);
-static int (*real_unlink) (const char *path);
-static int (*real_symlink) (const char *oldpath, const char *newpath);
-static int (*real_readlink) (const char *path, char *buf, size_t bufsize);
-static char * (*real_realpath) (const char *path, char *resolved);
-static DIR * (*real_opendir) (const char *path);
-static struct dirent * (*real_readdir) (DIR *dir);
-static struct dirent64 * (*real_readdir64) (DIR *dir);
-static int (*real_readdir_r) (DIR *dir, struct dirent *entry,
-                              struct dirent **result);
-static int (*real_readdir64_r) (DIR *dir, struct dirent64 *entry,
-                                struct dirent64 **result);
-static int (*real_closedir) (DIR *dh);
-static int (*real___xstat) (int ver, const char *path, struct stat *buf);
-static int (*real___xstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_stat) (const char *path, struct stat *buf);
-static int (*real_stat64) (const char *path, struct stat64 *buf);
-static int (*real___fxstat) (int ver, int fd, struct stat *buf);
-static int (*real___fxstat64) (int ver, int fd, struct stat64 *buf);
-static int (*real_fstat) (int fd, struct stat *buf);
-static int (*real_fstat64) (int fd , struct stat64 *buf);
-static int (*real___lxstat) (int ver, const char *path, struct stat *buf);
-static int (*real___lxstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_lstat) (const char *path, struct stat *buf);
-static int (*real_lstat64) (const char *path, struct stat64 *buf);
-static int (*real_statfs) (const char *path, struct statfs *buf);
-static int (*real_statfs64) (const char *path, struct statfs64 *buf);
-static int (*real_statvfs) (const char *path, struct statvfs *buf);
-static int (*real_statvfs64) (const char *path, struct statvfs64 *buf);
-static ssize_t (*real_getxattr) (const char *path, const char *name,
-                                 void *value, size_t size);
-static ssize_t (*real_lgetxattr) (const char *path, const char *name,
-                                  void *value, size_t size);
-static int (*real_remove) (const char* path);
-static int (*real_lchown) (const char *path, uid_t owner, gid_t group);
-static void (*real_rewinddir) (DIR *dirp);
-static void (*real_seekdir) (DIR *dirp, off_t offset);
-static off_t (*real_telldir) (DIR *dirp);
-
-static ssize_t (*real_sendfile) (int out_fd, int in_fd, off_t *offset,
-                                 size_t count);
-static ssize_t (*real_sendfile64) (int out_fd, int in_fd, off_t *offset,
-                                   size_t count);
-static int (*real_fcntl) (int fd, int cmd, ...);
-static int (*real_chdir) (const char *path);
-static int (*real_fchdir) (int fd);  
-static char * (*real_getcwd) (char *buf, size_t size);
-static int (*real_truncate) (const char *path, off_t length);
-static int (*real_truncate64) (const char *path, loff_t length);
-static int (*real_setxattr) (const char *path, const char *name,
-                             const void *value, size_t size, int flags);
-static int (*real_lsetxattr) (const char *path, const char *name,
-                              const void *value, size_t size, int flags);
-static int (*real_fsetxattr) (int filedes, const char *name,
-                              const void *value, size_t size, int flags);
-
-
-#define RESOLVE(sym) do {                                       \
-                if (!real_##sym)                                \
-                        real_##sym = dlsym (RTLD_NEXT, #sym);   \
-        } while (0)
-
-/*TODO: set proper value */
-#define MOUNT_HASH_SIZE 256
-
-struct booster_mount {
-        dev_t st_dev;
-        glusterfs_handle_t handle;
-        struct list_head device_list;
-};
-typedef struct booster_mount booster_mount_t;
-
-static booster_fdtable_t *booster_fdtable = NULL;
-
-extern int booster_configure (char *confpath);
-/* This is dup'ed every time VMP open/creat wants a new fd.
- * This is needed so we occupy an entry in the process' file
- * table.
- */
-int process_piped_fd = -1;
-
-static int
-booster_get_process_fd ()
-{
-        return real_dup (process_piped_fd);
-}
-
-/* The following two define which file contains
- * the FSTAB configuration for VMP-based usage.
- */
-#define DEFAULT_BOOSTER_CONF    CONFDIR"/booster.conf"
-#define BOOSTER_CONF_ENV_VAR    "GLUSTERFS_BOOSTER_FSTAB"
-
-
-/* The following define which log file is used when
- * using the old mount point bypass approach.
- */
-#define BOOSTER_DEFAULT_LOG     CONFDIR"/booster.log"
-#define BOOSTER_LOG_ENV_VAR     "GLUSTERFS_BOOSTER_LOG"
-
-void
-do_open (int fd, const char *pathname, int flags, mode_t mode, booster_op_t op)
-{
-        char                   *specfile = NULL;
-        char                   *mount_point = NULL; 
-        int32_t                 size = 0;
-        int32_t                 ret = -1;
-        FILE                   *specfp = NULL;
-        glusterfs_file_t        fh = NULL;
-        char                   *logfile = NULL;
-        glusterfs_init_params_t iparams = {
-                .loglevel = "error",
-                .lookup_timeout = 600,
-                .stat_timeout = 600,
-        };
-      
-        gf_log ("booster", GF_LOG_DEBUG, "Opening using MPB: %s", pathname);
-        size = fgetxattr (fd, "user.glusterfs-booster-volfile", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        specfile = calloc (1, size);
-        if (!specfile) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-
-        ret = fgetxattr (fd, "user.glusterfs-booster-volfile", specfile,
-                         size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-    
-        specfp = tmpfile ();
-        if (!specfp) {
-                gf_log ("booster", GF_LOG_ERROR, "Temp file creation failed"
-                        ": %s", strerror (errno));
-                goto out;
-        }
-
-        ret = fwrite (specfile, size, 1, specfp);
-        if (ret != 1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to write volfile: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        fseek (specfp, 0L, SEEK_SET);
-
-        size = fgetxattr (fd, "user.glusterfs-booster-mount", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-        
-        mount_point = calloc (size, sizeof (char));
-        if (!mount_point) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-	
-        ret = fgetxattr (fd, "user.glusterfs-booster-mount", mount_point, size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        logfile = getenv (BOOSTER_LOG_ENV_VAR);
-        if (logfile) {
-                if (strlen (logfile) > 0)
-                        iparams.logfile = strdup (logfile);
-                else
-                        iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        } else {
-                iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Using log-file: %s", iparams.logfile);
-        iparams.specfp = specfp;
-
-        ret = glusterfs_mount (mount_point, &iparams);
-        if (ret == -1) {
-                if (errno != EEXIST) {
-                        gf_log ("booster", GF_LOG_ERROR, "Mount failed over"
-                                " glusterfs");
-                        goto out;
-                } else
-                        gf_log ("booster", GF_LOG_ERROR, "Already mounted");
-        }
-
-        switch (op) {
-        case BOOSTER_OPEN:
-                gf_log ("booster", GF_LOG_TRACE, "Booster open call");
-                fh = glusterfs_open (pathname, flags, mode);
-                break;
-
-        case BOOSTER_CREAT:
-                gf_log ("booster", GF_LOG_TRACE, "Booster create call");
-                fh = glusterfs_creat (pathname, mode);
-                break;
-        }
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Error performing operation");
-                goto out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to get unused FD");
-                goto out;
-        }
-        fh = NULL;
-
-out:
-        if (specfile) {
-                free (specfile);
-        }
-
-        if (specfp) {
-                fclose (specfp);
-        }
-
-        if (mount_point) {
-                free (mount_point);
-        }
-
-        if (fh) {
-                glusterfs_close (fh);
-        }
-
-        return;
-}
-
-int
-vmp_open (const char *pathname, int flags, ...)
-{
-        mode_t                  mode = 0;
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-        va_list                 ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                fh = glusterfs_open (pathname, flags, mode);
-        }
-        else
-                fh = glusterfs_open (pathname, flags);
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP open failed");
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create open fd");
-                goto fh_close_out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map fd into table");
-                goto realfd_close_out;
-        }
-
-        return fd;
-
-realfd_close_out:
-        real_close (fd);
-        fd = -1;
-
-fh_close_out:
-        glusterfs_close (fh);
-
-out:
-        return fd;
-}
-
-#define BOOSTER_USE_OPEN64          1
-#define BOOSTER_DONT_USE_OPEN64     0
-
-int
-booster_open (const char *pathname, int use64, int flags, ...)
-{
-        int     ret = -1;
-        mode_t  mode = 0;
-        va_list ap;
-        int     (*my_open) (const char *pathname, int flags, ...);
-
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Open: %s", pathname);
-        /* First try opening through the virtual mount point.
-         * The difference lies in the fact that:
-         * 1. We depend on libglusterfsclient library to perform
-         * the translation from the path to handle.
-         * 2. We do not go to the file system for the fd, instead
-         * we use booster_get_process_fd (), which returns a dup'ed
-         * fd of a pipe created in booster_init.
-         */
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-                ret = vmp_open (pathname, flags, mode);
-        }
-        else
-                ret = vmp_open (pathname, flags);
-
-        /* We receive an ENODEV if the VMP does not exist. If we
-         * receive an error other than ENODEV, it means, there
-         * actually was an error performing vmp_open. This must
-         * be returned to the user.
-         */
-        if ((ret < 0) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Error in opening file over "
-                        " VMP: %s", strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File opened");
-                goto out;
-        }
-
-        if (use64) {
-                gf_log ("booster", GF_LOG_TRACE, "Using 64-bit open");
-		my_open = real_open64;
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Using 32-bit open");
-		my_open = real_open;
-        }
-
-        /* It is possible the RESOLVE macro is not able
-         * to resolve the symbol of a function, in that case
-         * we dont want to seg-fault on calling a NULL functor.
-         */
-        if (my_open == NULL) {
-                gf_log ("booster", GF_LOG_ERROR, "open not resolved");
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-	if (flags & GF_O_CREAT) {
-		va_start (ap, flags);
-		mode = va_arg (ap, mode_t);
-		va_end (ap);
-
-                ret = my_open (pathname, flags, mode);
-	} else
-                ret = my_open (pathname, flags);
-
-        if (ret != -1) {
-                do_open (ret, pathname, flags, mode, BOOSTER_OPEN);
-        }
-
-out:
-        return ret;
-}
-
-/* This is done to over-write existing definitions of open and open64 inside
- * libc with our own copies. __REDIRECT is provided by libc.
- *
- * XXX: This will not work anywhere other than libc based systems.
- */
-int __REDIRECT (booster_false_open, (__const char *__file, int __oflag, ...),
-                open) __nonnull ((1));
-int __REDIRECT (booster_false_open64, (__const char *__file, int __oflag, ...),
-                open64) __nonnull ((1));
-int
-booster_false_open (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags,
-                                    mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-booster_false_open64 (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags, mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-vmp_creat (const char *pathname, mode_t mode)
-{
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-
-        fh = glusterfs_creat (pathname, mode);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Create failed: %s: %s",
-                        pathname, strerror (errno));
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create fd");
-                goto close_out;
-        }
-
-        if ((booster_fd_unused_get (booster_fdtable, fh, fd)) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map unused fd");
-                goto real_close_out;
-        }
-
-        return fd;
-
-real_close_out:
-        real_close (fd);
-        fd = -1;
-
-close_out:
-        glusterfs_close (fh);
-
-out:
-        return -1;
-}
-
-int __REDIRECT (booster_false_creat, (const char *pathname, mode_t mode),
-                creat) __nonnull ((1));
-int __REDIRECT (booster_false_creat64, (const char *pathname, mode_t mode),
-                creat64) __nonnull ((1));
-
-int
-booster_false_creat (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_creat64 (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat64 (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-/* pread */
-
-ssize_t
-pread (int fd, void *buf, size_t count, unsigned long offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread: fd %d, count %lu, offset %lu"
-                ,fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pread64 (int fd, void *buf, size_t count, uint64_t offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread64: fd %d, count %lu, offset %"
-                PRIu64, fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-read (int fd, void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd;
-
-        gf_log ("booster", GF_LOG_TRACE, "read: fd %d, count %lu", fd,
-                (long unsigned)count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_read == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_read (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_read (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-readv (int fd, const struct iovec *vector, int count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "readv: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_readv == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_readv (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		ret = glusterfs_readv (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-write (int fd, const void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "write: fd %d, count %"GF_PRI_SIZET,
-                fd, count);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_write (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_write (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-ssize_t
-writev (int fd, const struct iovec *vector, int count)
-{
-        int ret = 0;
-        glusterfs_file_t glfs_fd = 0; 
-
-        gf_log ("booster", GF_LOG_TRACE, "writev: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_writev == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_writev (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_writev (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite (int fd, const void *buf, size_t count, unsigned long offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite: fd %d, count %"GF_PRI_SIZET
-                ", offset %lu", fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite64 (int fd, const void *buf, size_t count, uint64_t offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite64: fd %d, count %"GF_PRI_SIZET
-                ", offset %"PRIu64, fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-close (int fd)
-{
-        int ret = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "close: fd %d", fd);
-	glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-    
-	if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		booster_fd_put (booster_fdtable, fd);
-		ret = glusterfs_close (glfs_fd);
-		booster_fdptr_put (glfs_fd);
-	}
-
-        ret = real_close (fd);
-
-        return ret;
-}
-
-#ifndef _LSEEK_DECLARED
-#define _LSEEK_DECLARED
-off_t
-lseek (int filedes, unsigned long offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %ld",
-                filedes, offset);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek (filedes, offset, whence);
-        }
-
-        return ret;
-}
-#endif
-
-off_t
-lseek64 (int filedes, uint64_t offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %"PRIu64,
-                filedes, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek64 (filedes, offset, whence);
-        }
-
-        return ret;
-}
-
-int 
-dup (int oldfd)
-{
-        int ret = -1, new_fd = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "dup: fd %d", oldfd);
-        glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_fd = real_dup (oldfd);
-
-        if (new_fd >=0 && glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = booster_fd_unused_get (booster_fdtable, glfs_fd,
-                                             new_fd);
-                fd_ref ((fd_t *)glfs_fd);
-                if (ret == -1) {
-                        gf_log ("booster", GF_LOG_ERROR,"Failed to map new fd");
-                        real_close (new_fd);
-                } 
-        }
-
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return new_fd;
-}
-
-
-int 
-dup2 (int oldfd, int newfd)
-{
-        int ret = -1;
-        glusterfs_file_t old_glfs_fd = NULL, new_glfs_fd = NULL;
-
-        if (oldfd == newfd) {
-                return newfd;
-        }
-
-        old_glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_glfs_fd = booster_fdptr_get (booster_fdtable, newfd);
- 
-        ret = real_dup2 (oldfd, newfd); 
-        if (ret >= 0) {
-                if (new_glfs_fd) {
-                        glusterfs_close (new_glfs_fd);
-                        booster_fdptr_put (new_glfs_fd);
-                        booster_fd_put (booster_fdtable, newfd);
-                        new_glfs_fd = 0;
-                }
-
-                if (old_glfs_fd) {
-                        ret = booster_fd_unused_get (booster_fdtable,
-                                                     old_glfs_fd, newfd);
-                        fd_ref ((fd_t *)old_glfs_fd);
-                        if (ret == -1) {
-                                real_close (newfd);
-                        }
-                }
-        } 
-
-        if (old_glfs_fd) {
-                booster_fdptr_put (old_glfs_fd);
-        }
-
-        if (new_glfs_fd) {
-                booster_fdptr_put (new_glfs_fd);
-        }
-
-        return ret;
-}
-
-int
-mkdir (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkdir: path %s", pathname);
-        ret = glusterfs_mkdir (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory created");
-                return ret;
-        }
-
-        if (real_mkdir == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-        } else
-                ret = real_mkdir (pathname, mode);
-
-        return ret;
-}
-
-int
-rmdir (const char *pathname)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "rmdir: path %s", pathname);
-        ret = glusterfs_rmdir (pathname);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "rmdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory removed");
-                return ret;
-        }
-
-        if (real_rmdir == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rmdir (pathname);
-
-        return ret;
-}
-
-int
-chmod (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chmod: path %s", pathname);
-        ret = glusterfs_chmod (pathname, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chmod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chmod succeeded");
-                return ret;
-        }
-
-        if (real_chmod == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chmod (pathname, mode);
-
-        return ret;
-}
-
-int
-chown (const char *pathname, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chown: path: %s", pathname);
-        ret = glusterfs_chown (pathname, owner, group);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chown failed: %s\n",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chown succeeded");
-                return ret;
-        }
-
-        if (real_chown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chown (pathname, owner, group);
-
-        return ret;
-}
-
-int
-fchown (int fd, uid_t owner, gid_t group)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchown: fd %d, uid %d, gid %d", fd,
-                owner, group);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchown == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchown (fd, owner, group);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchown (fh, owner, group);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-
-#define MOUNT_TABLE_HASH_SIZE 256
-
-
-static void booster_cleanup (void);
-static int 
-booster_init (void)
-{
-        char    *booster_conf_path = NULL;
-        int     ret = -1;
-        int     pipefd[2];
-
-        booster_fdtable = booster_fdtable_alloc ();
-        if (!booster_fdtable) {
-                fprintf (stderr, "cannot allocate fdtable: %s\n",
-                         strerror (errno));
-		goto err;
-        }
- 
-        if (pipe (pipefd) == -1) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Pipe creation failed:%s"
-                        , strerror (errno));
-                goto err;
-        }
-
-        process_piped_fd = pipefd[0];
-        real_close (pipefd[1]);
-        /* libglusterfsclient based VMPs should be inited only
-         * after the file tables are inited so that if the socket
-         * calls use the fd based syscalls, the fd tables are
-         * correctly initialized to return a NULL handle, on which the
-         * socket calls will fall-back to the real API.
-         */
-        booster_conf_path = getenv (BOOSTER_CONF_ENV_VAR);
-        if (booster_conf_path != NULL) {
-                if (strlen (booster_conf_path) > 0)
-                        ret = booster_configure (booster_conf_path);
-                else {
-                        gf_log ("booster", GF_LOG_ERROR, "%s not defined, "
-                                "using default path: %s", BOOSTER_CONF_ENV_VAR,
-                                DEFAULT_BOOSTER_CONF);
-                        ret = booster_configure (DEFAULT_BOOSTER_CONF);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_ERROR, "%s not defined, using default"
-                        " path: %s", BOOSTER_CONF_ENV_VAR,DEFAULT_BOOSTER_CONF);
-                ret = booster_configure (DEFAULT_BOOSTER_CONF);
-        }
-
-        atexit (booster_cleanup);
-        if (ret == 0)
-                gf_log ("booster", GF_LOG_DEBUG, "booster is inited");
-	return 0;
-
-err:
-        /* Sure we return an error value here
-         * but who cares about booster.
-         */
-	return -1; 
-}
-
-
-static void
-booster_cleanup (void)
-{
-        /* Ideally, we should be de-initing the fd-table
-         * here but the problem is that I've seen file accesses through booster
-         * continuing while the atexit registered function is called. That means
-         * , we cannot dealloc the fd-table since then there could be a crash
-         * while trying to determine whether a given fd is for libc or for
-         * libglusterfsclient.
-         * We should be satisfied with having cleaned up glusterfs contexts.
-         */
-        glusterfs_umount_all ();
-	glusterfs_reset ();
-}
-
-int
-fchmod (int fd, mode_t mode)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchmod: fd %d, mode: 0x%x", fd, mode);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchmod == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchmod (fd, mode);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchmod (fh, mode);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-fsync (int fd)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsync: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsync == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fsync (fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsync (fh);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int __REDIRECT (booster_false_ftruncate, (int fd, off_t length),
-                ftruncate);
-int __REDIRECT (booster_false_ftruncate64, (int fd, loff_t length),
-                ftruncate64);
-
-int
-booster_false_ftruncate (int fd, off_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-booster_false_ftruncate64 (int fd, loff_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate64 (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-link (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_link (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Link failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "link call succeeded");
-                return ret;
-        }
-
-        if (real_link == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_link (old, new);
-
-        return ret;
-}
-
-int
-rename (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_rename (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Rename failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "Rename succeeded");
-                return ret;
-        }
-
-        if (real_rename == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rename (old, new);
-
-        return ret;
-}
-
-int
-utimes (const char *path, const struct timeval times[2])
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utimes: path %s", path);
-        ret = glusterfs_utimes (path, times);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utimes failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utimes succeeded");
-                return ret;
-        }
-
-        if (real_utimes == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utimes (path, times);
-
-        return ret;
-}
-
-int
-utime (const char *path, const struct utimbuf *buf)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utime: path %s", path);
-        ret = glusterfs_utime (path, buf);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utime failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utime succeeded");
-                return ret;
-        }
-
-        if (real_utime == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utime (path, buf);
-
-        return ret;
-}
-
-int
-mknod (const char *path, mode_t mode, dev_t dev)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mknod: path %s", path);
-        ret = glusterfs_mknod (path, mode, dev);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mknod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mknod succeeded");
-                return ret;
-        }
-
-        if (real_mknod) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mknod (path, mode, dev);
-
-        return ret;
-}
-
-int
-mkfifo (const  char *path, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkfifo: path %s", path);
-        ret = glusterfs_mkfifo (path, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkfifo failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mkfifo succeeded");
-                return ret;
-        }
-
-        if (real_mkfifo == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mkfifo (path, mode);
-
-        return ret;
-}
-
-int
-unlink (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "unlink: path %s", path);
-        ret = glusterfs_unlink (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "unlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "unlink succeeded");
-                return ret;
-        }
-
-        if (real_unlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_unlink (path);
-
-        return ret;
-}
-
-int
-symlink (const char *oldpath, const char *newpath)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "symlink: old: %s, new: %s",
-                oldpath, newpath);
-        ret = glusterfs_symlink (oldpath, newpath);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "symlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "symlink succeeded");
-                return ret;
-        }
-
-        if (real_symlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_symlink (oldpath, newpath);
-
-        return ret;
-}
-
-int
-readlink (const char *path, char *buf, size_t bufsize)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "readlink: path %s", path);
-        ret = glusterfs_readlink (path, buf, bufsize);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "readlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "readlink succeeded");
-                return ret;
-        }
-
-        if (real_readlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_readlink (path, buf, bufsize);
-
-        return ret;
-}
-
-char *
-realpath (const char *path, char *resolved_path)
-{
-        char    *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "realpath: path %s", path);
-        res = glusterfs_realpath (path, resolved_path);
-        if ((res == NULL) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "realpath failed: %s",
-                        strerror (errno));
-                return res;
-        }
-
-        if (res != NULL) {
-                gf_log ("booster", GF_LOG_TRACE, "realpath succeeded");
-                return res;
-        }
-
-        if (real_realpath == NULL) {
-                errno = ENOSYS;
-                res = NULL;
-        } else
-                res = real_realpath (path, resolved_path);
-
-        return res;
-}
-
-#define BOOSTER_GL_DIR          1
-#define BOOSTER_POSIX_DIR       2
-
-struct booster_dir_handle {
-        int type;
-        void *dirh;
-};
-
-DIR *
-opendir (const char *path)
-{
-        glusterfs_dir_t                 gdir = NULL;
-        struct booster_dir_handle       *bh = NULL;
-        DIR                             *pdir = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "opendir: path: %s", path);
-        bh = calloc (1, sizeof (struct booster_dir_handle));
-        if (!bh) {
-                gf_log ("booster", GF_LOG_ERROR, "memory allocation failed");
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gdir = glusterfs_opendir (path);
-        if (gdir) {
-                gf_log ("booster", GF_LOG_TRACE, "Gluster dir opened");
-                bh->type = BOOSTER_GL_DIR;
-                bh->dirh = (void *)gdir;
-                goto out;
-        } else if ((!gdir) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Opendir failed");
-                goto free_out;
-        }
-
-        if (real_opendir == NULL) {
-                errno = ENOSYS;
-                goto free_out;
-        }
-
-        pdir = real_opendir (path);
-
-        if (pdir) {
-                bh->type = BOOSTER_POSIX_DIR;
-                bh->dirh = (void *)pdir;
-                goto out;
-        }
-
-free_out:
-        if (bh) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return (DIR *)bh;
-}
-
-int __REDIRECT (booster_false_readdir_r, (DIR *dir, struct dirent *entry,
-                struct dirent **result), readdir_r) __nonnull ((1));
-int __REDIRECT (booster_false_readdir64_r, (DIR *dir, struct dirent64 *entry,
-                struct dirent64 **result), readdir64_r) __nonnull ((1));
-
-int
-booster_false_readdir_r (DIR *dir, struct dirent *entry, struct dirent **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh, entry,
-                                           result);
-                
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-int
-booster_false_readdir64_r (DIR *dir, struct dirent64 *entry,
-                           struct dirent64 **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh,
-                                           (struct dirent *)entry,
-                                           (struct dirent **)result);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir64_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir64_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-struct dirent *
-__REDIRECT (booster_false_readdir, (DIR *dir), readdir) __nonnull ((1));
-
-struct dirent64 *
-__REDIRECT (booster_false_readdir64, (DIR *dir), readdir64) __nonnull ((1));
-
-struct dirent *
-booster_false_readdir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent                   *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-struct dirent64 *
-booster_false_readdir64 (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent64                 *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir64 ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-int
-closedir (DIR *dh)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dh;
-        int                             ret = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on gluster");
-                ret = glusterfs_closedir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on posix");
-                if (real_closedir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_closedir ((DIR *)bh->dirh);
-        } else {
-                errno = EBADF;
-        }
-
-        if (ret == 0) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return ret;
-}
-
-/* The real stat functions reside in booster_stat.c to
- * prevent clash with the statX prototype and functions
- * declared from sys/stat.h
- */
-int
-booster_xstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat succeeded");
-                goto out;
-        }
-
-        if (real___xstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___xstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_xstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat64: path: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat64 succeeded");
-                goto out;
-        }
-
-        if (real___xstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___xstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_stat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat succeeded");
-                goto out;
-        }
-
-        if (real_stat != NULL)
-                ret = real_stat (path, sbuf);
-        else if (real___xstat != NULL)
-                ret = real___xstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_stat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat64 succeeded");
-                goto out;
-        }
-
-        if (real_stat64 != NULL)
-                ret = real_stat64 (path, sbuf);
-        else if (real___xstat64 != NULL)
-                ret = real___xstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat (int ver, int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = real___fxstat (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat64 (int ver, int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat64 == NULL) {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-                ret = real___fxstat64 (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat (int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat != NULL)
-                        ret = real_fstat (fd, sbuf);
-                else if (real___fxstat != NULL)
-                        ret = real___fxstat (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat64 (int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat64 != NULL)
-                        ret = real_fstat64 (fd, sbuf);
-                else if (real___fxstat64 != NULL)
-                        /* Not sure how portable the use of 0 for
-                         * version number is but it works over glibc.
-                         * We need this because, I've
-                         * observed that all the above real* functors can be
-                         * NULL. In that case, this is our last and only option.
-                         */
-                        ret = real___fxstat64 (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_lxstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat succeeded");
-                goto out;
-        }
-
-        if (real___lxstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___lxstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lxstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat64 succeeded");
-                goto out;
-        }
-
-        if (real___lxstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___lxstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lstat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat succeeded");
-                goto out;
-        }
-
-        if (real_lstat != NULL)
-                ret = real_lstat (path, sbuf);
-        else if (real___lxstat != NULL)
-                ret = real___lxstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_lstat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat64 succeeded");
-                goto out;
-        }
-
-        if (real_lstat64 != NULL)
-                ret = real_lstat64 (path, sbuf);
-        else if (real___lxstat64 != NULL)
-                ret = real___lxstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_statfs (const char *pathname, struct statfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statfs: path %s", pathname);
-        ret = glusterfs_statfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs succeeded");
-                goto out;
-        }
-
-        if (real_statfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statfs64 (const char *pathname, struct statfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: path %s", pathname);
-        ret = glusterfs_statfs (pathname, (struct statfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs (const char *pathname, struct statvfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs succeeded");
-                goto out;
-        }
-
-        if (real_statvfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs64 (const char *pathname, struct statvfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs64: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, (struct statvfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statvfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-ssize_t
-getxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "getxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_getxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "getxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "getxattr succeeded");
-                return ret;
-        }
-
-        if (real_getxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_getxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-
-ssize_t
-lgetxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lgetxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_lgetxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lgetxattr failed: %s",
-                        strerror (errno));
-
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lgetxattr succeeded");
-                return ret;
-        }
-
-        if (real_lgetxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_lgetxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-int
-remove (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "remove: %s", path);
-        ret = glusterfs_remove (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "remove failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "remove succeeded");
-                goto out;
-        }
-
-        if (real_remove == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_remove (path);
-
-out:
-        return ret;
-}
-
-int
-lchown (const char *path, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lchown: path %s", path);
-        ret = glusterfs_lchown (path, owner, group);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown succeeded");
-                goto out;
-        }
-
-        if (real_lchown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_lchown (path, owner, group);
-
-out:
-        return ret;
-}
-
-void
-booster_rewinddir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on glusterfs");
-                glusterfs_rewinddir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_rewinddir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on posix");
-                real_rewinddir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-void
-booster_seekdir (DIR *dir, off_t offset)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on glusterfs");
-                glusterfs_seekdir ((glusterfs_dir_t)bh->dirh, offset);
-         } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_seekdir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on posix");
-                real_seekdir ((DIR *)bh->dirh, offset);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-off_t
-booster_telldir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        off_t	offset = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "telldir on glusterfs");
-                offset = glusterfs_telldir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_telldir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "telldir on posix");
-                offset = real_telldir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return offset;
-}
-
-
-pid_t 
-fork (void)
-{
-	pid_t pid = 0;
-	char child = 0;
-
-	glusterfs_log_lock ();
-	{
-		pid = real_fork ();
-	}
-	glusterfs_log_unlock ();
-
-	child = (pid == 0);
-	if (child) {
-		booster_cleanup ();
-		booster_init ();
-	}
-
-	return pid;
-}
-
-ssize_t
-sendfile (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile: in fd %d, out fd %d, offset"
-                " %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd, *offset,
-                count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-ssize_t
-sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile64: in fd %d, out fd %d,"
-                " offset %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd,
-                *offset, count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile64 (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-
-int
-fcntl (int fd, int cmd, ...)
-{
-        va_list           ap;
-        int               ret = -1;
-        long              arg = 0;
-        struct flock     *lock = NULL;
-        glusterfs_file_t  glfs_fd = 0; 
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        gf_log ("booster", GF_LOG_TRACE, "fcntl: fd %d, cmd %d", fd, cmd);
-	switch (cmd) {
-	case F_DUPFD:
-                ret = dup (fd);
-                break;
-                /* 
-                 * FIXME: Consider this case when implementing F_DUPFD, F_GETFD
-                 *        etc flags in libglusterfsclient. Commenting it out for
-                 *        timebeing since it is defined only in linux kernel 
-                 *        versions >= 2.6.24.
-                 */
-                /* case F_DUPFD_CLOEXEC: */
-	case F_GETFD:
-                if (glfs_fd != NULL) {
-                        ret = booster_get_close_on_exec (booster_fdtable, fd)
-                                ? FD_CLOEXEC : 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_GETFL:
-	case F_GETOWN:
-	case F_GETSIG:
-	case F_GETLEASE:
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd);
-                }
-		break;
-
-	case F_SETFD:
-                if (glfs_fd != NULL) {
-                        booster_set_close_on_exec (booster_fdtable, fd);
-                        ret = 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_SETFL:
-	case F_SETOWN:
-	case F_SETSIG:
-	case F_SETLEASE:
-	case F_NOTIFY:
-                va_start (ap, cmd);
-                arg = va_arg (ap, long);
-                va_end (ap);
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, arg);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, arg);
-                }
-		break;
-
-	case F_GETLK:
-	case F_SETLK:
-	case F_SETLKW:
-#if F_GETLK != F_GETLK64 
-        case F_GETLK64:
-#endif
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-                va_start (ap, cmd);
-                lock = va_arg (ap, struct flock *);
-                va_end (ap);
-
-                if (lock == NULL) {
-                        errno = EINVAL;
-                        goto out;
-                }
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, lock);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, lock);
-                }
-		break;
-
-	default:
-                errno = EINVAL;
-		break;
-	}
-
-out:
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-chdir (const char *path)
-{
-        int     ret = -1;
-        char    cwd[PATH_MAX];
-        char   *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "chdir: path %s", path);
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                res = glusterfs_getcwd (cwd, PATH_MAX);
-                if (res == NULL) {
-                        gf_log ("booster", GF_LOG_ERROR, "getcwd failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                ret = glusterfs_chdir (path);
-                if ((ret == -1) && (errno != ENODEV)) {
-                        gf_log ("booster", GF_LOG_ERROR, "chdir failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                if (ret == 0) {
-                        gf_log ("booster", GF_LOG_TRACE, "chdir succeeded");
-                        goto unlock;
-                }
-
-                if (real_chdir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto unlock;
-                }
-
-                ret = real_chdir (path);
-                if (ret == -1) {
-                        glusterfs_chdir (cwd);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&cwdlock);
-
-        return ret;
-}
-
-
-int
-fchdir (int fd)
-{
-        int              ret     = -1;
-        glusterfs_file_t glfs_fd = 0;
-        char             cwd[PATH_MAX]; 
-        char            *res     = NULL;
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_fchdir (fd);
-                        if (ret == 0) {
-                                res = real_getcwd (cwd, PATH_MAX);
-                                if (res == NULL) {
-                                        gf_log ("booster", GF_LOG_ERROR,
-                                                "getcwd failed (%s)",
-                                                strerror (errno));
-                                        ret = -1;
-                                } else {
-                                        glusterfs_chdir (cwd);
-                                }
-                        }
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchdir (glfs_fd);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-
-char *
-getcwd (char *buf, size_t size)
-{
-        char *res = NULL;
-
-        res = glusterfs_getcwd (buf, size);
-        if ((res == NULL) && (errno == ENODEV)) {
-                res = real_getcwd (buf, size);
-        }
-
-        return res;
-}
-
-
-int __REDIRECT (booster_false_truncate, (const char *path, off_t length),
-                truncate) __nonnull ((1));
-int __REDIRECT (booster_false_truncate64, (const char *path, loff_t length),
-                truncate64) __nonnull ((1));;
-
-int
-booster_false_truncate (const char *path, off_t length)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "truncate: path (%s) length (%"PRIu64
-                ")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate succeeded");
-                goto out;
-        }
-
-        if (real_truncate != NULL)
-                ret = real_truncate (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_truncate64 (const char *path, loff_t length)
-{
-        int             ret = -1;
-  
-        gf_log ("booster", GF_LOG_TRACE, "truncate64: path (%s) length "
-                "(%"PRIu64")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate64 succeeded");
-                goto out;
-        }
-
-        if (real_truncate64 != NULL)
-                ret = real_truncate64 (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-setxattr (const char *path, const char *name, const void *value, size_t size,
-          int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "setxattr: path: %s", path);
-        ret = glusterfs_setxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "setxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "setxattr succeeded");
-                goto out;
-        }
-
-        if (real_setxattr != NULL)
-                ret = real_setxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-lsetxattr (const char *path, const char *name, const void *value, size_t size,
-           int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lsetxattr: path: %s", path);
-        ret = glusterfs_lsetxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lsetxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lsetxattr succeeded");
-                goto out;
-        }
-
-        if (real_lsetxattr != NULL)
-                ret = real_lsetxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-fsetxattr (int fd, const char *name, const void *value, size_t size, int flags)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsetxattr: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsetxattr != NULL)
-                        ret = real_fsetxattr (fd, name, value, size, flags);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsetxattr (fh, name, value, size, flags);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-
-void
-booster_lib_init (void)
-{
-
-        RESOLVE (open);
-        RESOLVE (open64);
-        RESOLVE (creat);
-        RESOLVE (creat64);
-
-        RESOLVE (read);
-        RESOLVE (readv);
-        RESOLVE (pread);
-        RESOLVE (pread64);
-
-        RESOLVE (write);
-        RESOLVE (writev);
-        RESOLVE (pwrite);
-        RESOLVE (pwrite64);
-
-        RESOLVE (lseek);
-        RESOLVE (lseek64);
-
-        RESOLVE (close);
-
-        RESOLVE (dup);
-        RESOLVE (dup2);
-
-	RESOLVE (fork); 
-        RESOLVE (mkdir);
-        RESOLVE (rmdir);
-        RESOLVE (chmod);
-        RESOLVE (chown);
-        RESOLVE (fchmod);
-        RESOLVE (fchown);
-        RESOLVE (fsync);
-        RESOLVE (ftruncate);
-        RESOLVE (ftruncate64);
-        RESOLVE (link);
-        RESOLVE (rename);
-        RESOLVE (utimes);
-        RESOLVE (utime);
-        RESOLVE (mknod);
-        RESOLVE (mkfifo);
-        RESOLVE (unlink);
-        RESOLVE (symlink);
-        RESOLVE (readlink);
-        RESOLVE (realpath);
-        RESOLVE (opendir);
-        RESOLVE (readdir);
-        RESOLVE (readdir64);
-        RESOLVE (closedir);
-        RESOLVE (__xstat);
-        RESOLVE (__xstat64);
-        RESOLVE (stat);
-        RESOLVE (stat64);
-        RESOLVE (__fxstat);
-        RESOLVE (__fxstat64);
-        RESOLVE (fstat);
-        RESOLVE (fstat64);
-        RESOLVE (__lxstat);
-        RESOLVE (__lxstat64);
-        RESOLVE (lstat);
-        RESOLVE (lstat64);
-        RESOLVE (statfs);
-        RESOLVE (statfs64);
-        RESOLVE (statvfs);
-        RESOLVE (statvfs64);
-        RESOLVE (getxattr);
-        RESOLVE (lgetxattr);
-        RESOLVE (remove);
-        RESOLVE (lchown);
-	RESOLVE (rewinddir);
-	RESOLVE (seekdir);
-	RESOLVE (telldir);
-        RESOLVE (sendfile);
-        RESOLVE (sendfile64);
-        RESOLVE (readdir_r);
-        RESOLVE (readdir64_r);
-        RESOLVE (fcntl);
-        RESOLVE (chdir);
-        RESOLVE (fchdir);
-        RESOLVE (getcwd);
-        RESOLVE (truncate);
-        RESOLVE (truncate64);
-        RESOLVE (setxattr);
-        RESOLVE (lsetxattr);
-        RESOLVE (fsetxattr);
-
-        /* This must be called after resolving real functions
-         * above so that the socket based IO calls in libglusterfsclient
-         * can fall back to a non-NULL real_XXX function pointer.
-         * Calling booster_init before resolving the names above
-         * results in seg-faults because the function symbols above are NULL.
-         */
-	booster_init ();
-}
-
diff --git a/booster/src/booster_fstab.c b/booster/src/booster_fstab.c
deleted file mode 100644
index 202249cadf3..00000000000
--- a/booster/src/booster_fstab.c
+++ /dev/null
@@ -1,452 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995-2000, 2001, 2002, 2003, 2006
-   Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#include <alloca.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include "booster_fstab.h"
-#include <stdlib.h>
-#include <libglusterfsclient.h>
-#include <errno.h>
-
-/* The default timeout for inode and stat cache. */
-#define BOOSTER_DEFAULT_ATTR_TIMEO      5 /* In Secs */
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-glusterfs_fstab_t *
-glusterfs_fstab_init (const char *file, const char *mode)
-{
-        glusterfs_fstab_t *handle = NULL;
-        handle = calloc (1, sizeof (glusterfs_fstab_t));
-        if (!handle) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Memory allocation"
-                        " failed");
-                goto out;
-        }
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "FSTAB file: %s", file);
-        FILE *result = fopen (file,mode);
-        if (result != NULL) {
-                handle->fp = result;
-        } else {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "FSTAB file open failed:"
-                        " %s", strerror (errno));
-                free (handle);
-                handle = NULL;
-        }
-
-out:
-
-        return handle;
-}
-
-int
-glusterfs_fstab_close (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return -1;
-
-        if (h->fp)
-                fclose (h->fp);
-
-        return 0;
-}
-
-/* Since the values in a line are separated by spaces, a name cannot
-   contain a space.  Therefore some programs encode spaces in names
-   by the strings "\040".  We undo the encoding when reading an entry.
-   The decoding happens in place.  */
-static char *
-decode_name (char *buf)
-{
-        char *rp = buf;
-        char *wp = buf;
-
-        do
-                if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '4'
-                                && rp[3] == '0')
-                {
-                        /* \040 is a SPACE.  */
-                        *wp++ = ' ';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '1')
-                {
-                        /* \011 is a TAB.  */
-                        *wp++ = '\t';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '2')
-                {
-                        /* \012 is a NEWLINE.  */
-                        *wp++ = '\n';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '\\')
-                {
-                        /* We have to escape \\ to be able to represent all
-                         * characters.  */
-                        *wp++ = '\\';
-                        rp += 1;
-                }
-                else if (rp[0] == '\\' && rp[1] == '1' && rp[2] == '3'
-                                && rp[3] == '4')
-                {
-                        /* \134 is also \\.  */
-                        *wp++ = '\\';
-                        rp += 3;
-                }
-                else
-                        *wp++ = *rp;
-        while (*rp++ != '\0');
-
-        return buf;
-}
-
-
-/* Read one mount table entry from STREAM.  Returns a pointer to storage
-   reused on the next call, or null for EOF or error (use feof/ferror to
-   check).  */
-struct glusterfs_mntent *
-__glusterfs_fstab_getent (FILE *stream, struct glusterfs_mntent *mp,
-                          char *buffer, int bufsiz)
-{
-        char *cp;
-        char *head;
-
-        do
-        {
-                char *end_ptr;
-
-                if (fgets (buffer, bufsiz, stream) == NULL)
-                {
-                        return NULL;
-                }
-
-                end_ptr = strchr (buffer, '\n');
-                if (end_ptr != NULL)	/* chop newline */
-                        *end_ptr = '\0';
-                else
-                {
-                        /* Not the whole line was read.  Do it now but forget
-                         * it.  */
-                        char tmp[1024];
-                        while (fgets (tmp, sizeof tmp, stream) != NULL)
-                                if (strchr (tmp, '\n') != NULL)
-                                        break;
-                }
-
-                head = buffer + strspn (buffer, " \t");
-                /* skip empty lines and comment lines:  */
-        }
-        while (head[0] == '\0' || head[0] == '#');
-
-        cp = strsep (&head, " \t");
-        mp->mnt_fsname = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_dir = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_type = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_opts = cp != NULL ? decode_name (cp) : (char *) "";
-        switch (head ? sscanf (head, " %d %d ", &mp->mnt_freq,
-                               &mp->mnt_passno) : 0)
-        {
-                case 0:
-                        mp->mnt_freq = 0;
-                case 1:
-                        mp->mnt_passno = 0;
-                case 2:
-                        break;
-        }
-
-        return mp;
-}
-
-struct glusterfs_mntent *
-glusterfs_fstab_getent (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return NULL;
-
-        if (!h->fp)
-                return NULL;
-
-        return __glusterfs_fstab_getent (h->fp, &h->tmpent, h->buf,
-                                         GF_MNTENT_BUFSIZE);
-}
-
-/* We have to use an encoding for names if they contain spaces or tabs.
-   To be able to represent all characters we also have to escape the
-   backslash itself.  This "function" must be a macro since we use
-   `alloca'.  */
-#define encode_name(name)                                               \
-        do {                                                            \
-                const char *rp = name;		                        \
-                                                                        \
-                while (*rp != '\0')     	                        \
-                        if (*rp == ' ' || *rp == '\t' || *rp == '\\')   \
-                                break;                                  \
-                        else	                                        \
-                                ++rp;                                   \
-                                                                        \
-                if (*rp != '\0')                                        \
-                {                                               \
-                /* In the worst case the length of the string   \
-                 * can increase to four times the current       \
-                 * length.  */				        \
-                        char *wp;				\
-                                                                \
-                        rp = name;				\
-                        name = wp = (char *) alloca (strlen (name) * 4 + 1);                                                                 \
-                                                                \
-                        do {                            \
-                                if (*rp == ' ')		\
-                                {       		\
-                                        *wp++ = '\\';   \
-                                        *wp++ = '0';	\
-                                        *wp++ = '4';	\
-                                        *wp++ = '0';    \
-                                }			\
-                                else if (*rp == '\t')	\
-                                {			\
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '1';	\
-                                }			\
-                                else if (*rp == '\n')	\
-                                {	                \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '2';	\
-                                }	                \
-                                else if (*rp == '\\')	\
-                                {                       \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '\\';	\
-                                }                       \
-                                else	                \
-                                        *wp++ = *rp;	\
-                        } while (*rp++ != '\0');	\
-                }                                       \
-        } while (0)                                     \
-
-
-int
-glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt)
-{
-        struct glusterfs_mntent mntcopy = *mnt;
-        if (!h)
-                return -1;
-
-        if (!h->fp)
-                return -1;
-
-        if (fseek (h->fp, 0, SEEK_END))
-                return -1;
-
-        /* Encode spaces and tabs in the names.  */
-        encode_name (mntcopy.mnt_fsname);
-        encode_name (mntcopy.mnt_dir);
-        encode_name (mntcopy.mnt_type);
-        encode_name (mntcopy.mnt_opts);
-
-        return (fprintf (h->fp, "%s %s %s %s %d %d\n",
-                                mntcopy.mnt_fsname,
-                                mntcopy.mnt_dir,
-                                mntcopy.mnt_type,
-                                mntcopy.mnt_opts,
-                                mntcopy.mnt_freq,
-                                mntcopy.mnt_passno)
-                        < 0 ? 1 : 0);
-}
-
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-char *
-glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt, const char *opt)
-{
-        const size_t optlen = strlen (opt);
-        char *rest = mnt->mnt_opts, *p;
-
-        while ((p = strstr (rest, opt)) != NULL)
-        {
-                if ((p == rest || p[-1] == ',')
-                                && (p[optlen] == '\0' || p[optlen] == '=' || p[optlen] == ','))
-                        return p;
-
-                rest = strchr (p, ',');
-                if (rest == NULL)
-                        break;
-                ++rest;
-        }
-
-        return NULL;
-}
-
-void
-clean_init_params (glusterfs_init_params_t *ipars)
-{
-        if (!ipars)
-                return;
-
-        if (ipars->volume_name)
-                free (ipars->volume_name);
-
-        if (ipars->specfile)
-                free (ipars->specfile);
-
-        if (ipars->logfile)
-                free (ipars->logfile);
-
-        if (ipars->loglevel)
-                free (ipars->loglevel);
-
-        return;
-}
-
-char *
-get_option_value (char *opt)
-{
-        char *val = NULL;
-        char *saveptr = NULL;
-        char *copy_opt = NULL;
-        char *retval = NULL;
-
-        copy_opt = strdup (opt);
-
-        /* Get the = before the value of the option. */
-        val = index (copy_opt, '=');
-        if (val) {
-                /* Move to start of option */
-                ++val;
-
-                /* Now, to create a '\0' delimited string out of the
-                 * options string, first get the position where the
-                 * next option starts, that would be the next ','.
-                 */
-                saveptr = index (val, ',');
-                if (saveptr)
-                        *saveptr = '\0';
-                retval = strdup (val);
-        }
-
-        free (copy_opt);
-
-        return retval;
-}
-
-void
-booster_mount (struct glusterfs_mntent *ent)
-{
-        char                    *opt = NULL;
-        glusterfs_init_params_t ipars;
-        time_t                  timeout = BOOSTER_DEFAULT_ATTR_TIMEO;
-        char                    *timeostr = NULL;
-        char                    *endptr = NULL;
-
-        if (!ent)
-                return;
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "Mount entry: volfile: %s,"
-                " VMP: %s, Type: %s, Options: %s", ent->mnt_fsname,
-                ent->mnt_dir, ent->mnt_type, ent->mnt_opts);
-        if ((strcmp (ent->mnt_type, "glusterfs") != 0)) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Type is not glusterfs");
-                return;
-        }
-
-        memset (&ipars, 0, sizeof (glusterfs_init_params_t));
-        if (ent->mnt_fsname)
-                ipars.specfile = strdup (ent->mnt_fsname);
-
-        opt = glusterfs_fstab_hasoption (ent, "subvolume");
-        if (opt)
-                ipars.volume_name = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-file");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "logfile");
-
-        if (opt)
-                ipars.logfile = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-level");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "loglevel");
-
-        if (opt)
-                ipars.loglevel = get_option_value (opt);
-
-        /* Attribute cache timeout */
-        opt = glusterfs_fstab_hasoption (ent, "attr_timeout");
-        if (opt) {
-                 timeostr = get_option_value (opt);
-                 if (timeostr)
-                         timeout = strtol (timeostr, &endptr, 10);
-        }
-
-        ipars.lookup_timeout = timeout;
-        ipars.stat_timeout = timeout;
-
-        if ((glusterfs_mount (ent->mnt_dir, &ipars)) == -1)
-                gf_log ("booster-fstab", GF_LOG_ERROR, "VMP mounting failed");
-
-        clean_init_params (&ipars);
-}
-
-int
-booster_configure (char *confpath)
-{
-        int                     ret = -1;
-        glusterfs_fstab_t       *handle = NULL;
-        struct glusterfs_mntent *ent = NULL;
-
-        if (!confpath)
-                goto out;
-
-        handle = glusterfs_fstab_init (confpath, "r");
-        if (!handle)
-                goto out;
-
-        while ((ent = glusterfs_fstab_getent (handle)) != NULL)
-                booster_mount (ent);
-
-        glusterfs_fstab_close (handle);
-        ret = 0;
-out:
-        return ret;
-}
-
-
diff --git a/booster/src/booster_fstab.h b/booster/src/booster_fstab.h
deleted file mode 100644
index 9bab04c5aa0..00000000000
--- a/booster/src/booster_fstab.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995, 1996, 1997, 1998, 1999 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#ifndef	GLUSTERFS_FSTAB_MNTENT_H
-#define	GLUSTERFS_FSTAB_MNTENT_H	1
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
- 
-#include "compat.h"
-
-/* General filesystem types.  */
-#define GF_MNTTYPE_IGNORE	"ignore"	/* Ignore this entry.  */
-#define GF_MNTTYPE_NFS	"nfs"		/* Network file system.  */
-#define GF_MNTTYPE_SWAP	"swap"		/* Swap device.  */
-
-
-/* Generic mount options.  */
-#define GF_MNTOPT_DEFAULTS	"defaults"	/* Use all default options.  */
-#define GF_MNTOPT_RO	        "ro"		/* Read only.  */
-#define GF_MNTOPT_RW	        "rw"		/* Read/write.  */
-#define GF_MNTOPT_SUID	        "suid"		/* Set uid allowed.  */
-#define GF_MNTOPT_NOSUID	"nosuid"	/* No set uid allowed.  */
-#define GF_MNTOPT_NOAUTO	"noauto"	/* Do not auto mount.  */
-
-
-/* Structure describing a mount table entry.  */
-struct glusterfs_mntent
-{
-        char *mnt_fsname;		/* Device or server for filesystem.  */
-        char *mnt_dir;		/* Directory mounted on.  */
-        char *mnt_type;		/* Type of filesystem: ufs, nfs, etc.  */
-        char *mnt_opts;		/* Comma-separated options for fs.  */
-        int mnt_freq;		/* Dump frequency (in days).  */
-        int mnt_passno;		/* Pass number for `fsck'.  */
-};
-
-#define GF_MNTENT_BUFSIZE       1024
-typedef struct glusterfs_fstab_handle {
-        FILE *fp;
-        char buf[GF_MNTENT_BUFSIZE];
-        struct glusterfs_mntent tmpent;
-}glusterfs_fstab_t;
-
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-extern glusterfs_fstab_t *glusterfs_fstab_init (const char *file,
-                const char *mode);
-
-extern struct glusterfs_mntent *glusterfs_fstab_getent (glusterfs_fstab_t *h);
-
-/* Write the mount table entry described by MNT to STREAM.
-   Return zero on success, nonzero on failure.  */
-extern int glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt);
-
-/* Close a stream opened with `glusterfs_fstab_init'.  */
-extern int glusterfs_fstab_close (glusterfs_fstab_t *h);
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-extern char *glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt,
-                const char *opt);
-
-#endif
diff --git a/booster/src/booster_stat.c b/booster/src/booster_stat.c
deleted file mode 100644
index 42fd45f5ac3..00000000000
--- a/booster/src/booster_stat.c
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-
-extern int
-booster_stat (const char *path, void *buf);
-
-extern int
-booster_stat64 (const char *path, void *buf);
-
-extern int
-booster_xstat (int ver, const char *path, void *buf);
-
-extern int
-booster_xstat64 (int ver, const char *path, void *buf);
-
-extern int
-booster_fxstat (int ver, int fd, void *buf);
-extern int
-booster_fxstat64 (int ver, int fd, void *buf);
-extern int
-booster_fstat (int fd, void *buf);
-extern int
-booster_fstat64 (int fd, void *buf);
-
-extern int
-booster_lstat (const char *path, void *buf);
-extern int
-booster_lstat64 (const char *path, void *buf);
-extern int
-booster_lxstat (int ver, const char *path, void *buf);
-extern int
-booster_lxstat64 (int ver, const char *path, void *buf);
-
-
-extern int
-booster_statfs (const char *path, void *buf);
-extern int
-booster_statfs64 (const char *path, void *buf);
-
-extern int
-booster_statvfs (const char *path, void *buf);
-
-extern int
-booster_statvfs64 (const char *path, void *buf);
-
-extern void *
-booster_readdir (void *dir);
-
-extern void
-booster_rewinddir (void *dir);
-
-extern void
-booster_seekdir (void *dir, off_t offset);
-
-extern off_t
-booster_telldir (void *dir);
-
-int
-stat (const char *path, void *buf)
-{
-        return booster_stat (path, buf);
-}
-
-int
-stat64 (const char *path, void *buf)
-{
-        return booster_stat64 (path, buf);
-}
-
-int
-__xstat (int ver, const char *path, void *buf)
-{
-        return booster_xstat (ver, path, buf);
-}
-
-int
-__xstat64 (int ver, const char *path, void *buf)
-{
-        return booster_xstat64 (ver, path, buf);
-}
-
-int
-__fxstat (int ver, int fd, void *buf)
-{
-        return booster_fxstat (ver, fd, buf);
-}
-
-int
-__fxstat64 (int ver, int fd, void *buf)
-{
-        return booster_fxstat64 (ver, fd, buf);
-}
-
-int
-fstat (int fd, void *buf)
-{
-        return booster_fstat (fd, buf);
-}
-
-int
-fstat64 (int fd, void *buf)
-{
-        return booster_fstat64 (fd, buf);
-}
-
-int
-lstat (const char *path, void *buf)
-{
-        return booster_lstat (path, buf);
-}
-
-int
-lstat64 (const char *path, void *buf)
-{
-        return booster_lstat64 (path, buf);
-}
-
-int
-__lxstat (int ver, const char *path, void *buf)
-{
-        return booster_lxstat (ver, path, buf);
-}
-
-int
-__lxstat64 (int ver, const char *path, void *buf)
-{
-        return booster_lxstat64 (ver, path, buf);
-}
-
-int
-statfs (const char *pathname, void *buf)
-{
-        return booster_statfs (pathname, buf);
-}
-
-int
-statfs64 (const char *pathname, void *buf)
-{
-        return booster_statfs64 (pathname, buf);
-}
-
-int
-statvfs (const char *pathname, void *buf)
-{
-        return booster_statvfs (pathname, buf);
-}
-
-int
-statvfs64 (const char *pathname, void *buf)
-{
-        return booster_statvfs64 (pathname, buf);
-}
-
-void
-rewinddir (void *dir)
-{
-	return booster_rewinddir (dir);
-}
-
-void
-seekdir (void *dir, off_t offset)
-{
-	return booster_seekdir (dir, offset);
-}
-
-off_t
-telldir (void *dir)
-{
-	return booster_telldir (dir);
-}
diff --git a/build-aux/checkpatch.pl b/build-aux/checkpatch.pl
new file mode 100755
index 00000000000..17ae4e4d579
--- /dev/null
+++ b/build-aux/checkpatch.pl
@@ -0,0 +1,4326 @@
+#!/usr/bin/perl -w
+# (c) 2001, Dave Jones. (the file handling bit)
+# (c) 2005, Joel Schopp <jschopp@austin.ibm.com> (the ugly bit)
+# (c) 2007,2008, Andy Whitcroft <apw@uk.ibm.com> (new conditions, test suite)
+# (c) 2008-2010 Andy Whitcroft <apw@canonical.com>
+# (c) 2014 Gluster Community <gluster-devel@gluster.org>
+# Licensed under the terms of the GNU GPL License version 2
+
+use strict;
+use POSIX;
+
+my $P = $0;
+$P =~ s@.*/@@g;
+
+my $V = '0.32.1';
+
+use Getopt::Long qw(:config no_auto_abbrev);
+
+my $quiet = 0;
+my $tree = 1;
+my $chk_signoff = 1;
+my $chk_patch = 1;
+my $tst_only;
+my $emacs = 0;
+my $terse = 0;
+my $file = 0;
+my $check = 0;
+my $check_orig = 0;
+my $summary = 1;
+my $mailback = 0;
+my $summary_file = 0;
+my $show_types = 0;
+my $fix = 0;
+my $fix_inplace = 0;
+my $root;
+my %debug;
+my %camelcase = ();
+my %use_type = ();
+my @use = ();
+my %ignore_type = ();
+my @ignore = ();
+my $help = 0;
+my $configuration_file = ".checkpatch.conf";
+my $max_line_length = 80;
+my $ignore_perl_version = 0;
+my $minimum_perl_version = 5.10.0;
+my $gerrit_url = $ENV{GERRIT_URL};
+
+sub help {
+    my ($exitcode) = @_;
+
+    print << "EOM";
+Usage: $P [OPTION]... [FILE]...
+Version: $V
+
+Options:
+  -q, --quiet                quiet
+  --patch                    treat FILE as patchfile (default)
+  --emacs                    emacs compile window format
+  --gerrit-url=STRING        URL the patch was reviewed at
+  --terse                    one line per report
+  -f, --file                 treat FILE as regular source file
+  --subjective, --strict     enable more subjective tests
+  --types TYPE(,TYPE2...)    show only these comma separated message types
+  --ignore TYPE(,TYPE2...)   ignore various comma separated message types
+  --max-line-length=n        set the maximum line length, if exceeded, warn
+  --show-types               show the message "types" in the output
+  --root=PATH                PATH to the glusterfs tree root
+  --no-summary               suppress the per-file summary
+  --mailback                 only produce a report in case of warnings/errors
+  --summary-file             include the filename in summary
+  --debug KEY=[0|1]          turn on/off debugging of KEY, where KEY is one of
+                             'values', 'possible', 'type', and 'attr' (default
+                             is all off)
+  --test-only=WORD           report only warnings/errors containing WORD literally
+  --fix                      EXPERIMENTAL - may create horrible results
+                             If correctable single-line errors exist, create
+                             "<inputfile>.EXPERIMENTAL-checkpatch-fixes"
+                             with potential errors corrected to the preferred
+                             checkpatch style
+  --fix-inplace              EXPERIMENTAL - may create horrible results
+                             Is the same as --fix, but overwrites the input
+                             file.  It's your fault if there's no backup or git
+  --ignore-perl-version      override checking of perl version.  expect
+                             runtime errors.
+  -h, --help, --version      display this help and exit
+
+When FILE is - read standard input.
+EOM
+
+exit($exitcode);
+}
+
+my $conf = which_conf($configuration_file);
+if (-f $conf) {
+    my @conf_args;
+    open(my $conffile, '<', "$conf")
+        or warn "$P: Can't find a readable $configuration_file file $!\n";
+
+    while (<$conffile>) {
+        my $line = $_;
+
+        $line =~ s/\s*\n?$//g;
+        $line =~ s/^\s*//g;
+        $line =~ s/\s+/ /g;
+
+        next if ($line =~ m/^\s*#/);
+        next if ($line =~ m/^\s*$/);
+
+        my @words = split(" ", $line);
+        foreach my $word (@words) {
+            last if ($word =~ m/^#/);
+            push (@conf_args, $word);
+        }
+    }
+    close($conffile);
+    unshift(@ARGV, @conf_args) if @conf_args;
+}
+
+GetOptions(
+    'q|quiet+'  => \$quiet,
+    'patch!'    => \$chk_patch,
+    'emacs!'    => \$emacs,
+    'gerrit-url=s' => \$gerrit_url,
+    'terse!'    => \$terse,
+    'f|file!'   => \$file,
+    'subjective!'       => \$check,
+    'strict!'   => \$check,
+    'ignore=s'  => \@ignore,
+    'types=s'   => \@use,
+    'show-types!'       => \$show_types,
+    'max-line-length=i' => \$max_line_length,
+    'root=s'    => \$root,
+    'summary!'  => \$summary,
+    'mailback!' => \$mailback,
+    'summary-file!'     => \$summary_file,
+    'fix!'              => \$fix,
+    'fix-inplace!'      => \$fix_inplace,
+    'ignore-perl-version!' => \$ignore_perl_version,
+    'debug=s'   => \%debug,
+    'test-only=s'       => \$tst_only,
+    'h|help'    => \$help,
+    'version'   => \$help
+) or help(1);
+
+help(0) if ($help);
+
+$fix = 1 if ($fix_inplace);
+$check_orig = $check;
+
+my $exit = 0;
+
+if ($^V && $^V lt $minimum_perl_version) {
+    printf "$P: requires at least perl version %vd\n", $minimum_perl_version;
+    if (!$ignore_perl_version) {
+        exit(1);
+    }
+}
+
+if ($#ARGV < 0) {
+    print "$P: no input files\n";
+    exit(1);
+}
+
+sub hash_save_array_words {
+    my ($hashRef, $arrayRef) = @_;
+
+    my @array = split(/,/, join(',', @$arrayRef));
+    foreach my $word (@array) {
+        $word =~ s/\s*\n?$//g;
+        $word =~ s/^\s*//g;
+        $word =~ s/\s+/ /g;
+        $word =~ tr/[a-z]/[A-Z]/;
+
+        next if ($word =~ m/^\s*#/);
+        next if ($word =~ m/^\s*$/);
+
+        $hashRef->{$word}++;
+    }
+}
+
+sub hash_show_words {
+    my ($hashRef, $prefix) = @_;
+
+    if ($quiet == 0 && keys %$hashRef) {
+        print "NOTE: $prefix message types:";
+        foreach my $word (sort keys %$hashRef) {
+            print " $word";
+        }
+        print "\n\n";
+    }
+}
+
+hash_save_array_words(\%ignore_type, \@ignore);
+hash_save_array_words(\%use_type, \@use);
+
+my $dbg_values = 0;
+my $dbg_possible = 0;
+my $dbg_type = 0;
+my $dbg_attr = 0;
+for my $key (keys %debug) {
+    ## no critic
+    eval "\${dbg_$key} = '$debug{$key}';";
+    die "$@" if ($@);
+}
+
+my $rpt_cleaners = 0;
+
+if ($terse) {
+    $emacs = 1;
+    $quiet++;
+}
+
+if ($tree) {
+    if (defined $root) {
+        if (!top_of_glusterfs_tree($root)) {
+            die "$P: $root: --root does not point at a valid tree\n";
+        }
+    } else {
+        if (top_of_glusterfs_tree('.')) {
+            $root = '.';
+        } elsif ($0 =~ m@(.*)/extras/[^/]*$@ &&
+                 top_of_glusterfs_tree($1)) {
+            $root = $1;
+        }
+    }
+
+    if (!defined $root) {
+        print "Must be run from the top-level dir. of a GlusterFS tree\n";
+        exit(2);
+    }
+}
+
+my $emitted_corrupt = 0;
+
+our $Ident      = qr{
+                        [A-Za-z_][A-Za-z\d_]*
+                        (?:\s*\#\#\s*[A-Za-z_][A-Za-z\d_]*)*
+                }x;
+our $Storage    = qr{extern|static|asmlinkage};
+our $Sparse     = qr{
+                        __user|
+                        __kernel|
+                        __force|
+                        __iomem|
+                        __must_check|
+                        __init_refok|
+                        __kprobes|
+                        __ref|
+                        __rcu
+                }x;
+our $InitAttributePrefix = qr{__(?:mem|cpu|dev|net_|)};
+our $InitAttributeData = qr{$InitAttributePrefix(?:initdata\b)};
+our $InitAttributeConst = qr{$InitAttributePrefix(?:initconst\b)};
+our $InitAttributeInit = qr{$InitAttributePrefix(?:init\b)};
+our $InitAttribute = qr{$InitAttributeData|$InitAttributeConst|$InitAttributeInit};
+
+# Notes to $Attribute:
+# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
+our $Attribute  = qr{
+                        const|
+                        __percpu|
+                        __nocast|
+                        __safe|
+                        __bitwise__|
+                        __packed__|
+                        __packed2__|
+                        __naked|
+                        __maybe_unused|
+                        __always_unused|
+                        __noreturn|
+                        __used|
+                        __cold|
+                        __noclone|
+                        __deprecated|
+                        __read_mostly|
+                        __kprobes|
+                        $InitAttribute|
+                        ____cacheline_aligned|
+                        ____cacheline_aligned_in_smp|
+                        ____cacheline_internodealigned_in_smp|
+                        __weak
+                  }x;
+our $Modifier;
+our $Inline     = qr{inline|__always_inline|noinline|__inline|__inline__};
+our $Member     = qr{->$Ident|\.$Ident|\[[^]]*\]};
+our $Lval       = qr{$Ident(?:$Member)*};
+
+our $Int_type   = qr{(?i)llu|ull|ll|lu|ul|l|u};
+our $Binary     = qr{(?i)0b[01]+$Int_type?};
+our $Hex        = qr{(?i)0x[0-9a-f]+$Int_type?};
+our $Int        = qr{[0-9]+$Int_type?};
+our $Octal      = qr{0[0-7]+$Int_type?};
+our $Float_hex  = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?};
+our $Float_dec  = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?};
+our $Float_int  = qr{(?i)[0-9]+e-?[0-9]+[fl]?};
+our $Float      = qr{$Float_hex|$Float_dec|$Float_int};
+our $Constant   = qr{$Float|$Binary|$Octal|$Hex|$Int};
+our $Assignment = qr{\*\=|/=|%=|\+=|-=|<<=|>>=|&=|\^=|\|=|=};
+our $Compare    = qr{<=|>=|==|!=|<|(?<!-)>};
+our $Arithmetic = qr{\+|-|\*|\/|%};
+our $Operators  = qr{
+                        <=|>=|==|!=|
+                        =>|->|<<|>>|<|>|!|~|
+                        &&|\|\||,|\^|\+\+|--|&|\||$Arithmetic
+                  }x;
+
+our $c90_Keywords = qr{do|for|while|if|else|return|goto|continue|switch|default|case|break}x;
+
+our $NonptrType;
+our $NonptrTypeWithAttr;
+our $Type;
+our $Declare;
+
+our $NON_ASCII_UTF8     = qr{
+        [\xC2-\xDF][\x80-\xBF]               # non-overlong 2-byte
+        |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+        | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}  # straight 3-byte
+        |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+        |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+        | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+        |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+}x;
+
+our $UTF8       = qr{
+        [\x09\x0A\x0D\x20-\x7E]              # ASCII
+        | $NON_ASCII_UTF8
+}x;
+
+our $typeTypedefs = qr{(?x:
+        (?:__)?(?:u|s|be|le)(?:8|16|32|64)|
+        atomic_t
+)};
+
+our $logFunctions = qr{(?x:
+        printk(?:_ratelimited|_once|)|
+        (?:[a-z0-9]+_){1,2}(?:printk|emerg|alert|crit|err|warning|warn|notice|info|debug|dbg|vdbg|devel|cont|WARN)(?:_ratelimited|_once|)|
+        WARN(?:_RATELIMIT|_ONCE|)|
+        panic|
+        MODULE_[A-Z_]+|
+        seq_vprintf|seq_printf|seq_puts
+)};
+
+our $signature_tags = qr{(?xi:
+        Signed-off-by:|
+        Acked-by:|
+        Tested-by:|
+        Reviewed-by:|
+        Reviewed-on:|
+        Reported-by:|
+        Original-author:|
+        Original-Author:|
+        Original-Authors:|
+        Suggested-by:|
+        To:|
+        Cc:
+)};
+
+our $url_tags = qr{http:|https:};
+
+our @typeList = (
+        qr{void},
+        qr{(?:unsigned\s+)?char},
+        qr{(?:unsigned\s+)?short},
+        qr{(?:unsigned\s+)?int},
+        qr{(?:unsigned\s+)?long},
+        qr{(?:unsigned\s+)?long\s+int},
+        qr{(?:unsigned\s+)?long\s+long},
+        qr{(?:unsigned\s+)?long\s+long\s+int},
+        qr{unsigned},
+        qr{float},
+        qr{double},
+        qr{bool},
+        qr{struct\s+$Ident},
+        qr{union\s+$Ident},
+        qr{enum\s+$Ident},
+        qr{${Ident}_t},
+        qr{${Ident}_handler},
+        qr{${Ident}_handler_fn},
+);
+our @typeListWithAttr = (
+        @typeList,
+        qr{struct\s+$InitAttribute\s+$Ident},
+        qr{union\s+$InitAttribute\s+$Ident},
+);
+
+our @modifierList = (
+        qr{fastcall},
+);
+
+our @mode_permission_funcs = (
+        ["module_param", 3],
+        ["module_param_(?:array|named|string)", 4],
+        ["module_param_array_named", 5],
+        ["debugfs_create_(?:file|u8|u16|u32|u64|x8|x16|x32|x64|size_t|atomic_t|bool|blob|regset32|u32_array)", 2],
+        ["proc_create(?:_data|)", 2],
+        ["(?:CLASS|DEVICE|SENSOR)_ATTR", 2],
+);
+
+#Create a search pattern for all these functions to speed up a loop below
+our $mode_perms_search = "";
+foreach my $entry (@mode_permission_funcs) {
+        $mode_perms_search .= '|' if ($mode_perms_search ne "");
+        $mode_perms_search .= $entry->[0];
+}
+
+our $declaration_macros = qr{(?x:
+        (?:$Storage\s+)?(?:DECLARE|DEFINE)_[A-Z]+\s*\(|
+        (?:$Storage\s+)?LIST_HEAD\s*\(
+)};
+
+our $allowed_asm_includes = qr{(?x:
+        irq|
+        memory
+)};
+# memory.h: ARM has a custom one
+
+sub build_types {
+    my $mods = "(?x:\n" . join("|\n  ", @modifierList) . "\n)";
+    my $all = "(?x:\n" . join("|\n  ", @typeList) . "\n)";
+    my $allWithAttr = "(?x:\n" . join("|\n  ", @typeListWithAttr) . "\n)";
+    $Modifier   = qr{(?:$Attribute|$Sparse|$mods)};
+    $NonptrType = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${all}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $NonptrTypeWithAttr     = qr{
+                        (?:$Modifier\s+|const\s+)*
+                        (?:
+                                (?:typeof|__typeof__)\s*\([^\)]*\)|
+                                (?:$typeTypedefs\b)|
+                                (?:${allWithAttr}\b)
+                        )
+                        (?:\s+$Modifier|\s+const)*
+                  }x;
+        $Type   = qr{
+                        $NonptrType
+                        (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*|\[\])+|(?:\s*\[\s*\])+)?
+                        (?:\s+$Inline|\s+$Modifier)*
+                  }x;
+        $Declare        = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type};
+}
+build_types();
+
+our $Typecast   = qr{\s*(\(\s*$NonptrType\s*\)){0,1}\s*};
+
+# Using $balanced_parens, $LvalOrFunc, or $FuncArg
+# requires at least perl version v5.10.0
+# Any use must be runtime checked with $^V
+
+our $balanced_parens = qr/(\((?:[^\(\)]++|(?-1))*\))/;
+our $LvalOrFunc = qr{((?:[\&\*]\s*)?$Lval)\s*($balanced_parens{0,1})\s*};
+our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant)};
+
+sub deparenthesize {
+    my ($string) = @_;
+    return "" if (!defined($string));
+
+    while ($string =~ /^\s*\(.*\)\s*$/) {
+        $string =~ s@^\s*\(\s*@@;
+        $string =~ s@\s*\)\s*$@@;
+    }
+
+    $string =~ s@\s+@ @g;
+
+    return $string;
+}
+
+sub seed_camelcase_file {
+    my ($file) = @_;
+
+    return if (!(-f $file));
+
+    local $/;
+
+    open(my $include_file, '<', "$file")
+        or warn "$P: Can't read '$file' $!\n";
+    my $text = <$include_file>;
+    close($include_file);
+
+    my @lines = split('\n', $text);
+
+    foreach my $line (@lines) {
+        next if ($line !~ /(?:[A-Z][a-z]|[a-z][A-Z])/);
+        if ($line =~ /^[ \t]*(?:#[ \t]*define|typedef\s+$Type)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*$Declare\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[\(\[,;]/) {
+            $camelcase{$1} = 1;
+        } elsif ($line =~ /^\s*(?:union|struct|enum)\s+(\w*(?:[A-Z][a-z]|[a-z][A-Z])\w*)\s*[;\{]/) {
+            $camelcase{$1} = 1;
+        }
+    }
+}
+
+my $camelcase_seeded = 0;
+sub seed_camelcase_includes {
+    return if ($camelcase_seeded);
+
+    my $files;
+    my $camelcase_cache = "";
+    my @include_files = ();
+
+    $camelcase_seeded = 1;
+
+    if (-e ".git") {
+        my $git_last_include_commit = `git log --no-merges --pretty=format:"%h%n" -1 -- include`;
+        chomp $git_last_include_commit;
+        $camelcase_cache = ".checkpatch-camelcase.git.$git_last_include_commit";
+    } else {
+        my $last_mod_date = 0;
+        $files = `find $root/include -name "*.h"`;
+        @include_files = split('\n', $files);
+        foreach my $file (@include_files) {
+            my $date = POSIX::strftime("%Y%m%d%H%M",
+                                       localtime((stat $file)[9]));
+            $last_mod_date = $date if ($last_mod_date < $date);
+        }
+        $camelcase_cache = ".checkpatch-camelcase.date.$last_mod_date";
+    }
+
+    if ($camelcase_cache ne "" && -f $camelcase_cache) {
+        open(my $camelcase_file, '<', "$camelcase_cache")
+            or warn "$P: Can't read '$camelcase_cache' $!\n";
+        while (<$camelcase_file>) {
+            chomp;
+            $camelcase{$_} = 1;
+        }
+        close($camelcase_file);
+        return;
+    }
+
+    if (-e ".git") {
+        $files = `git ls-files "include/*.h"`;
+        @include_files = split('\n', $files);
+    }
+
+    foreach my $file (@include_files) {
+        seed_camelcase_file($file);
+    }
+
+    if ($camelcase_cache ne "") {
+        unlink glob ".checkpatch-camelcase.*";
+        open(my $camelcase_file, '>', "$camelcase_cache")
+            or warn "$P: Can't write '$camelcase_cache' $!\n";
+        foreach (sort { lc($a) cmp lc($b) } keys(%camelcase)) {
+            print $camelcase_file ("$_\n");
+        }
+        close($camelcase_file);
+    }
+}
+
+$chk_signoff = 0 if ($file);
+
+my @rawlines = ();
+my @lines = ();
+my @fixed = ();
+my $vname;
+for my $filename (@ARGV) {
+    my $FILE;
+    if ($file) {
+        open($FILE, '-|', "diff -u /dev/null $filename") ||
+            die "$P: $filename: diff failed - $!\n";
+    } elsif ($filename eq '-') {
+        open($FILE, '<&STDIN');
+    } else {
+        open($FILE, '<', "$filename") ||
+            die "$P: $filename: open failed - $!\n";
+    }
+    if ($filename eq '-') {
+        $vname = 'Your patch';
+    } else {
+        $vname = $filename;
+    }
+    while (<$FILE>) {
+        chomp;
+        push(@rawlines, $_);
+    }
+    close($FILE);
+    if (!process($filename)) {
+        $exit = 1;
+    }
+    @rawlines = ();
+    @lines = ();
+    @fixed = ();
+}
+
+exit($exit);
+
+sub top_of_glusterfs_tree {
+    my ($root) = @_;
+
+    # Add here if the tree changes
+    my @tree_check = (
+        "api",
+        "AUTHORS",
+        "autogen.sh",
+        "build-aux",
+        "ChangeLog",
+        "cli",
+        "configure.ac",
+        "contrib",
+        "CONTRIBUTING",
+        "COPYING-GPLV2",
+        "COPYING-LGPLV3",
+        "doc",
+        "extras",
+        "geo-replication",
+        "glusterfs-api.pc.in",
+        "glusterfsd",
+        "glusterfs.spec.in",
+        "heal",
+        "INSTALL",
+        "libgfchangelog.pc.in",
+        "libglusterfs",
+        "MAINTAINERS",
+        "Makefile.am",
+        "NEWS",
+        "README.md",
+        "rfc.sh",
+        "rpc",
+        "run-tests.sh",
+        "tests",
+        "THANKS",
+        "xlators",
+        );
+
+    foreach my $check (@tree_check) {
+        if (! -e $root . '/' . $check) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+sub parse_email {
+    my ($formatted_email) = @_;
+
+    my $name = "";
+    my $address = "";
+    my $comment = "";
+
+    if ($formatted_email =~ /^(.*)<(\S+\@\S+)>(.*)$/) {
+        $name = $1;
+        $address = $2;
+        $comment = $3 if defined $3;
+    } elsif ($formatted_email =~ /^\s*<(\S+\@\S+)>(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+    } elsif ($formatted_email =~ /(\S+\@\S+)(.*)$/) {
+        $address = $1;
+        $comment = $2 if defined $2;
+        $formatted_email =~ s/$address.*$//;
+        $name = $formatted_email;
+        $name = trim($name);
+        $name =~ s/^\"|\"$//g;
+        # If there's a name left after stripping spaces and
+        # leading quotes, and the address doesn't have both
+        # leading and trailing angle brackets, the address
+        # is invalid. ie:
+        #   "joe smith joe@smith.com" bad
+        #   "joe smith <joe@smith.com" bad
+        if ($name ne "" && $address !~ /^<[^>]+>$/) {
+            $name = "";
+            $address = "";
+            $comment = "";
+        }
+    }
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+    $address =~ s/^\<|\>$//g;
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    return ($name, $address, $comment);
+}
+
+sub format_email {
+    my ($name, $address) = @_;
+
+    my $formatted_email;
+
+    $name = trim($name);
+    $name =~ s/^\"|\"$//g;
+    $address = trim($address);
+
+    if ($name =~ /[^\w \-]/i) { ##has "must quote" chars
+        $name =~ s/(?<!\\)"/\\"/g; ##escape quotes
+        $name = "\"$name\"";
+    }
+
+    if ("$name" eq "") {
+        $formatted_email = "$address";
+    } else {
+        $formatted_email = "$name <$address>";
+    }
+
+    return $formatted_email;
+}
+
+sub which_conf {
+    my ($conf) = @_;
+
+    foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) {
+        if (-e "$path/$conf") {
+            return "$path/$conf";
+        }
+    }
+
+    return "";
+}
+
+sub expand_tabs {
+    my ($str) = @_;
+
+    my $res = '';
+    my $n = 0;
+    for my $c (split(//, $str)) {
+        if ($c eq "\t") {
+            $res .= ' ';
+            $n++;
+            for (; ($n % 8) != 0; $n++) {
+                $res .= ' ';
+            }
+            next;
+        }
+        $res .= $c;
+        $n++;
+    }
+    return $res;
+}
+sub copy_spacing {
+    (my $res = shift) =~ tr/\t/ /c;
+    return $res;
+}
+
+sub line_stats {
+    my ($line) = @_;
+
+    # Drop the diff line leader and expand tabs
+    $line =~ s/^.//;
+    $line = expand_tabs($line);
+
+    # Pick the indent from the front of the line.
+    my ($white) = ($line =~ /^(\s*)/);
+
+    return (length($line), length($white));
+}
+
+my $sanitise_quote = '';
+
+sub sanitise_line_reset {
+    my ($in_comment) = @_;
+
+    if ($in_comment) {
+        $sanitise_quote = '*/';
+    } else {
+        $sanitise_quote = '';
+    }
+}
+sub sanitise_line {
+    my ($line) = @_;
+
+    my $res = '';
+    my $l = '';
+
+    my $qlen = 0;
+    my $off = 0;
+    my $c;
+
+    # Always copy over the diff marker.
+    $res = substr($line, 0, 1);
+
+    for ($off = 1; $off < length($line); $off++) {
+        $c = substr($line, $off, 1);
+
+        # Comments we are wacking completly including the begin
+        # and end, all to $;.
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '/*') {
+            $sanitise_quote = '*/';
+
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '*/' && substr($line, $off, 2) eq '*/') {
+            $sanitise_quote = '';
+            substr($res, $off, 2, "$;$;");
+            $off++;
+            next;
+        }
+        if ($sanitise_quote eq '' && substr($line, $off, 2) eq '//') {
+            $sanitise_quote = '//';
+
+            substr($res, $off, 2, $sanitise_quote);
+            $off++;
+            next;
+        }
+
+        # A \ in a string means ignore the next character.
+        if (($sanitise_quote eq "'" || $sanitise_quote eq '"') &&
+            $c eq "\\") {
+            substr($res, $off, 2, 'XX');
+            $off++;
+            next;
+        }
+        # Regular quotes.
+        if ($c eq "'" || $c eq '"') {
+            if ($sanitise_quote eq '') {
+                $sanitise_quote = $c;
+
+                substr($res, $off, 1, $c);
+                next;
+            } elsif ($sanitise_quote eq $c) {
+                $sanitise_quote = '';
+            }
+        }
+
+        #print "c<$c> SQ<$sanitise_quote>\n";
+        if ($off != 0 && $sanitise_quote eq '*/' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote eq '//' && $c ne "\t") {
+            substr($res, $off, 1, $;);
+        } elsif ($off != 0 && $sanitise_quote && $c ne "\t") {
+            substr($res, $off, 1, 'X');
+        } else {
+            substr($res, $off, 1, $c);
+        }
+    }
+
+    if ($sanitise_quote eq '//') {
+        $sanitise_quote = '';
+    }
+
+    # The pathname on a #include may be surrounded by '<' and '>'.
+    if ($res =~ /^.\s*\#\s*include\s+\<(.*)\>/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@\<.*\>@<$clean>@;
+
+    # The whole of a #error is a string.
+    } elsif ($res =~ /^.\s*\#\s*(?:error|warning)\s+(.*)\b/) {
+        my $clean = 'X' x length($1);
+        $res =~ s@(\#\s*(?:error|warning)\s+).*@$1$clean@;
+    }
+
+    return $res;
+}
+
+sub get_quoted_string {
+    my ($line, $rawline) = @_;
+
+    return "" if ($line !~ m/(\"[X]+\")/g);
+    return substr($rawline, $-[0], $+[0] - $-[0]);
+}
+
+sub ctx_statement_block {
+    my ($linenr, $remain, $off) = @_;
+    my $line = $linenr - 1;
+    my $blk = '';
+    my $soff = $off;
+    my $coff = $off - 1;
+    my $coff_set = 0;
+
+    my $loff = 0;
+
+    my $type = '';
+    my $level = 0;
+    my @stack = ();
+    my $p;
+    my $c;
+    my $len = 0;
+
+    my $remainder;
+    while (1) {
+        @stack = (['', 0]) if ($#stack == -1);
+
+        #warn "CSB: blk<$blk> remain<$remain>\n";
+        # If we are about to drop off the end, pull in more
+        # context.
+        if ($off >= $len) {
+            for (; $remain > 0; $line++) {
+                last if (!defined $lines[$line]);
+                next if ($lines[$line] =~ /^-/);
+                $remain--;
+                $loff = $len;
+                $blk .= $lines[$line] . "\n";
+                $len = length($blk);
+                $line++;
+                last;
+            }
+            # Bail if there is no further context.
+            #warn "CSB: blk<$blk> off<$off> len<$len>\n";
+            if ($off >= $len) {
+                last;
+            }
+            if ($level == 0 && substr($blk, $off) =~ /^.\s*#\s*define/) {
+                $level++;
+                $type = '#';
+            }
+        }
+        $p = $c;
+        $c = substr($blk, $off, 1);
+        $remainder = substr($blk, $off);
+
+        #warn "CSB: c<$c> type<$type> level<$level> remainder<$remainder> coff_set<$coff_set>\n";
+
+        # Handle nested #if/#else.
+        if ($remainder =~ /^#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, [ $type, $level ]);
+        } elsif ($remainder =~ /^#\s*(?:else|elif)\b/) {
+            ($type, $level) = @{$stack[$#stack - 1]};
+        } elsif ($remainder =~ /^#\s*endif\b/) {
+            ($type, $level) = @{pop(@stack)};
+        }
+
+        # Statement ends at the ';' or a close '}' at the
+        # outermost level.
+        if ($level == 0 && $c eq ';') {
+            last;
+        }
+
+        # An else is really a conditional as long as its not else if
+        if ($level == 0 && $coff_set == 0 &&
+            (!defined($p) || $p =~ /(?:\s|\}|\+)/) &&
+            $remainder =~ /^(else)(?:\s|{)/ &&
+            $remainder !~ /^else\s+if\b/) {
+            $coff = $off + length($1) - 1;
+            $coff_set = 1;
+            #warn "CSB: mark coff<$coff> soff<$soff> 1<$1>\n";
+            #warn "[" . substr($blk, $soff, $coff - $soff + 1) . "]\n";
+        }
+
+        if (($type eq '' || $type eq '(') && $c eq '(') {
+            $level++;
+            $type = '(';
+        }
+        if ($type eq '(' && $c eq ')') {
+            $level--;
+            $type = ($level != 0)? '(' : '';
+
+            if ($level == 0 && $coff < $soff) {
+                $coff = $off;
+                $coff_set = 1;
+                #warn "CSB: mark coff<$coff>\n";
+            }
+        }
+        if (($type eq '' || $type eq '{') && $c eq '{') {
+            $level++;
+            $type = '{';
+        }
+        if ($type eq '{' && $c eq '}') {
+            $level--;
+            $type = ($level != 0)? '{' : '';
+
+            if ($level == 0) {
+                if (substr($blk, $off + 1, 1) eq ';') {
+                    $off++;
+                }
+                last;
+            }
+        }
+        # Preprocessor commands end at the newline unless escaped.
+        if ($type eq '#' && $c eq "\n" && $p ne "\\") {
+            $level--;
+            $type = '';
+            $off++;
+            last;
+        }
+        $off++;
+    }
+    # We are truly at the end, so shuffle to the next line.
+    if ($off == $len) {
+        $loff = $len + 1;
+        $line++;
+        $remain--;
+    }
+
+    my $statement = substr($blk, $soff, $off - $soff + 1);
+    my $condition = substr($blk, $soff, $coff - $soff + 1);
+
+    #warn "STATEMENT<$statement>\n";
+    #warn "CONDITION<$condition>\n";
+
+    #print "coff<$coff> soff<$off> loff<$loff>\n";
+
+    return ($statement, $condition,
+            $line, $remain + 1, $off - $loff + 1, $level);
+}
+
+sub statement_lines {
+    my ($stmt) = @_;
+
+    # Strip the diff line prefixes and rip blank lines at start and end.
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_rawlines {
+    my ($stmt) = @_;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+
+    return $#stmt_lines + 2;
+}
+
+sub statement_block_size {
+    my ($stmt) = @_;
+
+    $stmt =~ s/(^|\n)./$1/g;
+    $stmt =~ s/^\s*{//;
+    $stmt =~ s/}\s*$//;
+    $stmt =~ s/^\s*//;
+    $stmt =~ s/\s*$//;
+
+    my @stmt_lines = ($stmt =~ /\n/g);
+    my @stmt_statements = ($stmt =~ /;/g);
+
+    my $stmt_lines = $#stmt_lines + 2;
+    my $stmt_statements = $#stmt_statements + 1;
+
+    if ($stmt_lines > $stmt_statements) {
+        return $stmt_lines;
+    } else {
+        return $stmt_statements;
+    }
+}
+
+sub ctx_statement_full {
+    my ($linenr, $remain, $off) = @_;
+    my ($statement, $condition, $level);
+
+    my (@chunks);
+
+    # Grab the first conditional/block pair.
+    ($statement, $condition, $linenr, $remain, $off, $level) =
+        ctx_statement_block($linenr, $remain, $off);
+    #print "F: c<$condition> s<$statement> remain<$remain>\n";
+    push(@chunks, [ $condition, $statement ]);
+    if (!($remain > 0 && $condition =~ /^\s*(?:\n[+-])?\s*(?:if|else|do)\b/s)) {
+        return ($level, $linenr, @chunks);
+    }
+
+    # Pull in the following conditional/block pairs and see if they
+    # could continue the statement.
+    for (;;) {
+        ($statement, $condition, $linenr, $remain, $off, $level) =
+            ctx_statement_block($linenr, $remain, $off);
+        #print "C: c<$condition> s<$statement> remain<$remain>\n";
+        last if (!($remain > 0 && $condition =~ /^(?:\s*\n[+-])*\s*(?:else|do)\b/s));
+        #print "C: push\n";
+        push(@chunks, [ $condition, $statement ]);
+    }
+
+    return ($level, $linenr, @chunks);
+}
+
+sub ctx_block_get {
+    my ($linenr, $remain, $outer, $open, $close, $off) = @_;
+    my $line;
+    my $start = $linenr - 1;
+    my $blk = '';
+    my @o;
+    my @c;
+    my @res = ();
+
+    my $level = 0;
+    my @stack = ($level);
+    for ($line = $start; $remain > 0; $line++) {
+        next if ($rawlines[$line] =~ /^-/);
+        $remain--;
+
+        $blk .= $rawlines[$line];
+
+        # Handle nested #if/#else.
+        if ($lines[$line] =~ /^.\s*#\s*(?:ifndef|ifdef|if)\s/) {
+            push(@stack, $level);
+        } elsif ($lines[$line] =~ /^.\s*#\s*(?:else|elif)\b/) {
+            $level = $stack[$#stack - 1];
+        } elsif ($lines[$line] =~ /^.\s*#\s*endif\b/) {
+            $level = pop(@stack);
+        }
+
+        foreach my $c (split(//, $lines[$line])) {
+            ##print "C<$c>L<$level><$open$close>O<$off>\n";
+            if ($off > 0) {
+                $off--;
+                next;
+            }
+
+            if ($c eq $close && $level > 0) {
+                $level--;
+                last if ($level == 0);
+            } elsif ($c eq $open) {
+                $level++;
+            }
+        }
+
+        if (!$outer || $level <= 1) {
+            push(@res, $rawlines[$line]);
+        }
+
+        last if ($level == 0);
+    }
+
+    return ($level, @res);
+}
+sub ctx_block_outer {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 1, '{', '}', 0);
+    return @r;
+}
+sub ctx_block {
+    my ($linenr, $remain) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+    return @r;
+}
+sub ctx_statement {
+    my ($linenr, $remain, $off) = @_;
+
+    my ($level, @r) = ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+    return @r;
+}
+sub ctx_block_level {
+    my ($linenr, $remain) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '{', '}', 0);
+}
+sub ctx_statement_level {
+    my ($linenr, $remain, $off) = @_;
+
+    return ctx_block_get($linenr, $remain, 0, '(', ')', $off);
+}
+
+sub ctx_locate_comment {
+    my ($first_line, $end_line) = @_;
+
+    # Catch a comment on the end of the line itself.
+    my ($current_comment) = ($rawlines[$end_line - 1] =~ m@.*(/\*.*\*/)\s*(?:\\\s*)?$@);
+    return $current_comment if (defined $current_comment);
+
+    # Look through the context and try and figure out if there is a
+    # comment.
+    my $in_comment = 0;
+    $current_comment = '';
+    for (my $linenr = $first_line; $linenr < $end_line; $linenr++) {
+        my $line = $rawlines[$linenr - 1];
+        #warn "           $line\n";
+        if ($linenr == $first_line and $line =~ m@^.\s*\*@) {
+            $in_comment = 1;
+        }
+        if ($line =~ m@/\*@) {
+            $in_comment = 1;
+        }
+        if (!$in_comment && $current_comment ne '') {
+            $current_comment = '';
+        }
+        $current_comment .= $line . "\n" if ($in_comment);
+        if ($line =~ m@\*/@) {
+            $in_comment = 0;
+        }
+    }
+
+    chomp($current_comment);
+    return($current_comment);
+}
+sub ctx_has_comment {
+    my ($first_line, $end_line) = @_;
+    my $cmt = ctx_locate_comment($first_line, $end_line);
+
+    ##print "LINE: $rawlines[$end_line - 1 ]\n";
+    ##print "CMMT: $cmt\n";
+
+    return ($cmt ne '');
+}
+
+sub raw_line {
+    my ($linenr, $cnt) = @_;
+
+    my $offset = $linenr - 1;
+    $cnt++;
+
+    my $line;
+    while ($cnt) {
+        $line = $rawlines[$offset++];
+        next if (defined($line) && $line =~ /^-/);
+        $cnt--;
+    }
+    return $line;
+}
+
+sub cat_vet {
+    my ($vet) = @_;
+    my ($res, $coded);
+
+    $res = '';
+    while ($vet =~ /([^[:cntrl:]]*)([[:cntrl:]]|$)/g) {
+        $res .= $1;
+        if ($2 ne '') {
+            $coded = sprintf("^%c", unpack('C', $2) + 64);
+            $res .= $coded;
+        }
+    }
+    $res =~ s/$/\$/;
+    return $res;
+}
+
+my $av_preprocessor = 0;
+my $av_pending;
+my @av_paren_type;
+my $av_pend_colon;
+
+sub annotate_reset {
+    $av_preprocessor = 0;
+    $av_pending = '_';
+    @av_paren_type = ('E');
+    $av_pend_colon = 'O';
+}
+
+sub annotate_values {
+    my ($stream, $type) = @_;
+
+    my $res;
+    my $var = '_' x length($stream);
+    my $cur = $stream;
+
+    print "$stream\n" if ($dbg_values > 1);
+
+    while (length($cur)) {
+        @av_paren_type = ('E') if ($#av_paren_type < 0);
+        print " <" . join('', @av_paren_type) .
+            "> <$type> <$av_pending>" if ($dbg_values > 1);
+        if ($cur =~ /^(\s+)/o) {
+            print "WS($1)\n" if ($dbg_values > 1);
+            if ($1 =~ /\n/ && $av_preprocessor) {
+                $type = pop(@av_paren_type);
+                $av_preprocessor = 0;
+            }
+
+        } elsif ($cur =~ /^(\(\s*$Type\s*)\)/ && $av_pending eq '_') {
+            print "CAST($1)\n" if ($dbg_values > 1);
+            push(@av_paren_type, $type);
+            $type = 'c';
+
+        } elsif ($cur =~ /^($Type)\s*(?:$Ident|,|\)|\(|\s*$)/) {
+            print "DECLARE($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^($Modifier)\s*/) {
+            print "MODIFIER($1)\n" if ($dbg_values > 1);
+            $type = 'T';
+
+        } elsif ($cur =~ /^(\#\s*define\s*$Ident)(\(?)/o) {
+            print "DEFINE($1,$2)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+            if ($2 ne '') {
+                $av_pending = 'N';
+            }
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:undef\s*$Ident|include\b))/o) {
+            print "UNDEF($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+            push(@av_paren_type, $type);
+
+        } elsif ($cur =~ /^(\#\s*(?:ifdef|ifndef|if))/o) {
+            print "PRE_START($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:else|elif))/o) {
+            print "PRE_RESTART($1)\n" if ($dbg_values > 1);
+            $av_preprocessor = 1;
+
+            push(@av_paren_type, $av_paren_type[$#av_paren_type]);
+
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\#\s*(?:endif))/o) {
+            print "PRE_END($1)\n" if ($dbg_values > 1);
+
+            $av_preprocessor = 1;
+
+            # Assume all arms of the conditional end as this
+            # one does, and continue as if the #endif was not here.
+            pop(@av_paren_type);
+            push(@av_paren_type, $type);
+            $type = 'E';
+
+        } elsif ($cur =~ /^(\\\n)/o) {
+            print "PRECONT($1)\n" if ($dbg_values > 1);
+
+        } elsif ($cur =~ /^(__attribute__)\s*\(?/o) {
+            print "ATTR($1)\n" if ($dbg_values > 1);
+            $av_pending = $type;
+            $type = 'N';
+
+        } elsif ($cur =~ /^(sizeof)\s*(\()?/o) {
+            print "SIZEOF($1)\n" if ($dbg_values > 1);
+            if (defined $2) {
+                $av_pending = 'V';
+            }
+            $type = 'N';
+
+        } elsif ($cur =~ /^(if|while|for)\b/o) {
+            print "COND($1)\n" if ($dbg_values > 1);
+            $av_pending = 'E';
+            $type = 'N';
+
+        } elsif ($cur =~/^(case)/o) {
+            print "CASE($1)\n" if ($dbg_values > 1);
+            $av_pend_colon = 'C';
+            $type = 'N';
+
+        } elsif ($cur =~/^(return|else|goto|typeof|__typeof__)\b/o) {
+            print "KEYWORD($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\()/o) {
+            print "PAREN('$1')\n" if ($dbg_values > 1);
+            push(@av_paren_type, $av_pending);
+            $av_pending = '_';
+            $type = 'N';
+
+        } elsif ($cur =~ /^(\))/o) {
+            my $new_type = pop(@av_paren_type);
+            if ($new_type ne '_') {
+                $type = $new_type;
+                print "PAREN('$1') -> $type\n"
+                    if ($dbg_values > 1);
+            } else {
+                print "PAREN('$1')\n" if ($dbg_values > 1);
+            }
+
+        } elsif ($cur =~ /^($Ident)\s*\(/o) {
+            print "FUNC($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+            $av_pending = 'V';
+
+        } elsif ($cur =~ /^($Ident\s*):(?:\s*\d+\s*(,|=|;))?/) {
+            if (defined $2 && $type eq 'C' || $type eq 'T') {
+                $av_pend_colon = 'B';
+            } elsif ($type eq 'E') {
+                $av_pend_colon = 'L';
+            }
+            print "IDENT_COLON($1,$type>$av_pend_colon)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Ident|$Constant)/o) {
+            print "IDENT($1)\n" if ($dbg_values > 1);
+            $type = 'V';
+
+        } elsif ($cur =~ /^($Assignment)/o) {
+            print "ASSIGN($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~/^(;|{|})/) {
+            print "END($1)\n" if ($dbg_values > 1);
+            $type = 'E';
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~/^(,)/) {
+            print "COMMA($1)\n" if ($dbg_values > 1);
+            $type = 'C';
+
+        } elsif ($cur =~ /^(\?)/o) {
+            print "QUESTION($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(:)/o) {
+            print "COLON($1,$av_pend_colon)\n" if ($dbg_values > 1);
+
+            substr($var, length($res), 1, $av_pend_colon);
+            if ($av_pend_colon eq 'C' || $av_pend_colon eq 'L') {
+                $type = 'E';
+            } else {
+                $type = 'N';
+            }
+            $av_pend_colon = 'O';
+
+        } elsif ($cur =~ /^(\[)/o) {
+            print "CLOSE($1)\n" if ($dbg_values > 1);
+            $type = 'N';
+
+        } elsif ($cur =~ /^(-(?![->])|\+(?!\+)|\*|\&\&|\&)/o) {
+            my $variant;
+
+            print "OPV($1)\n" if ($dbg_values > 1);
+            if ($type eq 'V') {
+                $variant = 'B';
+            } else {
+                $variant = 'U';
+            }
+
+            substr($var, length($res), 1, $variant);
+            $type = 'N';
+
+        } elsif ($cur =~ /^($Operators)/o) {
+            print "OP($1)\n" if ($dbg_values > 1);
+            if ($1 ne '++' && $1 ne '--') {
+                $type = 'N';
+            }
+
+        } elsif ($cur =~ /(^.)/o) {
+            print "C($1)\n" if ($dbg_values > 1);
+        }
+        if (defined $1) {
+            $cur = substr($cur, length($1));
+            $res .= $type x length($1);
+        }
+    }
+
+    return ($res, $var);
+}
+
+sub possible {
+    my ($possible, $line) = @_;
+    my $notPermitted = qr{(?:
+                ^(?:
+                        $Modifier|
+                        $Storage|
+                        $Type|
+                        DEFINE_\S+
+                )$|
+                ^(?:
+                        goto|
+                        return|
+                        case|
+                        else|
+                        asm|__asm__|
+                        do|
+                        \#|
+                        \#\#|
+                )(?:\s|$)|
+                ^(?:typedef|struct|enum)\b
+            )}x;
+    warn "CHECK<$possible> ($line)\n" if ($dbg_possible > 2);
+    if ($possible !~ $notPermitted) {
+        # Check for modifiers.
+        $possible =~ s/\s*$Storage\s*//g;
+        $possible =~ s/\s*$Sparse\s*//g;
+        if ($possible =~ /^\s*$/) {
+
+        } elsif ($possible =~ /\s/) {
+            $possible =~ s/\s*$Type\s*//g;
+            for my $modifier (split(' ', $possible)) {
+                if ($modifier !~ $notPermitted) {
+                    warn "MODIFIER: $modifier ($possible) ($line)\n" if ($dbg_possible);
+                    push(@modifierList, $modifier);
+                }
+            }
+
+        } else {
+            warn "POSSIBLE: $possible ($line)\n" if ($dbg_possible);
+            push(@typeList, $possible);
+        }
+        build_types();
+    } else {
+        warn "NOTPOSS: $possible ($line)\n" if ($dbg_possible > 1);
+    }
+}
+
+my $prefix = '';
+
+sub show_type {
+    my ($type) = @_;
+
+    return defined $use_type{$type} if (scalar keys %use_type > 0);
+
+    return !defined $ignore_type{$type};
+}
+
+sub report {
+    my ($level, $type, $msg) = @_;
+
+    if (!show_type($type) ||
+        (defined $tst_only && $msg !~ /\Q$tst_only\E/)) {
+        return 0;
+    }
+    my $line;
+    if ($show_types) {
+        $line = "$prefix$level:$type: $msg\n";
+    } else {
+        $line = "$prefix$level: $msg\n";
+    }
+    $line = (split('\n', $line))[0] . "\n" if ($terse);
+
+    if ($quiet == 0) {
+        push(our @report, $line);
+    }
+    return 1;
+}
+
+sub report_dump {
+    our @report;
+}
+
+sub ERROR {
+    my ($type, $msg) = @_;
+
+    if (report("ERROR", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_error++;
+        return 1;
+    }
+    return 0;
+}
+sub WARN {
+    my ($type, $msg) = @_;
+
+    if (report("WARNING", $type, $msg)) {
+        ## Warning is okay to submit
+        our $clean = 0;
+        our $cnt_warn++;
+        return 1;
+    }
+    return 0;
+}
+sub CHK {
+    my ($type, $msg) = @_;
+
+    if ($check && report("CHECK", $type, $msg)) {
+        our $clean = 0;
+        our $cnt_chk++;
+        return 1;
+    }
+    return 0;
+}
+
+sub check_absolute_file {
+    my ($absolute, $herecurr) = @_;
+    my $file = $absolute;
+
+    ##print "absolute<$absolute>\n";
+
+    # See if any suffix of this path is a path within the tree.
+    while ($file =~ s@^[^/]*/@@) {
+        if (-f "$root/$file") {
+            ##print "file<$file>\n";
+            last;
+        }
+    }
+    if (! -f _)  {
+        return 0;
+    }
+
+    # It is, so see if the prefix is acceptable.
+    my $prefix = $absolute;
+    substr($prefix, -length($file)) = '';
+
+    ##print "prefix<$prefix>\n";
+    if ($prefix ne ".../") {
+        WARN("USE_RELATIVE_PATH",
+             "use relative pathname instead of absolute in changelog text\n" . $herecurr);
+    }
+}
+
+sub trim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+|\s+$//g;
+
+    return $string;
+}
+
+sub ltrim {
+    my ($string) = @_;
+
+    $string =~ s/^\s+//;
+
+    return $string;
+}
+
+sub rtrim {
+    my ($string) = @_;
+
+    $string =~ s/\s+$//;
+
+    return $string;
+}
+
+sub string_find_replace {
+    my ($string, $find, $replace) = @_;
+
+    $string =~ s/$find/$replace/g;
+
+    return $string;
+}
+
+sub tabify {
+    my ($leading) = @_;
+
+    my $source_indent = 8;
+    my $max_spaces_before_tab = $source_indent - 1;
+    my $spaces_to_tab = " " x $source_indent;
+
+    #convert leading spaces to tabs
+    1 while $leading =~ s@^([\t]*)$spaces_to_tab@$1\t@g;
+    #Remove spaces before a tab
+    1 while $leading =~ s@^([\t]*)( {1,$max_spaces_before_tab})\t@$1\t@g;
+    return "$leading";
+}
+
+sub pos_last_openparen {
+    my ($line) = @_;
+
+    my $pos = 0;
+
+    my $opens = $line =~ tr/\(/\(/;
+    my $closes = $line =~ tr/\)/\)/;
+
+    my $last_openparen = 0;
+
+    if (($opens == 0) || ($closes >= $opens)) {
+        return -1;
+    }
+
+    my $len = length($line);
+
+    for ($pos = 0; $pos < $len; $pos++) {
+        my $string = substr($line, $pos);
+        if ($string =~ /^($FuncArg|$balanced_parens)/) {
+            $pos += length($1) - 1;
+        } elsif (substr($line, $pos, 1) eq '(') {
+            $last_openparen = $pos;
+        } elsif (index($string, '(') == -1) {
+            last;
+        }
+    }
+
+    return length(expand_tabs(substr($line, 0, $last_openparen))) + 1;
+}
+
+sub process {
+    my $filename = shift;
+
+    my $linenr=0;
+    my $prevline="";
+    my $prevrawline="";
+    my $stashline="";
+    my $stashrawline="";
+
+    my $length;
+    my $indent;
+    my $previndent=0;
+    my $stashindent=0;
+
+    our $clean = 1;
+    my $signoff = 0;
+    my $subject_trailing_dot = 0;
+    my $is_patch = 0;
+
+    my $in_header_lines = 1;
+    my $in_commit_log = 0; #Scanning lines before patch
+
+    my $non_utf8_charset = 0;
+
+    our @report = ();
+    our $cnt_lines = 0;
+    our $cnt_error = 0;
+    our $cnt_warn = 0;
+    our $cnt_chk = 0;
+
+    # Trace the real file/line as we go.
+    my $realfile = '';
+    my $realline = 0;
+    my $realcnt = 0;
+    my $here = '';
+    my $in_comment = 0;
+    my $comment_edge = 0;
+    my $first_line = 0;
+    my $p1_prefix = '';
+
+    my $prev_values = 'E';
+
+    # suppression flags
+    my %suppress_ifbraces;
+    my %suppress_whiletrailers;
+    my %suppress_export;
+    my $suppress_statement = 0;
+
+    my %signatures = ();
+
+    # Pre-scan the patch sanitizing the lines.
+    # Pre-scan the patch looking for any __setup documentation.
+    #
+    my @setup_docs = ();
+    my $setup_docs = 0;
+
+    my $camelcase_file_seeded = 0;
+
+    sanitise_line_reset();
+    my $line;
+    foreach my $rawline (@rawlines) {
+        $linenr++;
+        $line = $rawline;
+
+        push(@fixed, $rawline) if ($fix);
+
+        if ($rawline=~/^\+\+\+\s+(\S+)/) {
+            $setup_docs = 0;
+            if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
+                $setup_docs = 1;
+            }
+            #next;
+        }
+        if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            $in_comment = 0;
+
+            # Guestimate if this is a continuing comment.  Run
+            # the context looking for a comment "edge".  If this
+            # edge is a close comment then we must be in a comment
+            # at context start.
+            my $edge;
+            my $cnt = $realcnt;
+            for (my $ln = $linenr + 1; $cnt > 0; $ln++) {
+                next if (defined $rawlines[$ln - 1] &&
+                         $rawlines[$ln - 1] =~ /^-/);
+                $cnt--;
+                #print "RAW<$rawlines[$ln - 1]>\n";
+                last if (!defined $rawlines[$ln - 1]);
+                if ($rawlines[$ln - 1] =~ m@(/\*|\*/)@ &&
+                    $rawlines[$ln - 1] !~ m@"[^"]*(?:/\*|\*/)[^"]*"@) {
+                    ($edge) = $1;
+                    last;
+                }
+            }
+            if (defined $edge && $edge eq '*/') {
+                $in_comment = 1;
+            }
+
+            # Guestimate if this is a continuing comment.  If this
+            # is the start of a diff block and this line starts
+            # ' *' then it is very likely a comment.
+            if (!defined $edge &&
+                $rawlines[$linenr] =~ m@^.\s*(?:\*\*+| \*)(?:\s|$)@)
+            {
+                $in_comment = 1;
+            }
+
+            ##print "COMMENT:$in_comment edge<$edge> $rawline\n";
+            sanitise_line_reset($in_comment);
+
+        } elsif ($realcnt && $rawline =~ /^(?:\+| |$)/) {
+            # Standardise the strings and chars within the input to
+            # simplify matching -- only bother with positive lines.
+            $line = sanitise_line($rawline);
+        }
+        push(@lines, $line);
+
+        if ($realcnt > 1) {
+            $realcnt-- if ($line =~ /^(?:\+| |$)/);
+        } else {
+            $realcnt = 0;
+        }
+
+        #print "==>$rawline\n";
+        #print "-->$line\n";
+
+        if ($setup_docs && $line =~ /^\+/) {
+            push(@setup_docs, $line);
+        }
+    }
+
+    $prefix = '';
+
+    $realcnt = 0;
+    $linenr = 0;
+    foreach my $line (@lines) {
+        $linenr++;
+        my $sline = $line;      #copy of $line
+        $sline =~ s/$;/ /g;     #with comments as spaces
+
+        my $rawline = $rawlines[$linenr - 1];
+
+#extract the line range in the file after the patch is applied
+        if ($line=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
+            $is_patch = 1;
+            $first_line = $linenr + 1;
+            $realline=$1-1;
+            if (defined $2) {
+                $realcnt=$3+1;
+            } else {
+                $realcnt=1+1;
+            }
+            annotate_reset();
+            $prev_values = 'E';
+
+            %suppress_ifbraces = ();
+            %suppress_whiletrailers = ();
+            %suppress_export = ();
+            $suppress_statement = 0;
+            next;
+
+# track the line number as we move through the hunk, note that
+# new versions of GNU diff omit the leading space on completely
+# blank context lines so we need to count that too.
+        } elsif ($line =~ /^( |\+|$)/) {
+            $realline++;
+            $realcnt-- if ($realcnt != 0);
+
+            # Measure the line length and indent.
+            ($length, $indent) = line_stats($rawline);
+
+            # Track the previous line.
+            ($prevline, $stashline) = ($stashline, $line);
+            ($previndent, $stashindent) = ($stashindent, $indent);
+            ($prevrawline, $stashrawline) = ($stashrawline, $rawline);
+
+            #warn "line<$line>\n";
+
+        } elsif ($realcnt == 1) {
+            $realcnt--;
+        }
+
+        my $hunk_line = ($realcnt != 0);
+
+#make up the handle for any error we report on this line
+        $prefix = "$filename:$realline: " if ($emacs && $file);
+        $prefix = "$filename:$linenr: " if ($emacs && !$file);
+
+        $here = "#$linenr: " if (!$file);
+        $here = "#$realline: " if ($file);
+
+        my $found_file = 0;
+        # extract the filename as it passes
+        if ($line =~ /^diff --git.*?(\S+)$/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+            $found_file = 1;
+        } elsif ($line =~ /^\+\+\+\s+(\S+)/) {
+            $realfile = $1;
+            $realfile =~ s@^([^/]*)/@@ if (!$file);
+            $in_commit_log = 0;
+
+            $p1_prefix = $1;
+            if (!$file && $tree && $p1_prefix ne '' &&
+                -e "$root/$p1_prefix") {
+                WARN("PATCH_PREFIX",
+                     "patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
+            }
+
+            $found_file = 1;
+        }
+
+        if ($found_file) {
+            if ($realfile =~ m@^(drivers/net/|net/)@) {
+                $check = 1;
+            } else {
+                $check = $check_orig;
+            }
+            next;
+        }
+
+        $here .= "FILE: $realfile:$realline:" if ($realcnt != 0);
+
+        my $hereline = "$here\n$rawline\n";
+        my $herecurr = "$here\n$rawline\n";
+        my $hereprev = "$here\n$prevrawline\n$rawline\n";
+
+        $cnt_lines++ if ($realcnt != 0);
+
+# Check for incorrect file permissions
+        if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
+            my $permhere = $here . "FILE: $realfile\n";
+            if ($realfile !~ m@scripts/@ &&
+                $realfile !~ /\.(py|pl|awk|sh|t)$/) {
+                ERROR("EXECUTE_PERMISSIONS",
+                      "do not set execute permissions for source files\n" . $permhere);
+            }
+        }
+
+        next if ($realfile =~ /(checkpatch.pl)/);
+        next if ($realfile =~ /\.(md|txt|doc|8|pdf|tex)$/);
+
+# Check that the subject does not have a trailing dot
+        if ($in_header_lines &&
+            $line =~ /^Subject: \[PATCH\] (.+)\.(\s*)$/) {
+                $subject_trailing_dot++;
+        }
+
+# Check the patch for a signoff:
+        if ($line =~ /^\s*signed-off-by:/i) {
+            $signoff++;
+            $in_commit_log = 0;
+        }
+
+# Check signature styles
+        if (!$in_header_lines &&
+            $line =~ /^(\s*)([a-z0-9_-]+by:|$signature_tags)(\s*)(.*)/i) {
+            my $space_before = $1;
+            my $sign_off = $2;
+            my $space_after = $3;
+            my $email = $4;
+            my $ucfirst_sign_off = ucfirst(lc($sign_off));
+
+            if ($sign_off !~ /$signature_tags/) {
+                WARN("BAD_SIGN_OFF",
+                     "Non-standard signature: $sign_off\n" . $herecurr);
+            }
+            if (defined $space_before && $space_before ne "") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Do not use whitespace before $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+            if ($sign_off =~ /-by:$/i && $sign_off ne $ucfirst_sign_off) {
+                if (WARN("BAD_SIGN_OFF",
+                         "'$ucfirst_sign_off' is the preferred signature form\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+
+            }
+            if (!defined $space_after || $space_after ne " ") {
+                if (WARN("BAD_SIGN_OFF",
+                         "Use a single space after $ucfirst_sign_off\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =
+                        "$ucfirst_sign_off $email";
+                }
+            }
+
+            # Check if email is really Gerrit URL
+            if ($email =~ /^($url_tags)(.*)/) {
+                my $uri = $1;
+                my $url = $2;
+                if ($uri && $url !~ /$gerrit_url/) {
+                    ERROR("BAD_URL",
+                          "Unrecognized url address: '$email'\n" . $herecurr);
+                }
+            } else {
+                my ($email_name, $email_address, $comment) = parse_email($email);
+                my $suggested_email = format_email(($email_name, $email_address));
+                if ($suggested_email eq "") {
+                    ERROR("BAD_SIGN_OFF",
+                          "Unrecognized email address: '$email'\n" . $herecurr);
+                } else {
+                    my $dequoted = $suggested_email;
+                    $dequoted =~ s/^"//;
+                    $dequoted =~ s/" </ </;
+                    # Don't force email to have quotes
+                    # Allow just an angle bracketed address
+                    if ("$dequoted$comment" ne $email &&
+                        "<$email_address>$comment" ne $email &&
+                        "$suggested_email$comment" ne $email) {
+                        WARN("BAD_SIGN_OFF",
+                             "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr);
+                    }
+                }
+            }
+
+# Check for duplicate signatures
+            my $sig_nospace = $line;
+            $sig_nospace =~ s/\s//g;
+            $sig_nospace = lc($sig_nospace);
+            if (defined $signatures{$sig_nospace}) {
+                WARN("BAD_SIGN_OFF",
+                     "Duplicate signature\n" . $herecurr);
+            } else {
+                $signatures{$sig_nospace} = 1;
+            }
+        }
+
+# Check for wrappage within a valid hunk of the file
+        if ($realcnt != 0 && $line !~ m{^(?:\+|-| |\\ No newline|$)}) {
+            ERROR("CORRUPTED_PATCH",
+                  "patch seems to be corrupt (line wrapped?)\n" .
+                  $herecurr) if (!$emitted_corrupt++);
+        }
+
+# Check for absolute kernel paths.
+        if ($tree) {
+            while ($line =~ m{(?:^|\s)(/\S*)}g) {
+                my $file = $1;
+
+                if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
+                    check_absolute_file($1, $herecurr)) {
+                    #
+                } else {
+                    check_absolute_file($file, $herecurr);
+                }
+            }
+        }
+
+# UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+        if (($realfile =~ /^$/ || $line =~ /^\+/) &&
+            $rawline !~ m/^$UTF8*$/) {
+            my ($utf8_prefix) = ($rawline =~ /^($UTF8*)/);
+
+            my $blank = copy_spacing($rawline);
+            my $ptr = substr($blank, 0, length($utf8_prefix)) . "^";
+            my $hereptr = "$hereline$ptr\n";
+
+            CHK("INVALID_UTF8",
+                "Invalid UTF-8, patch and commit message should be encoded in UTF-8\n" . $hereptr);
+        }
+
+# Check if it's the start of a commit log
+# (not a header line and we haven't seen the patch filename)
+        if ($in_header_lines && $realfile =~ /^$/ &&
+            $rawline !~ /^(commit\b|from\b|[\w-]+:).+$/i) {
+            $in_header_lines = 0;
+            $in_commit_log = 1;
+        }
+
+# Check if there is UTF-8 in a commit log when a mail header has explicitly
+# declined it, i.e defined some charset where it is missing.
+        if ($in_header_lines &&
+            $rawline =~ /^Content-Type:.+charset="(.+)".*$/ &&
+            $1 !~ /utf-8/i) {
+            $non_utf8_charset = 1;
+        }
+
+        if ($in_commit_log && $non_utf8_charset && $realfile =~ /^$/ &&
+            $rawline =~ /$NON_ASCII_UTF8/) {
+            WARN("UTF8_BEFORE_PATCH",
+                 "8-bit UTF-8 used in possible commit log\n" . $herecurr);
+        }
+
+# ignore non-hunk lines and lines being removed
+        next if (!$hunk_line || $line =~ /^-/);
+
+#trailing whitespace
+        if ($line =~ /^\+.*\015/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("DOS_LINE_ENDINGS",
+                      "DOS line endings\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/[\s\015]+$//;
+            }
+        } elsif ($rawline =~ /^\+.*\S\s+$/ || $rawline =~ /^\+\s+$/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (ERROR("TRAILING_WHITESPACE",
+                      "trailing whitespace\n" . $herevet) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+
+            $rpt_cleaners = 1;
+        }
+
+        if (($realfile =~ /Makefile.*/) &&
+            ($line =~ /\+(EXTRA_[A-Z]+FLAGS).*/)) {
+            my $flag = $1;
+            my $replacement = {
+                            'EXTRA_AFLAGS' =>   'asflags-y',
+                            'EXTRA_CFLAGS' =>   'ccflags-y',
+                            'EXTRA_CPPFLAGS' => 'cppflags-y',
+                            'EXTRA_LDFLAGS' =>  'ldflags-y',
+            };
+
+            WARN("DEPRECATED_VARIABLE",
+                 "Use of $flag is deprecated, please use \`$replacement->{$flag} instead.\n" . $herecurr) if ($replacement->{$flag});
+        }
+
+# check we are in .spec file, then ignore this hunk
+        next if ($realfile eq "glusterfs.spec.in");
+
+# check we are in a valid source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl|py|l|y|sh|in)$/);
+
+#line length limit
+        if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
+            $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
+            !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:|,|\)\s*;)\s*$/ ||
+              $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+            $length > $max_line_length)
+        {
+            WARN("LONG_LINE",
+                 "line over $max_line_length characters\n" . $herecurr);
+        }
+
+# check for spaces before a quoted newline
+        if ($rawline =~ /^.*\".*\s\\n/) {
+            if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE",
+                     "unnecessary whitespace before a quoted newline\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*\".*)\s+\\n/$1\\n/;
+            }
+
+        }
+
+# check for adding lines without a newline.
+        if ($line =~ /^\+/ && defined $lines[$linenr] && $lines[$linenr] =~ /^\\ No newline at end of file/) {
+            WARN("MISSING_EOF_NEWLINE",
+                 "adding a line without newline at end of file\n" . $herecurr);
+        }
+
+# check we are in a valid source file C or perl if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c|pl)$/);
+
+# check for space before tabs.
+        if ($rawline =~ /^\+/ && $rawline =~ / \t/) {
+            my $herevet = "$here\n" . cat_vet($rawline) . "\n";
+            if (WARN("SPACE_BEFORE_TAB",
+                     "please, no space before tabs\n" . $herevet) &&
+                $fix) {
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) {8,8}\t/$1\t\t/) {}
+                while ($fixed[$linenr - 1] =~
+                       s/(^\+.*) +\t/$1\t/) {}
+            }
+        }
+
+# check for && or || at the start of a line
+        if ($rawline =~ /^\+\s*(&&|\|\|)/) {
+            CHK("LOGICAL_CONTINUATIONS",
+                "Logical continuations should be on the previous line\n" . $hereprev);
+        }
+
+# check multi-line statement indentation matches previous line
+        if ($^V && $^V ge 5.10.0 &&
+            $prevline =~ /^\+([ \t]*)((?:$c90_Keywords(?:\s+if)\s*)|(?:$Declare\s*)?(?:$Ident|\(\s*\*\s*$Ident\s*\))\s*|$Ident\s*=\s*$Ident\s*)\(.*(\&\&|\|\||,)\s*$/) {
+            $prevline =~ /^\+(\t*)(.*)$/;
+            my $oldindent = $1;
+            my $rest = $2;
+
+            my $pos = pos_last_openparen($rest);
+            if ($pos >= 0) {
+                $line =~ /^(\+| )([ \t]*)/;
+                my $newindent = $2;
+
+                my $goodtabindent = $oldindent .
+                    "\t" x ($pos / 8) .
+                    " "  x ($pos % 8);
+                my $goodspaceindent = $oldindent . " "  x $pos;
+
+                if ($newindent ne $goodtabindent &&
+                    $newindent ne $goodspaceindent) {
+
+                    if (CHK("PARENTHESIS_ALIGNMENT",
+                            "Alignment should match open parenthesis\n" . $hereprev) &&
+                        $fix && $line =~ /^\+/) {
+                        $fixed[$linenr - 1] =~
+                            s/^\+[ \t]*/\+$goodtabindent/;
+                    }
+                }
+            }
+        }
+
+        if ($line =~ /^\+.*\*[ \t]*\)[ \t]+(?!$Assignment|$Arithmetic)/) {
+            if (CHK("SPACING",
+                    "No space is necessary after a cast\n" . $hereprev) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(\+.*\*[ \t]*\))[ \t]+/$1/;
+            }
+        }
+
+
+# check for missing blank lines after declarations
+        if ($sline =~ /^\+\s+\S/ &&  #Not at char 1
+            # actual declarations
+            ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+             # foo bar; where foo is some local typedef or #define
+             $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+             # known declaration macros
+             $prevline =~ /^\+\s+$declaration_macros/) &&
+            # for "else if" which can look like "$Ident $Ident"
+            !($prevline =~ /^\+\s+$c90_Keywords\b/ ||
+              # other possible extensions of declaration lines
+              $prevline =~ /(?:$Compare|$Assignment|$Operators)\s*$/ ||
+              # not starting a section or a macro "\" extended line
+              $prevline =~ /(?:\{\s*|\\)$/) &&
+            # looks like a declaration
+            !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
+              # foo bar; where foo is some local typedef or #define
+              $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
+              # known declaration macros
+              $sline =~ /^\+\s+$declaration_macros/ ||
+              # start of struct or union or enum
+              $sline =~ /^\+\s+(?:union|struct|enum|typedef)\b/ ||
+              # start or end of block or continuation of declaration
+              $sline =~ /^\+\s+(?:$|[\{\}\.\#\"\?\:\(\[])/ ||
+              # bitfield continuation
+              $sline =~ /^\+\s+$Ident\s*:\s*\d+\s*[,;]/ ||
+              # other possible extensions of declaration lines
+              $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) &&
+            # indentation of previous and current line are the same
+            (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) {
+            WARN("SPACING",
+                 "Missing a blank line after declarations\n" . $hereprev);
+        }
+
+# check we are in a valid C source file if not then ignore this hunk
+        next if ($realfile !~ /\.(h|c)$/);
+
+# check for RCS/CVS revision markers
+        if ($rawline =~ /^\+.*\$(Revision|Log|Id)(?:\$|)/) {
+            WARN("CVS_KEYWORD",
+                 "CVS style keyword markers, these will _not_ be updated\n". $herecurr);
+        }
+
+# Check for potential 'bare' types
+        my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
+            $realline_next);
+#print "LINE<$line>\n";
+        if ($linenr >= $suppress_statement &&
+            $realcnt && $sline =~ /.\s*\S/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $stat =~ s/\n./\n /g;
+            $cond =~ s/\n./\n /g;
+
+#print "linenr<$linenr> <$stat>\n";
+            # If this statement has no statement boundaries within
+            # it there is no point in retrying a statement scan
+            # until we hit end of it.
+            my $frag = $stat; $frag =~ s/;+\s*$//;
+            if ($frag !~ /(?:{|;)/) {
+#print "skip<$line_nr_next>\n";
+                $suppress_statement = $line_nr_next;
+            }
+
+            # Find the real next line.
+            $realline_next = $line_nr_next;
+            if (defined $realline_next &&
+                (!defined $lines[$realline_next - 1] ||
+                 substr($lines[$realline_next - 1], $off_next) =~ /^\s*$/)) {
+                $realline_next++;
+            }
+
+            my $s = $stat;
+            $s =~ s/{.*$//s;
+
+            # Ignore goto labels.
+            if ($s =~ /$Ident:\*$/s) {
+
+                # Ignore functions being called
+            } elsif ($s =~ /^.\s*$Ident\s*\(/s) {
+
+            } elsif ($s =~ /^.\s*else\b/s) {
+
+                # declarations always start with types
+            } elsif ($prev_values eq 'E' && $s =~ /^.\s*(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?((?:\s*$Ident)+?)\b(?:\s+$Sparse)?\s*\**\s*(?:$Ident|\(\*[^\)]*\))(?:\s*$Modifier)?\s*(?:;|=|,|\()/s) {
+                my $type = $1;
+                $type =~ s/\s+/ /g;
+                possible($type, "A:" . $s);
+
+                # definitions in global scope can only start with types
+            } elsif ($s =~ /^.(?:$Storage\s+)?(?:$Inline\s+)?(?:const\s+)?($Ident)\b\s*(?!:)/s) {
+                possible($1, "B:" . $s);
+            }
+
+            # any (foo ... *) is a pointer cast, and foo is a type
+            while ($s =~ /\(($Ident)(?:\s+$Sparse)*[\s\*]+\s*\)/sg) {
+                possible($1, "C:" . $s);
+            }
+
+            # Check for any sort of function declaration.
+            # int foo(something bar, other baz);
+            # void (*store_gdt)(x86_descr_ptr *);
+            if ($prev_values eq 'E' && $s =~ /^(.(?:typedef\s*)?(?:(?:$Storage|$Inline)\s*)*\s*$Type\s*(?:\b$Ident|\(\*\s*$Ident\))\s*)\(/s) {
+                my ($name_len) = length($1);
+
+                my $ctx = $s;
+                substr($ctx, 0, $name_len + 1, '');
+                $ctx =~ s/\)[^\)]*$//;
+
+                for my $arg (split(/\s*,\s*/, $ctx)) {
+                    if ($arg =~ /^(?:const\s+)?($Ident)(?:\s+$Sparse)*\s*\**\s*(:?\b$Ident)?$/s || $arg =~ /^($Ident)$/s) {
+
+                        possible($1, "D:" . $s);
+                    }
+                }
+            }
+
+        }
+
+#
+# Checks which may be anchored in the context.
+#
+
+# Check for switch () and associated case and default
+# statements should be at the same indent.
+        if ($line=~/\bswitch\s*\(.*\)/) {
+            my $err = '';
+            my $sep = '';
+            my @ctx = ctx_block_outer($linenr, $realcnt);
+            shift(@ctx);
+            for my $ctx (@ctx) {
+                my ($clen, $cindent) = line_stats($ctx);
+                if ($ctx =~ /^\+\s*(case\s+|default:)/ &&
+                    $indent != $cindent) {
+                    $err .= "$sep$ctx\n";
+                    $sep = '';
+                } else {
+                    $sep = "[...]\n";
+                }
+            }
+            if ($err ne '') {
+                ERROR("SWITCH_CASE_INDENT_LEVEL",
+                      "switch and case should be at the same indent\n$hereline$err");
+            }
+        }
+
+# if/while/etc brace do not go on next line, unless defining a do while loop,
+# or if that brace on the next line is for something else
+        if ($line =~ /(.*)\b((?:if|while|for|switch)\s*\(|do\b|else\b)/ && $line !~ /^.\s*\#/) {
+            my $pre_ctx = "$1$2";
+
+            my ($level, @ctx) = ctx_statement_level($linenr, $realcnt, 0);
+
+            if ($line =~ /^\+\t{6,}/) {
+                WARN("DEEP_INDENTATION",
+                     "Too many leading tabs - consider code refactoring\n" . $herecurr);
+            }
+
+            my $ctx_cnt = $realcnt - $#ctx - 1;
+            my $ctx = join("\n", @ctx);
+
+            my $ctx_ln = $linenr;
+            my $ctx_skip = $realcnt;
+
+            while ($ctx_skip > $ctx_cnt || ($ctx_skip == $ctx_cnt &&
+                                            defined $lines[$ctx_ln - 1] &&
+                                            $lines[$ctx_ln - 1] =~ /^-/)) {
+                ##print "SKIP<$ctx_skip> CNT<$ctx_cnt>\n";
+                $ctx_skip-- if (!defined $lines[$ctx_ln - 1] || $lines[$ctx_ln - 1] !~ /^-/);
+                $ctx_ln++;
+            }
+
+            #print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n";
+            #print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n";
+
+            if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln -1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) {
+                ERROR("OPEN_BRACE",
+                      "that open brace { should be on the previous line\n" .
+                      "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+            }
+            if ($level == 0 && $pre_ctx !~ /}\s*while\s*\($/ &&
+                $ctx =~ /\)\s*\;\s*$/ &&
+                defined $lines[$ctx_ln - 1])
+            {
+                my ($nlength, $nindent) = line_stats($lines[$ctx_ln - 1]);
+                if ($nindent > $indent) {
+                    WARN("TRAILING_SEMICOLON",
+                         "trailing semicolon indicates no statements, indent implies otherwise\n" .
+                         "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
+                }
+            }
+        }
+
+# Check relative indent for conditionals and blocks.
+        if ($line =~ /\b(?:(?:if|while|for)\s*\(|do\b)/ && $line !~ /^.\s*#/ && $line !~ /\}\s*while\s*/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($s, $c) = ($stat, $cond);
+
+            substr($s, 0, length($c), '');
+
+            # Make sure we remove the line prefixes as we have
+            # none on the first line, and are going to readd them
+            # where necessary.
+            $s =~ s/\n./\n/gs;
+
+            # Find out how long the conditional actually is.
+            my @newlines = ($c =~ /\n/gs);
+            my $cond_lines = 1 + $#newlines;
+
+            # We want to check the first line inside the block
+            # starting at the end of the conditional, so remove:
+            #  1) any blank line termination
+            #  2) any opening brace { on end of the line
+            #  3) any do (...) {
+            my $continuation = 0;
+            my $check = 0;
+            $s =~ s/^.*\bdo\b//;
+            $s =~ s/^\s*{//;
+            if ($s =~ s/^\s*\\//) {
+                $continuation = 1;
+            }
+            if ($s =~ s/^\s*?\n//) {
+                $check = 1;
+                $cond_lines++;
+            }
+
+            # Also ignore a loop construct at the end of a
+            # preprocessor statement.
+            if (($prevline =~ /^.\s*#\s*define\s/ ||
+                 $prevline =~ /\\\s*$/) && $continuation == 0) {
+                $check = 0;
+            }
+
+            my $cond_ptr = -1;
+            $continuation = 0;
+            while ($cond_ptr != $cond_lines) {
+                $cond_ptr = $cond_lines;
+
+                # If we see an #else/#elif then the code
+                # is not linear.
+                if ($s =~ /^\s*\#\s*(?:else|elif)/) {
+                    $check = 0;
+                }
+
+                # Ignore:
+                #  1) blank lines, they should be at 0,
+                #  2) preprocessor lines, and
+                #  3) labels.
+                if ($continuation ||
+                    $s =~ /^\s*?\n/ ||
+                    $s =~ /^\s*#\s*?/ ||
+                    $s =~ /^\s*$Ident\s*:/) {
+                    $continuation = ($s =~ /^.*?\\\n/) ? 1 : 0;
+                    if ($s =~ s/^.*?\n//) {
+                        $cond_lines++;
+                    }
+                }
+            }
+
+            my (undef, $sindent) = line_stats("+" . $s);
+            my $stat_real = raw_line($linenr, $cond_lines);
+
+            # Check if either of these lines are modified, else
+            # this is not this patch's fault.
+            if (!defined($stat_real) ||
+                $stat !~ /^\+/ && $stat_real !~ /^\+/) {
+                $check = 0;
+            }
+            if (defined($stat_real) && $cond_lines > 1) {
+                $stat_real = "[...]\n$stat_real";
+            }
+
+            #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n";
+
+            if ($check && (($sindent % 8) != 0 ||
+                           ($sindent <= $indent && $s ne ''))) {
+                WARN("SUSPECT_CODE_INDENT",
+                     "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
+            }
+        }
+
+        # Track the 'values' across context and added lines.
+        my $opline = $line; $opline =~ s/^./ /;
+        my ($curr_values, $curr_vars) =
+            annotate_values($opline . "\n", $prev_values);
+        $curr_values = $prev_values . $curr_values;
+        if ($dbg_values) {
+            my $outline = $opline; $outline =~ s/\t/ /g;
+            print "$linenr > .$outline\n";
+            print "$linenr > $curr_values\n";
+            print "$linenr >  $curr_vars\n";
+        }
+        $prev_values = substr($curr_values, -1);
+
+#ignore lines not being added
+        next if ($line =~ /^[^\+]/);
+
+# TEST: allow direct testing of the type matcher.
+        if ($dbg_type) {
+            if ($line =~ /^.\s*$Declare\s*$/) {
+                ERROR("TEST_TYPE",
+                      "TEST: is type\n" . $herecurr);
+            } elsif ($dbg_type > 1 && $line =~ /^.+($Declare)/) {
+                ERROR("TEST_NOT_TYPE",
+                      "TEST: is not type ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+# TEST: allow direct testing of the attribute matcher.
+        if ($dbg_attr) {
+            if ($line =~ /^.\s*$Modifier\s*$/) {
+                ERROR("TEST_ATTR",
+                      "TEST: is attr\n" . $herecurr);
+            } elsif ($dbg_attr > 1 && $line =~ /^.+($Modifier)/) {
+                ERROR("TEST_NOT_ATTR",
+                      "TEST: is not attr ($1 is)\n". $herecurr);
+            }
+            next;
+        }
+
+# check for initialisation to aggregates open brace on the next line
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /(?:^|[^=])=\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "that open brace { should be on the previous line\n" . $hereprev);
+        }
+
+#
+# Checks which are anchored on the added line.
+#
+
+# check for malformed paths in #include statements (uses RAW line)
+        if ($rawline =~ m{^.\s*\#\s*include\s+[<"](.*)[">]}) {
+            my $path = $1;
+            if ($path =~ m{//}) {
+                ERROR("MALFORMED_INCLUDE",
+                      "malformed #include filename\n" . $herecurr);
+            }
+            if ($path =~ "^uapi/" && $realfile =~ m@\binclude/uapi/@) {
+                ERROR("UAPI_INCLUDE",
+                      "No #include in ...include/uapi/... should use a uapi/ path prefix\n" . $herecurr);
+            }
+        }
+
+# no C99 // comments
+        if ($line =~ m{//}) {
+            if (ERROR("C99_COMMENTS",
+                      "do not use C99 // comments\n" . $herecurr) &&
+                $fix) {
+                my $line = $fixed[$linenr - 1];
+                if ($line =~ /\/\/(.*)$/) {
+                    my $comment = trim($1);
+                    $fixed[$linenr - 1] =~ s@\/\/(.*)$@/\* $comment \*/@;
+                }
+            }
+        }
+        # Remove C99 comments.
+        $line =~ s@//.*@@;
+        $opline =~ s@//.*@@;
+
+# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
+# the whole statement.
+#print "APW <$lines[$realline_next - 1]>\n";
+        if (defined $realline_next &&
+            exists $lines[$realline_next - 1] &&
+            !defined $suppress_export{$realline_next} &&
+            ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+            # Handle definitions which produce identifiers with
+            # a prefix:
+            #   XXX(foo);
+            #   EXPORT_SYMBOL(something_foo);
+            my $name = $1;
+            if ($stat =~ /^(?:.\s*}\s*\n)?.([A-Z_]+)\s*\(\s*($Ident)/ &&
+                $name =~ /^${Ident}_$2/) {
+#print "FOO C name<$name>\n";
+                $suppress_export{$realline_next} = 1;
+
+            } elsif ($stat !~ /(?:
+                                \n.}\s*$|
+                                ^.DEFINE_$Ident\(\Q$name\E\)|
+                                ^.DECLARE_$Ident\(\Q$name\E\)|
+                                ^.LIST_HEAD\(\Q$name\E\)|
+                                ^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
+                                \b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
+                            )/x) {
+#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
+                $suppress_export{$realline_next} = 2;
+            } else {
+                $suppress_export{$realline_next} = 1;
+            }
+        }
+        if (!defined $suppress_export{$linenr} &&
+            $prevline =~ /^.\s*$/ &&
+            ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
+             $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
+#print "FOO B <$lines[$linenr - 1]>\n";
+            $suppress_export{$linenr} = 2;
+        }
+        if (defined $suppress_export{$linenr} &&
+            $suppress_export{$linenr} == 2) {
+            WARN("EXPORT_SYMBOL",
+                 "EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
+        }
+
+# check for global initialisers.
+        if ($line =~ /^\+(\s*$Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("GLOBAL_INITIALISERS",
+                      "do not initialise globals to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/($Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+# check for static initialisers.
+        if ($line =~ /^\+.*\bstatic\s.*=\s*(0|NULL|false)\s*;/) {
+            if (ERROR("INITIALISED_STATIC",
+                      "do not initialise statics to 0 or NULL\n" .
+                      $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\bstatic\s.*?)\s*=\s*(0|NULL|false)\s*;/$1;/;
+            }
+        }
+
+# check for static const char * arrays.
+        if ($line =~ /\bstatic\s+const\s+char\s*\*\s*(\w+)\s*\[\s*\]\s*=\s*/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static const char * array should probably be static const char * const\n" .
+                 $herecurr);
+        }
+
+# check for static char foo[] = "bar" declarations.
+        if ($line =~ /\bstatic\s+char\s+(\w+)\s*\[\s*\]\s*=\s*"/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "static char array declaration should probably be static const char\n" .
+                 $herecurr);
+        }
+
+# check for non-global char *foo[] = {"bar", ...} declarations.
+        if ($line =~ /^.\s+(?:static\s+|const\s+)?char\s+\*\s*\w+\s*\[\s*\]\s*=\s*\{/) {
+            WARN("STATIC_CONST_CHAR_ARRAY",
+                 "char * array declaration might be better as static const\n" .
+                 $herecurr);
+        }
+
+# check for function declarations without arguments like "int foo()"
+        if ($line =~ /(\b$Type\s+$Ident)\s*\(\s*\)/) {
+            if (ERROR("FUNCTION_WITHOUT_ARGS",
+                      "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\b($Type)\s+($Ident))\s*\(\s*\)/$2 $3(void)/;
+            }
+        }
+
+# * goes on variable not on type
+        # (char*[ const])
+        while ($line =~ m{(\($NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)\))}g) {
+            #print "AA<$1>\n";
+            my ($ident, $from, $to) = ($1, $2, $2);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+
+##   print "1: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"(foo$from)\" should be \"(foo$to)\"\n" .  $herecurr) &&
+                    $fix) {
+                    my $sub_from = $ident;
+                    my $sub_to = $ident;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+        while ($line =~ m{(\b$NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)($Ident))}g) {
+            #print "BB<$1>\n";
+            my ($match, $from, $to, $ident) = ($1, $2, $2, $3);
+
+            # Should start with a space.
+            $to =~ s/^(\S)/ $1/;
+            # Should not end with a space.
+            $to =~ s/\s+$//;
+            # '*'s should not have spaces between.
+            while ($to =~ s/\*\s+\*/\*\*/) {
+            }
+            # Modifiers should have spaces.
+            $to =~ s/(\b$Modifier$)/$1 /;
+
+##  print "2: from<$from> to<$to> ident<$ident>\n";
+            if ($from ne $to && $ident !~ /^$Modifier$/) {
+                if (ERROR("POINTER_LOCATION",
+                          "\"foo${from}bar\" should be \"foo${to}bar\"\n" .  $herecurr) &&
+                    $fix) {
+
+                    my $sub_from = $match;
+                    my $sub_to = $match;
+                    $sub_to =~ s/\Q$from\E/$to/;
+                    $fixed[$linenr - 1] =~
+                        s@\Q$sub_from\E@$sub_to@;
+                }
+            }
+        }
+
+# function brace can't be on same line, except for #defines of do while,
+# or if closed on same line
+        if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+            !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following function declarations go on the next line\n" . $herecurr);
+        }
+
+# open braces for enum, union and struct go on the same line.
+        if ($line =~ /^.\s*{/ &&
+            $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
+            ERROR("OPEN_BRACE",
+                  "open brace '{' following $1 go on the same line\n" . $hereprev);
+        }
+
+# missing space after union, struct or enum definition
+        if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident){1,2}[=\{]/) {
+            if (WARN("SPACING",
+                     "missing space after $1 definition\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*(?:typedef\s+)?(?:enum|union|struct)(?:\s+$Ident){1,2})([=\{])/$1 $2/;
+            }
+        }
+
+# Function pointer declarations
+# check spacing between type, funcptr, and args
+# canonical declaration is "type (*funcptr)(args...)"
+        if ($line =~ /^.\s*($Declare)\((\s*)\*(\s*)($Ident)(\s*)\)(\s*)\(/) {
+            my $declare = $1;
+            my $pre_pointer_space = $2;
+            my $post_pointer_space = $3;
+            my $funcname = $4;
+            my $post_funcname_space = $5;
+            my $pre_args_space = $6;
+
+# the $Declare variable will capture all spaces after the type
+# so check it for a missing trailing missing space but pointer return types
+# don't need a space so don't warn for those.
+            my $post_declare_space = "";
+            if ($declare =~ /(\s+)$/) {
+                $post_declare_space = $1;
+                $declare = rtrim($declare);
+            }
+            if ($declare !~ /\*$/ && $post_declare_space =~ /^$/) {
+                WARN("SPACING",
+                     "missing space after return type\n" . $herecurr);
+                $post_declare_space = " ";
+            }
+
+# unnecessary space "type ( *funcptr)(args...)"
+            if (defined $pre_pointer_space &&
+                $pre_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer open parenthesis\n" . $herecurr);
+            }
+
+# unnecessary space "type (* funcptr)(args...)"
+            if (defined $post_pointer_space &&
+                $post_pointer_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr )(args...)"
+            if (defined $post_funcname_space &&
+                $post_funcname_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space after function pointer name\n" . $herecurr);
+            }
+
+# unnecessary space "type (*funcptr) (args...)"
+            if (defined $pre_args_space &&
+                $pre_args_space =~ /^\s/) {
+                WARN("SPACING",
+                     "Unnecessary space before function pointer arguments\n" . $herecurr);
+            }
+
+            if (show_type("SPACING") && $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*)$Declare\s*\(\s*\*\s*$Ident\s*\)\s*\(/$1 . $declare . $post_declare_space . '(*' . $funcname . ')('/ex;
+            }
+        }
+
+# check for spacing round square brackets; allowed:
+#  1. with a type on the left -- int [] a;
+#  2. at the beginning of a line for slice initialisers -- [0...10] = 5,
+#  3. inside a curly brace -- = { [0...10] = 5 }
+        while ($line =~ /(.*?\s)\[/g) {
+            my ($where, $prefix) = ($-[1], $1);
+            if ($prefix !~ /$Type\s+$/ &&
+                ($where != 0 || $prefix !~ /^.\s+$/) &&
+                $prefix !~ /[{,]\s+$/) {
+                if (ERROR("BRACKET_SPACE",
+                          "space prohibited before open square bracket '['\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~
+                        s/^(\+.*?)\s+\[/$1\[/;
+                }
+            }
+        }
+
+# Check operator spacing.
+        if (!($line=~/\#\s*include/)) {
+            my $fixed_line = "";
+            my $line_fixed = 0;
+
+            my $ops = qr{
+                                <<=|>>=|<=|>=|==|!=|
+                                \+=|-=|\*=|\/=|%=|\^=|\|=|&=|
+                                =>|->|<<|>>|<|>|=|!|~|
+                                &&|\|\||,|\^|\+\+|--|&|\||\+|-|\*|\/|%|
+                                \?:|\?|:
+                        }x;
+            my @elements = split(/($ops|;)/, $opline);
+
+##                      print("element count: <" . $#elements . ">\n");
+##                      foreach my $el (@elements) {
+##                              print("el: <$el>\n");
+##                      }
+
+            my @fix_elements = ();
+            my $off = 0;
+
+            foreach my $el (@elements) {
+                push(@fix_elements, substr($rawline, $off, length($el)));
+                $off += length($el);
+            }
+
+            $off = 0;
+
+            my $blank = copy_spacing($opline);
+            my $last_after = -1;
+
+            for (my $n = 0; $n < $#elements; $n += 2) {
+
+                my $good = $fix_elements[$n] . $fix_elements[$n + 1];
+
+##                              print("n: <$n> good: <$good>\n");
+
+                $off += length($elements[$n]);
+
+                # Pick up the preceding and succeeding characters.
+                my $ca = substr($opline, 0, $off);
+                my $cc = '';
+                if (length($opline) >= ($off + length($elements[$n + 1]))) {
+                    $cc = substr($opline, $off + length($elements[$n + 1]));
+                }
+                my $cb = "$ca$;$cc";
+
+                my $a = '';
+                $a = 'V' if ($elements[$n] ne '');
+                $a = 'W' if ($elements[$n] =~ /\s$/);
+                $a = 'C' if ($elements[$n] =~ /$;$/);
+                $a = 'B' if ($elements[$n] =~ /(\[|\()$/);
+                $a = 'O' if ($elements[$n] eq '');
+                $a = 'E' if ($ca =~ /^\s*$/);
+
+                my $op = $elements[$n + 1];
+
+                my $c = '';
+                if (defined $elements[$n + 2]) {
+                    $c = 'V' if ($elements[$n + 2] ne '');
+                    $c = 'W' if ($elements[$n + 2] =~ /^\s/);
+                    $c = 'C' if ($elements[$n + 2] =~ /^$;/);
+                    $c = 'B' if ($elements[$n + 2] =~ /^(\)|\]|;)/);
+                    $c = 'O' if ($elements[$n + 2] eq '');
+                    $c = 'E' if ($elements[$n + 2] =~ /^\s*\\$/);
+                } else {
+                    $c = 'E';
+                }
+
+                my $ctx = "${a}x${c}";
+
+                my $at = "(ctx:$ctx)";
+
+                my $ptr = substr($blank, 0, $off) . "^";
+                my $hereptr = "$hereline$ptr\n";
+
+                # Pull out the value of this operator.
+                my $op_type = substr($curr_values, $off + 1, 1);
+
+                # Get the full operator variant.
+                my $opv = $op . substr($curr_vars, $off, 1);
+
+                # Ignore operators passed as parameters.
+                if ($op_type ne 'V' &&
+                    $ca =~ /\s$/ && $cc =~ /^\s*,/) {
+
+#                               # Ignore comments
+#                               } elsif ($op =~ /^$;+$/) {
+
+                    # ; should have either the end of line or a space or \ after it
+                } elsif ($op eq ';') {
+                    if ($ctx !~ /.x[WEBC]/ &&
+                        $cc !~ /^\\/ && $cc !~ /^;/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # // is a comment
+                } elsif ($op eq '//') {
+
+                    #   :   when part of a bitfield
+                } elsif ($opv eq ':B') {
+                    # skip the bitfield test for now
+
+                    # No spaces for:
+                    #   ->
+                } elsif ($op eq '->') {
+                    if ($ctx =~ /Wx.|.xW/) {
+                        if (ERROR("SPACING",
+                                  "spaces prohibited around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # , must have a space on the right.
+                } elsif ($op eq ',') {
+                    if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/) {
+                        if (ERROR("SPACING",
+                                  "space required after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                            $last_after = $n;
+                        }
+                    }
+
+                    # '*' as part of a type definition -- reported already.
+                } elsif ($opv eq '*_') {
+                    #warn "'*' is part of type\n";
+
+                    # unary operators should have a space before and
+                    # none after.  May be left adjacent to another
+                    # unary operator, or a cast
+                } elsif ($op eq '!' || $op eq '~' ||
+                         $opv eq '*U' || $opv eq '-U' ||
+                         $opv eq '&U' || $opv eq '&&U') {
+                    if ($ctx !~ /[WEBC]x./ && $ca !~ /(?:\)|!|~|\*|-|\&|\||\+\+|\-\-|\{)$/) {
+                        if (ERROR("SPACING",
+                                  "space required before that '$op' $at\n" . $hereptr)) {
+                            if ($n != $last_after + 2) {
+                                $good = $fix_elements[$n] . " " . ltrim($fix_elements[$n + 1]);
+                                $line_fixed = 1;
+                            }
+                        }
+                    }
+                    if ($op eq '*' && $cc =~/\s*$Modifier\b/) {
+                        # A unary '*' may be const
+
+                    } elsif ($ctx =~ /.xW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . rtrim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # unary ++ and unary -- are allowed no space on one side.
+                } elsif ($op eq '++' or $op eq '--') {
+                    if ($ctx !~ /[WEOBC]x[^W]/ && $ctx !~ /[^W]x[WOBEC]/) {
+                        if (ERROR("SPACING",
+                                  "space required one side of that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]) . " ";
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /Wx[BE]/ ||
+                        ($ctx =~ /Wx./ && $cc =~ /^;/)) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+                    if ($ctx =~ /ExW/) {
+                        if (ERROR("SPACING",
+                                  "space prohibited after that '$op' $at\n" . $hereptr)) {
+                            $good = $fix_elements[$n] . trim($fix_elements[$n + 1]);
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # << and >> may either have or not have spaces both sides
+                } elsif ($op eq '<<' or $op eq '>>' or
+                         $op eq '&' or $op eq '^' or $op eq '|' or
+                         $op eq '+' or $op eq '-' or
+                         $op eq '*' or $op eq '/' or
+                         $op eq '%')
+                {
+                    if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
+                        if (ERROR("SPACING",
+                                  "need consistent spacing around '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # A colon needs no spaces before when it is
+                    # terminating a case value or a label.
+                } elsif ($opv eq ':C' || $opv eq ':L') {
+                    if ($ctx =~ /Wx./) {
+                        if (ERROR("SPACING",
+                                  "space prohibited before that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]);
+                            $line_fixed = 1;
+                        }
+                    }
+
+                    # All the others need spaces both sides.
+                } elsif ($ctx !~ /[EWC]x[CWE]/) {
+                    my $ok = 0;
+
+                    # Ignore email addresses <foo@bar>
+                    if (($op eq '<' &&
+                         $cc =~ /^\S+\@\S+>/) ||
+                        ($op eq '>' &&
+                         $ca =~ /<\S+\@\S+$/))
+                    {
+                        $ok = 1;
+                    }
+
+                    # messages are ERROR, but ?: are CHK
+                    if ($ok == 0) {
+                        my $msg_type = \&ERROR;
+                        $msg_type = \&CHK if (($op eq '?:' || $op eq '?' || $op eq ':') && $ctx =~ /VxV/);
+
+                        if (&{$msg_type}("SPACING",
+                                         "spaces required around that '$op' $at\n" . $hereptr)) {
+                            $good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
+                            if (defined $fix_elements[$n + 2]) {
+                                $fix_elements[$n + 2] =~ s/^\s+//;
+                            }
+                            $line_fixed = 1;
+                        }
+                    }
+                }
+                $off += length($elements[$n + 1]);
+
+##                              print("n: <$n> GOOD: <$good>\n");
+
+                $fixed_line = $fixed_line . $good;
+            }
+
+            if (($#elements % 2) == 0) {
+                $fixed_line = $fixed_line . $fix_elements[$#elements];
+            }
+
+            if ($fix && $line_fixed && $fixed_line ne $fixed[$linenr - 1]) {
+                $fixed[$linenr - 1] = $fixed_line;
+            }
+
+
+        }
+
+# check for whitespace before a non-naked semicolon
+        if ($line =~ /^\+.*\S\s+;\s*$/) {
+            if (WARN("SPACING",
+                     "space prohibited before semicolon\n" . $herecurr) &&
+                $fix) {
+                1 while $fixed[$linenr - 1] =~
+                    s/^(\+.*\S)\s+;/$1;/;
+            }
+        }
+
+# check for multiple assignments
+        if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
+            CHK("MULTIPLE_ASSIGNMENTS",
+                "multiple assignments should be avoided\n" . $herecurr);
+        }
+
+## # check for multiple declarations, allowing for a function declaration
+## # continuation.
+##              if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
+##                  $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
+##
+##                      # Remove any bracketed sections to ensure we do not
+##                      # falsly report the parameters of functions.
+##                      my $ln = $line;
+##                      while ($ln =~ s/\([^\(\)]*\)//g) {
+##                      }
+##                      if ($ln =~ /,/) {
+##                              WARN("MULTIPLE_DECLARATION",
+##                                   "declaring multiple variables together should be avoided\n" . $herecurr);
+##                      }
+##              }
+
+#need space before brace following if, while, etc
+        if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+            $line =~ /do\{/) {
+            if (ERROR("SPACING",
+                      "space required before the open brace '{'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\)))\{/$1 {/;
+            }
+        }
+
+## # check for blank lines before declarations
+##              if ($line =~ /^.\t+$Type\s+$Ident(?:\s*=.*)?;/ &&
+##                  $prevrawline =~ /^.\s*$/) {
+##                      WARN("SPACING",
+##                           "No blank lines before declarations\n" . $hereprev);
+##              }
+##
+
+# closing brace should have a space following it when it has anything
+# on the line
+        if ($line =~ /}(?!(?:,|;|\)))\S/) {
+            if (ERROR("SPACING",
+                      "space required after that close brace '}'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/}((?!(?:,|;|\)))\S)/} $1/;
+            }
+        }
+
+# check spacing on square brackets
+        if ($line =~ /\[\s/ && $line !~ /\[\s*$/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open square bracket '['\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\[\s+/\[/;
+            }
+        }
+        if ($line =~ /\s\]/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close square bracket ']'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\]/\]/;
+            }
+        }
+
+# check spacing on parentheses
+        if ($line =~ /\(\s/ && $line !~ /\(\s*(?:\\)?$/ &&
+            $line !~ /for\s*\(\s+;/) {
+            if (ERROR("SPACING",
+                      "space prohibited after that open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\(\s+/\(/;
+            }
+        }
+        if ($line =~ /(\s+)\)/ && $line !~ /^.\s*\)/ &&
+            $line !~ /for\s*\(.*;\s+\)/ &&
+            $line !~ /:\s+\)/) {
+            if (ERROR("SPACING",
+                      "space prohibited before that close parenthesis ')'\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\s+\)/\)/;
+            }
+        }
+
+#goto labels aren't indented, allow a single space however
+        if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and
+            !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) {
+            if (WARN("INDENTED_LABEL",
+                     "labels should not be indented\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.)\s+/$1/;
+            }
+        }
+
+# return is not a function
+        if (defined($stat) && $stat =~ /^.\s*return(\s*)\(/s) {
+            my $spacing = $1;
+            if ($^V && $^V ge 5.10.0 &&
+                $stat =~ /^.\s*return\s*($balanced_parens)\s*;\s*$/) {
+                my $value = $1;
+                $value = deparenthesize($value);
+                if ($value =~ m/^\s*$FuncArg\s*(?:\?|$)/) {
+                    ERROR("RETURN_PARENTHESES",
+                          "return is not a function, parentheses are not required\n" . $herecurr);
+                }
+            } elsif ($spacing !~ /\s+/) {
+                ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr);
+            }
+        }
+
+# unnecessary return in a void function
+# at end-of-function, with the previous line a single leading tab, then return;
+# and the line before that not a goto label target like "out:"
+        if ($sline =~ /^[ \+]}\s*$/ &&
+            $prevline =~ /^\+\treturn\s*;\s*$/ &&
+            $linenr >= 3 &&
+            $lines[$linenr - 3] =~ /^[ +]/ &&
+            $lines[$linenr - 3] !~ /^[ +]\s*$Ident\s*:/) {
+            WARN("RETURN_VOID",
+                 "void function return statements are not generally useful\n" . $hereprev);
+        }
+
+# if statements using unnecessary parentheses - ie: if ((foo == bar))
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\bif\s*((?:\(\s*){2,})/) {
+            my $openparens = $1;
+            my $count = $openparens =~ tr@\(@\(@;
+            my $msg = "";
+            if ($line =~ /\bif\s*(?:\(\s*){$count,$count}$LvalOrFunc\s*($Compare)\s*$LvalOrFunc(?:\s*\)){$count,$count}/) {
+                my $comp = $4;  #Not $1 because of $LvalOrFunc
+                $msg = " - maybe == should be = ?" if ($comp eq "==");
+                WARN("UNNECESSARY_PARENTHESES",
+                     "Unnecessary parentheses$msg\n" . $herecurr);
+            }
+        }
+
+# Return of what appears to be an errno should normally be -'ve
+        if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
+            my $name = $1;
+            if ($name ne 'EOF' && $name ne 'ERROR') {
+                WARN("USE_NEGATIVE_ERRNO",
+                     "return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+            }
+        }
+
+# Need a space before open parenthesis after if, while etc
+        if ($line =~ /\b(if|while|for|switch)\(/) {
+            if (ERROR("SPACING",
+                      "space required before the open parenthesis '('\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/\b(if|while|for|switch)\(/$1 \(/;
+            }
+        }
+
+# Check for illegal assignment in if conditional -- and check for trailing
+# statements after the conditional.
+        if ($line =~ /do\s*(?!{)/) {
+            ($stat, $cond, $line_nr_next, $remain_next, $off_next) =
+                ctx_statement_block($linenr, $realcnt, 0)
+                if (!defined $stat);
+            my ($stat_next) = ctx_statement_block($line_nr_next,
+                                                  $remain_next, $off_next);
+            $stat_next =~ s/\n./\n /g;
+            ##print "stat<$stat> stat_next<$stat_next>\n";
+
+            if ($stat_next =~ /^\s*while\b/) {
+                # If the statement carries leading newlines,
+                # then count those as offsets.
+                my ($whitespace) =
+                    ($stat_next =~ /^((?:\s*\n[+-])*\s*)/s);
+                my $offset =
+                    statement_rawlines($whitespace) - 1;
+
+                $suppress_whiletrailers{$line_nr_next +
+                                            $offset} = 1;
+            }
+        }
+        if (!defined $suppress_whiletrailers{$linenr} &&
+            defined($stat) && defined($cond) &&
+            $line =~ /\b(?:if|while|for)\s*\(/ && $line !~ /^.\s*#/) {
+            my ($s, $c) = ($stat, $cond);
+
+            if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) {
+                ERROR("ASSIGN_IN_IF",
+                      "do not use assignment in if condition\n" . $herecurr);
+            }
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+            $s =~ s/$;//g;      # Remove any comments
+            if (length($c) && $s !~ /^\s*{?\s*\\*\s*$/ &&
+                $c !~ /}\s*while\s*/)
+            {
+                # Find out how long the conditional actually is.
+                my @newlines = ($c =~ /\n/gs);
+                my $cond_lines = 1 + $#newlines;
+                my $stat_real = '';
+
+                $stat_real = raw_line($linenr, $cond_lines)
+                    . "\n" if ($cond_lines);
+                if (defined($stat_real) && $cond_lines > 1) {
+                    $stat_real = "[...]\n$stat_real";
+                }
+
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr . $stat_real);
+            }
+        }
+
+# Check for bitwise tests written as boolean
+        if ($line =~ /
+                        (?:
+                                (?:\[|\(|\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\|)
+                        |
+                                (?:\&\&|\|\|)
+                                \s*0[xX][0-9]+\s*
+                                (?:\&\&|\|\||\)|\])
+                        )/x)
+        {
+            WARN("HEXADECIMAL_BOOLEAN_TEST",
+                 "boolean test with hexadecimal, perhaps just 1 \& or \|?\n" . $herecurr);
+        }
+
+# if and else should not have general statements after it
+        if ($line =~ /^.\s*(?:}\s*)?else\b(.*)/) {
+            my $s = $1;
+            $s =~ s/$;//g;      # Remove any comments
+            if ($s !~ /^\s*(?:\sif|(?:{|)\s*\\?\s*$)/) {
+                ERROR("TRAILING_STATEMENTS",
+                      "trailing statements should be on next line\n" . $herecurr);
+            }
+        }
+# if should not continue a brace
+        if ($line =~ /}\s*if\b/) {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" .
+                  $herecurr);
+        }
+# case and default should not have general statements after them
+        if ($line =~ /^.\s*(?:case\s*.*|default\s*):/g &&
+            $line !~ /\G(?:
+                        (?:\s*$;*)(?:\s*{)?(?:\s*$;*)(?:\s*\\)?\s*$|
+                        \s*return\s+
+                    )/xg)
+        {
+            ERROR("TRAILING_STATEMENTS",
+                  "trailing statements should be on next line\n" . $herecurr);
+        }
+
+        # Check for }<nl>else {, these must be at the same
+        # indent level to be relevant to each other.
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*else\s*/ and
+            $previndent == $indent) {
+            ERROR("ELSE_AFTER_BRACE",
+                  "else should follow close brace '}'\n" . $hereprev);
+        }
+
+        if ($prevline=~/}\s*$/ and $line=~/^.\s*while\s*/ and
+            $previndent == $indent) {
+            my ($s, $c) = ctx_statement_block($linenr, $realcnt, 0);
+
+            # Find out what is on the end of the line after the
+            # conditional.
+            substr($s, 0, length($c), '');
+            $s =~ s/\n.*//g;
+
+            if ($s =~ /^\s*;/) {
+                ERROR("WHILE_AFTER_BRACE",
+                      "while should follow close brace '}'\n" . $hereprev);
+            }
+        }
+
+#Specific variable tests
+        while ($line =~ m{($Constant|$Lval)}g) {
+            my $var = $1;
+
+#gcc binary extension
+            if ($var =~ /^$Binary$/) {
+                if (WARN("GCC_BINARY_CONSTANT",
+                         "Avoid gcc v4.3+ binary constant extension: <$var>\n" . $herecurr) &&
+                    $fix) {
+                    my $hexval = sprintf("0x%x", oct($var));
+                    $fixed[$linenr - 1] =~
+                        s/\b$var\b/$hexval/;
+                }
+            }
+
+#CamelCase
+            if ($var !~ /^$Constant$/ &&
+                $var =~ /[A-Z][a-z]|[a-z][A-Z]/ &&
+#Ignore Page<foo> variants
+                $var !~ /^(?:Clear|Set|TestClear|TestSet|)Page[A-Z]/ &&
+#Ignore SI style variants like nS, mV and dB (ie: max_uV, regulator_min_uA_show)
+                $var !~ /^(?:[a-z_]*?)_?[a-z][A-Z](?:_[a-z_]+)?$/) {
+                while ($var =~ m{($Ident)}g) {
+                    my $word = $1;
+                    next if ($word !~ /[A-Z][a-z]|[a-z][A-Z]/);
+                    if ($check) {
+                        seed_camelcase_includes();
+                        if (!$file && !$camelcase_file_seeded) {
+                            seed_camelcase_file($realfile);
+                            $camelcase_file_seeded = 1;
+                        }
+                    }
+                    if (!defined $camelcase{$word}) {
+                        $camelcase{$word} = 1;
+                        CHK("CAMELCASE",
+                            "Avoid CamelCase: <$word>\n" . $herecurr);
+                    }
+                }
+            }
+        }
+
+#no spaces allowed after \ in define
+        if ($line =~ /\#\s*define.*\\\s+$/) {
+            if (WARN("WHITESPACE_AFTER_LINE_CONTINUATION",
+                     "Whitespace after \\ makes next lines useless\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\s+$//;
+            }
+        }
+
+#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
+        if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
+            my $file = "$1.h";
+            my $checkfile = "include/linux/$file";
+            if (-f "$root/$checkfile" &&
+                $realfile ne $checkfile &&
+                $1 !~ /$allowed_asm_includes/)
+            {
+                if ($realfile =~ m{^arch/}) {
+                    CHK("ARCH_INCLUDE_LINUX",
+                        "Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                } else {
+                    WARN("INCLUDE_LINUX",
+                         "Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
+                }
+            }
+        }
+
+# multi-statement macros should be enclosed in a do while loop, grab the
+# first statement and ensure its the whole macro if its not enclosed
+# in a known good container
+        if ($realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s*$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+            #print "dstat<$dstat> dcond<$dcond> cnt<$cnt> off<$off>\n";
+            #print "LINE<$lines[$ln-1]> len<" . length($lines[$ln-1]) . "\n";
+
+            $dstat =~ s/^.\s*\#\s*define\s+$Ident(?:\([^\)]*\))?\s*//;
+            $dstat =~ s/$;//g;
+            $dstat =~ s/\\\n.//g;
+            $dstat =~ s/^\s*//s;
+            $dstat =~ s/\s*$//s;
+
+            # Flatten any parentheses and braces
+            while ($dstat =~ s/\([^\(\)]*\)/1/ ||
+                   $dstat =~ s/\{[^\{\}]*\}/1/ ||
+                   $dstat =~ s/\[[^\[\]]*\]/1/)
+            {
+            }
+
+            # Flatten any obvious string concatentation.
+            while ($dstat =~ s/("X*")\s*$Ident/$1/ ||
+                   $dstat =~ s/$Ident\s*("X*")/$1/)
+            {
+            }
+
+            my $exceptions = qr{
+                                $Declare|
+                                module_param_named|
+                                MODULE_PARM_DESC|
+                                DECLARE_PER_CPU|
+                                DEFINE_PER_CPU|
+                                __typeof__\(|
+                                union|
+                                struct|
+                                \.$Ident\s*=\s*|
+                                ^\"|\"$
+                        }x;
+            #print "REST<$rest> dstat<$dstat> ctx<$ctx>\n";
+            if ($dstat ne '' &&
+                $dstat !~ /^(?:$Ident|-?$Constant),$/ &&                        # 10, // foo(),
+                $dstat !~ /^(?:$Ident|-?$Constant);$/ &&                        # foo();
+                $dstat !~ /^[!~-]?(?:$Lval|$Constant)$/ &&              # 10 // foo() // !foo // ~foo // -foo // foo->bar // foo.bar->baz
+                $dstat !~ /^'X'$/ &&                                    # character constants
+                $dstat !~ /$exceptions/ &&
+                $dstat !~ /^\.$Ident\s*=/ &&                            # .foo =
+                $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ &&              # stringification #foo
+                $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ &&   # do {...} while (...); // do {...} while (...)
+                $dstat !~ /^for\s*$Constant$/ &&                                # for (...)
+                $dstat !~ /^for\s*$Constant\s+(?:$Ident|-?$Constant)$/ &&       # for (...) bar()
+                $dstat !~ /^do\s*{/ &&                                  # do {...
+                $dstat !~ /^\(\{/ &&                                             # ({...
+                $ctx !~ /^.\s*#\s*define\s+TRACE_(?:SYSTEM|INCLUDE_FILE|INCLUDE_PATH)\b/)
+            {
+                $ctx =~ s/\n*$//;
+                my $herectx = $here . "\n";
+                my $cnt = statement_rawlines($ctx);
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if ($dstat =~ /;/) {
+                    ERROR("MULTISTATEMENT_MACRO_USE_DO_WHILE",
+                          "Macros with multiple statements should be enclosed in a do - while loop\n" . "$herectx");
+                } else {
+                    ERROR("COMPLEX_MACRO",
+                          "Macros with complex values should be enclosed in parenthesis\n" . "$herectx");
+                }
+            }
+
+# check for line continuations outside of #defines, preprocessor #, and asm
+
+        } else {
+            if ($prevline !~ /^..*\\$/ &&
+                $line !~ /^\+\s*\#.*\\$/ &&             # preprocessor
+                $line !~ /^\+.*\b(__asm__|asm)\b.*\\$/ &&       # asm
+                $line =~ /^\+.*\\$/) {
+                WARN("LINE_CONTINUATIONS",
+                     "Avoid unnecessary line continuations\n" . $herecurr);
+            }
+        }
+
+# do {} while (0) macro tests:
+# single-statement macros do not need to be enclosed in do while (0) loop,
+# macro should not end with a semicolon
+        if ($^V && $^V ge 5.10.0 &&
+            $realfile !~ m@/vmlinux.lds.h$@ &&
+            $line =~ /^.\s*\#\s*define\s+$Ident(\()?/) {
+            my $ln = $linenr;
+            my $cnt = $realcnt;
+            my ($off, $dstat, $dcond, $rest);
+            my $ctx = '';
+            ($dstat, $dcond, $ln, $cnt, $off) =
+                ctx_statement_block($linenr, $realcnt, 0);
+            $ctx = $dstat;
+
+            $dstat =~ s/\\\n.//g;
+
+            if ($dstat =~ /^\+\s*#\s*define\s+$Ident\s*${balanced_parens}\s*do\s*{(.*)\s*}\s*while\s*\(\s*0\s*\)\s*([;\s]*)\s*$/) {
+                my $stmts = $2;
+                my $semis = $3;
+
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                if (($stmts =~ tr/;/;/) == 1 &&
+                    $stmts !~ /^\s*(if|while|for|switch)\b/) {
+                    WARN("SINGLE_STATEMENT_DO_WHILE_MACRO",
+                         "Single statement macros should not use a do {} while (0) loop\n" . "$herectx");
+                }
+                if (defined $semis && $semis ne "") {
+                    WARN("DO_WHILE_MACRO_WITH_TRAILING_SEMICOLON",
+                         "do {} while (0) macros should not be semicolon terminated\n" . "$herectx");
+                }
+            } elsif ($dstat =~ /^\+\s*#\s*define\s+$Ident.*;\s*$/) {
+                $ctx =~ s/\n*$//;
+                my $cnt = statement_rawlines($ctx);
+                my $herectx = $here . "\n";
+
+                for (my $n = 0; $n < $cnt; $n++) {
+                    $herectx .= raw_line($linenr, $n) . "\n";
+                }
+
+                WARN("TRAILING_SEMICOLON",
+                     "macros should not use a trailing semicolon\n" . "$herectx");
+            }
+        }
+
+# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
+# all assignments may have only one of the following with an assignment:
+#       .
+#       ALIGN(...)
+#       VMLINUX_SYMBOL(...)
+        if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
+            WARN("MISSING_VMLINUX_SYMBOL",
+                 "vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
+        }
+
+# check for redundant bracing round if etc
+        if ($line =~ /(^.*)\bif\b/ && $1 !~ /else\s*$/) {
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, 1);
+            #print "chunks<$#chunks> linenr<$linenr> endln<$endln> level<$level>\n";
+            #print "APW: <<$chunks[1][0]>><<$chunks[1][1]>>\n";
+            if ($#chunks > 0 && $level == 0) {
+                my @allowed = ();
+                my $allow = 0;
+                my $seen = 0;
+                my $herectx = $here . "\n";
+                my $ln = $linenr - 1;
+                for my $chunk (@chunks) {
+                    my ($cond, $block) = @{$chunk};
+
+                    # If the condition carries leading newlines, then count those as offsets.
+                    my ($whitespace) = ($cond =~ /^((?:\s*\n[+-])*\s*)/s);
+                    my $offset = statement_rawlines($whitespace) - 1;
+
+                    $allowed[$allow] = 0;
+                    #print "COND<$cond> whitespace<$whitespace> offset<$offset>\n";
+
+                    # We have looked at and allowed this specific line.
+                    $suppress_ifbraces{$ln + $offset} = 1;
+
+                    $herectx .= "$rawlines[$ln + $offset]\n[...]\n";
+                    $ln += statement_rawlines($block) - 1;
+
+                    substr($block, 0, length($cond), '');
+
+                    $seen++ if ($block =~ /^\s*{/);
+
+                    #print "cond<$cond> block<$block> allowed<$allowed[$allow]>\n";
+                    if (statement_lines($cond) > 1) {
+                        #print "APW: ALLOWED: cond<$cond>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if ($block =~/\b(?:if|for|while)\b/) {
+                        #print "APW: ALLOWED: block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    if (statement_block_size($block) > 1) {
+                        #print "APW: ALLOWED: lines block<$block>\n";
+                        $allowed[$allow] = 1;
+                    }
+                    $allow++;
+                }
+                if ($seen) {
+                    my $sum_allowed = 0;
+                    foreach (@allowed) {
+                        $sum_allowed += $_;
+                    }
+                    if ($sum_allowed != 0 && $sum_allowed != $allow
+		    			  && $seen != $allow) {
+                        CHK("BRACES",
+                            "braces {} should be used on all arms of this statement\n" . $herectx);
+                    }
+                }
+            }
+        }
+        if (!defined $suppress_ifbraces{$linenr - 1} &&
+            $line =~ /\b(if|while|for|else)\b/) {
+            my $allowed = 0;
+
+            # Check the pre-context.
+            if (substr($line, 0, $-[0]) =~ /(\}\s*)$/) {
+                #print "APW: ALLOWED: pre<$1>\n";
+                $allowed = 1;
+            }
+
+            my ($level, $endln, @chunks) =
+                ctx_statement_full($linenr, $realcnt, $-[0]);
+
+            # Check the condition.
+            my ($cond, $block) = @{$chunks[0]};
+            #print "CHECKING<$linenr> cond<$cond> block<$block>\n";
+            if (defined $cond) {
+                substr($block, 0, length($cond), '');
+            }
+            if (statement_lines($cond) > 1) {
+                #print "APW: ALLOWED: cond<$cond>\n";
+                $allowed = 1;
+            }
+            if ($block =~/\b(?:if|for|while)\b/) {
+                #print "APW: ALLOWED: block<$block>\n";
+                $allowed = 1;
+            }
+            if (statement_block_size($block) > 1) {
+                #print "APW: ALLOWED: lines block<$block>\n";
+                $allowed = 1;
+            }
+            # Check the post-context.
+            if (defined $chunks[1]) {
+                my ($cond, $block) = @{$chunks[1]};
+                if (defined $cond) {
+                    substr($block, 0, length($cond), '');
+                }
+                if ($block =~ /^\s*\{/) {
+                    #print "APW: ALLOWED: chunk-1 block<$block>\n";
+                    $allowed = 1;
+                }
+            }
+        }
+
+# check for unnecessary blank lines around braces
+        if (($line =~ /^.\s*}\s*$/ && $prevrawline =~ /^.\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary before a close brace '}'\n" . $hereprev);
+        }
+        if (($rawline =~ /^.\s*$/ && $prevline =~ /^..*{\s*$/)) {
+            CHK("BRACES",
+                "Blank lines aren't necessary after an open brace '{'\n" . $hereprev);
+        }
+
+# no volatiles please
+        my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
+        if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
+            WARN("VOLATILE",
+                 "Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
+        }
+
+# warn about #if 0
+        if ($line =~ /^.\s*\#\s*if\s+0\b/) {
+            CHK("REDUNDANT_CODE",
+                "if this code is redundant consider removing it\n" .
+                $herecurr);
+        }
+
+# check for needless "if (<foo>) fn(<foo>)" uses
+        if ($prevline =~ /\bif\s*\(\s*($Lval)\s*\)/) {
+            my $expr = '\s*\(\s*' . quotemeta($1) . '\s*\)\s*;';
+            if ($line =~ /\b(kfree|usb_free_urb|debugfs_remove(?:_recursive)?)$expr/) {
+                WARN('NEEDLESS_IF',
+                     "$1(NULL) is safe this check is probably not required\n" . $hereprev);
+            }
+        }
+
+# check for bad placement of section $InitAttribute (e.g.: __initdata)
+        if ($line =~ /(\b$InitAttribute\b)/) {
+            my $attr = $1;
+            if ($line =~ /^\+\s*static\s+(?:const\s+)?(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*[=;]/) {
+                my $ptr = $1;
+                my $var = $2;
+                if ((($ptr =~ /\b(union|struct)\s+$attr\b/ &&
+                      ERROR("MISPLACED_INIT",
+                            "$attr should be placed after $var\n" . $herecurr)) ||
+                     ($ptr !~ /\b(union|struct)\s+$attr\b/ &&
+                      WARN("MISPLACED_INIT",
+                           "$attr should be placed after $var\n" . $herecurr))) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/(\bstatic\s+(?:const\s+)?)(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*([=;])\s*/"$1" . trim(string_find_replace($2, "\\s*$attr\\s*", " ")) . " " . trim(string_find_replace($3, "\\s*$attr\\s*", "")) . " $attr" . ("$4" eq ";" ? ";" : " = ")/e;
+                }
+            }
+        }
+
+# check for $InitAttributeData (ie: __initdata) with const
+        if ($line =~ /\bconst\b/ && $line =~ /($InitAttributeData)/) {
+            my $attr = $1;
+            $attr =~ /($InitAttributePrefix)(.*)/;
+            my $attr_prefix = $1;
+            my $attr_type = $2;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of const init definition must use ${attr_prefix}initconst\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/$InitAttributeData/${attr_prefix}initconst/;
+            }
+        }
+
+# check for $InitAttributeConst (ie: __initconst) without const
+        if ($line !~ /\bconst\b/ && $line =~ /($InitAttributeConst)/) {
+            my $attr = $1;
+            if (ERROR("INIT_ATTRIBUTE",
+                      "Use of $attr requires a separate use of const\n" . $herecurr) &&
+                $fix) {
+                my $lead = $fixed[$linenr - 1] =~
+                    /(^\+\s*(?:static\s+))/;
+                $lead = rtrim($1);
+                $lead = "$lead " if ($lead !~ /^\+$/);
+                $lead = "${lead}const ";
+                $fixed[$linenr - 1] =~ s/(^\+\s*(?:static\s+))/$lead/;
+            }
+        }
+
+# don't use __constant_<foo> functions outside of include/uapi/
+        if ($realfile !~ m@^include/uapi/@ &&
+            $line =~ /(__constant_(?:htons|ntohs|[bl]e(?:16|32|64)_to_cpu|cpu_to_[bl]e(?:16|32|64)))\s*\(/) {
+            my $constant_func = $1;
+            my $func = $constant_func;
+            $func =~ s/^__constant_//;
+            if (WARN("CONSTANT_CONVERSION",
+                     "$constant_func should be $func\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b$constant_func\b/$func/g;
+            }
+        }
+
+# prefer usleep_range over udelay
+        if ($line =~ /\budelay\s*\(\s*(\d+)\s*\)/) {
+            my $delay = $1;
+            # ignore udelay's < 10, however
+            if (! ($delay < 10) ) {
+                CHK("USLEEP_RANGE",
+                    "usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+            if ($delay > 2000) {
+                WARN("LONG_UDELAY",
+                     "long udelay - prefer mdelay; see arch/arm/include/asm/delay.h\n" . $herecurr);
+            }
+        }
+
+# warn about unexpectedly long msleep's
+        if ($line =~ /\bmsleep\s*\((\d+)\);/) {
+            if ($1 < 20) {
+                WARN("MSLEEP",
+                     "msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $herecurr);
+            }
+        }
+
+# check for comparisons of jiffies
+        if ($line =~ /\bjiffies\s*$Compare|$Compare\s*jiffies\b/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing jiffies is almost always wrong; prefer time_after, time_before and friends\n" . $herecurr);
+        }
+
+# check for comparisons of get_jiffies_64()
+        if ($line =~ /\bget_jiffies_64\s*\(\s*\)\s*$Compare|$Compare\s*get_jiffies_64\s*\(\s*\)/) {
+            WARN("JIFFIES_COMPARISON",
+                 "Comparing get_jiffies_64() is almost always wrong; prefer time_after64, time_before64 and friends\n" . $herecurr);
+        }
+
+# warn about spacing in #ifdefs
+        if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
+            if (ERROR("SPACING",
+                      "exactly one space required after that #$1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~
+                    s/^(.\s*\#\s*(ifdef|ifndef|elif))\s{2,}/$1 /;
+            }
+
+        }
+
+# check for spinlock_t definitions without a comment.
+        if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
+            $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
+            my $which = $1;
+            if (!ctx_has_comment($first_line, $linenr)) {
+                CHK("UNCOMMENTED_DEFINITION",
+                    "$1 definition without comment\n" . $herecurr);
+            }
+        }
+# check for memory barriers without a comment.
+        if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+            if (!ctx_has_comment($first_line, $linenr)) {
+                WARN("MEMORY_BARRIER",
+                     "memory barrier without comment\n" . $herecurr);
+            }
+        }
+# check of hardware specific defines
+        if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
+            CHK("ARCH_DEFINES",
+                "architecture specific defines should be avoided\n" .  $herecurr);
+        }
+
+# Check that the storage class is at the beginning of a declaration
+        if ($line =~ /\b$Storage\b/ && $line !~ /^.\s*$Storage\b/) {
+            WARN("STORAGE_CLASS",
+                 "storage class should be at the beginning of the declaration\n" . $herecurr)
+        }
+
+# check the location of the inline attribute, that it is between
+# storage class and type.
+        if ($line =~ /\b$Type\s+$Inline\b/ ||
+            $line =~ /\b$Inline\s+$Storage\b/) {
+            ERROR("INLINE_LOCATION",
+                  "inline keyword should sit between storage class and type\n" . $herecurr);
+        }
+
+# Check for __inline__ and __inline, prefer inline
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b(__inline__|__inline)\b/) {
+            if (WARN("INLINE",
+                     "plain inline is preferred over $1\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b(__inline__|__inline)\b/inline/;
+
+            }
+        }
+
+# Check for __attribute__ packed, prefer __packed
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*\bpacked\b/) {
+            WARN("PREFER_PACKED",
+                 "__packed is preferred over __attribute__((packed))\n" . $herecurr);
+        }
+
+# Check for __attribute__ aligned, prefer __aligned
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(.*aligned/) {
+            WARN("PREFER_ALIGNED",
+                 "__aligned(size) is preferred over __attribute__((aligned(size)))\n" . $herecurr);
+        }
+
+# Check for __attribute__ format(printf, prefer __printf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf/) {
+            if (WARN("PREFER_PRINTF",
+                     "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex;
+
+            }
+        }
+
+# Check for __attribute__ format(scanf, prefer __scanf
+        if ($realfile !~ m@\binclude/uapi/@ &&
+            $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\b/) {
+            if (WARN("PREFER_SCANF",
+                     "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for sizeof(&)
+        if ($line =~ /\bsizeof\s*\(\s*\&/) {
+            WARN("SIZEOF_ADDRESS",
+                 "sizeof(& should be avoided\n" . $herecurr);
+        }
+
+# check for sizeof without parenthesis
+        if ($line =~ /\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/) {
+            if (WARN("SIZEOF_PARENTHESIS",
+                     "sizeof $1 should be sizeof($1)\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/"sizeof(" . trim($1) . ")"/ex;
+            }
+        }
+
+# check for line continuations in quoted strings with odd counts of "
+        if ($rawline =~ /\\$/ && $rawline =~ tr/"/"/ % 2) {
+            WARN("LINE_CONTINUATIONS",
+                 "Avoid line continuations in quoted strings\n" . $herecurr);
+        }
+
+# check for struct spinlock declarations
+        if ($line =~ /^.\s*\bstruct\s+spinlock\s+\w+\s*;/) {
+            WARN("USE_SPINLOCK_T",
+                 "struct spinlock should be spinlock_t\n" . $herecurr);
+        }
+
+# check for seq_printf uses that could be seq_puts
+        if ($sline =~ /\bseq_printf\s*\(.*"\s*\)\s*;\s*$/) {
+            my $fmt = get_quoted_string($line, $rawline);
+            if ($fmt ne "" && $fmt !~ /[^\\]\%/) {
+                if (WARN("PREFER_SEQ_PUTS",
+                         "Prefer seq_puts to seq_printf\n" . $herecurr) &&
+                    $fix) {
+                    $fixed[$linenr - 1] =~ s/\bseq_printf\b/seq_puts/;
+                }
+            }
+        }
+
+# Check for misused memsets
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\bmemset\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\,\s*$FuncArg\s*\)/s) {
+
+            my $ms_addr = $2;
+            my $ms_val = $7;
+            my $ms_size = $12;
+
+            if ($ms_size =~ /^(0x|)0$/i) {
+                ERROR("MEMSET",
+                      "memset to 0's uses 0 as the 2nd argument, not the 3rd\n" . "$here\n$stat\n");
+            } elsif ($ms_size =~ /^(0x|)1$/i) {
+                WARN("MEMSET",
+                     "single byte memset is suspicious. Swapped 2nd/3rd argument?\n" . "$here\n$stat\n");
+            }
+        }
+
+# typecasts on min/max could be min_t/max_t
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\b(min|max)\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\)/) {
+            if (defined $2 || defined $7) {
+                my $call = $1;
+                my $cast1 = deparenthesize($2);
+                my $arg1 = $3;
+                my $cast2 = deparenthesize($7);
+                my $arg2 = $8;
+                my $cast;
+
+                if ($cast1 ne "" && $cast2 ne "" && $cast1 ne $cast2) {
+                    $cast = "$cast1 or $cast2";
+                } elsif ($cast1 ne "") {
+                    $cast = $cast1;
+                } else {
+                    $cast = $cast2;
+                }
+                WARN("MINMAX",
+                     "$call() should probably be ${call}_t($cast, $arg1, $arg2)\n" . "$here\n$stat\n");
+            }
+        }
+
+# check usleep_range arguments
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+(?:.*?)\busleep_range\s*\(\s*($FuncArg)\s*,\s*($FuncArg)\s*\)/) {
+            my $min = $1;
+            my $max = $7;
+            if ($min eq $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range should not use min == max args; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            } elsif ($min =~ /^\d+$/ && $max =~ /^\d+$/ &&
+                     $min > $max) {
+                WARN("USLEEP_RANGE",
+                     "usleep_range args reversed, use min then max; see Documentation/timers/timers-howto.txt\n" . "$here\n$stat\n");
+            }
+        }
+
+# check for naked sscanf
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/ &&
+            ($stat !~ /$Ident\s*=\s*sscanf\s*$balanced_parens/ &&
+             $stat !~ /\bsscanf\s*$balanced_parens\s*(?:$Compare)/ &&
+             $stat !~ /(?:$Compare)\s*\bsscanf\s*$balanced_parens/)) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            WARN("NAKED_SSCANF",
+                 "unchecked sscanf return value\n" . "$here\n$stat_real\n");
+        }
+
+# check for simple sscanf that should be kstrto<foo>
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $line =~ /\bsscanf\b/) {
+            my $lc = $stat =~ tr@\n@@;
+            $lc = $lc + $linenr;
+            my $stat_real = raw_line($linenr, 0);
+            for (my $count = $linenr + 1; $count <= $lc; $count++) {
+                $stat_real = $stat_real . "\n" . raw_line($count, 0);
+            }
+            if ($stat_real =~ /\bsscanf\b\s*\(\s*$FuncArg\s*,\s*("[^"]+")/) {
+                my $format = $6;
+                my $count = $format =~ tr@%@%@;
+                if ($count == 1 &&
+                    $format =~ /^"\%(?i:ll[udxi]|[udxi]ll|ll|[hl]h?[udxi]|[udxi][hl]h?|[hl]h?|[udxi])"$/) {
+                    WARN("SSCANF_TO_KSTRTO",
+                         "Prefer kstrto<type> to single variable sscanf\n" . "$here\n$stat_real\n");
+                }
+            }
+        }
+
+# check for new externs in .h files.
+        if ($realfile =~ /\.h$/ &&
+            $line =~ /^\+\s*(extern\s+)$Type\s*$Ident\s*\(/s) {
+            if (CHK("AVOID_EXTERNS",
+                    "extern prototypes should be avoided in .h files\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(.*)\bextern\b\s*(.*)/$1$2/;
+            }
+        }
+
+# check for new externs in .c files.
+        if ($realfile =~ /\.c$/ && defined $stat &&
+            $stat =~ /^.\s*(?:extern\s+)?$Type\s+($Ident)(\s*)\(/s)
+        {
+            my $function_name = $1;
+            my $paren_space = $2;
+
+            my $s = $stat;
+            if (defined $cond) {
+                substr($s, 0, length($cond), '');
+            }
+            if ($s =~ /^\s*;/ &&
+                $function_name ne 'uninitialized_var')
+            {
+                WARN("AVOID_EXTERNS",
+                     "externs should be avoided in .c files\n" .  $herecurr);
+            }
+
+            if ($paren_space =~ /\n/) {
+                WARN("FUNCTION_ARGUMENTS",
+                     "arguments for function declarations should follow identifier\n" . $herecurr);
+            }
+
+        } elsif ($realfile =~ /\.c$/ && defined $stat &&
+                 $stat =~ /^.\s*extern\s+/)
+        {
+            WARN("AVOID_EXTERNS",
+                 "externs should be avoided in .c files\n" .  $herecurr);
+        }
+
+# check for pointless casting of kmalloc return
+        if ($line =~ /\*\s*\)\s*[kv][czm]alloc(_node){0,1}\b/) {
+            WARN("UNNECESSARY_CASTS",
+                 "unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
+        }
+
+# alloc style
+# p = alloc(sizeof(struct foo), ...) should be p = alloc(sizeof(*p), ...)
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*([kv][mz]alloc(?:_node)?)\s*\(\s*(sizeof\s*\(\s*struct\s+$Lval\s*\))/) {
+            CHK("ALLOC_SIZEOF_STRUCT",
+                "Prefer $3(sizeof(*$1)...) over $3($4...)\n" . $herecurr);
+        }
+
+# check for multiple semicolons
+        if ($line =~ /;\s*;\s*$/) {
+            if (WARN("ONE_SEMICOLON",
+                     "Statements terminations use 1 semicolon\n" . $herecurr) &&
+                $fix) {
+                $fixed[$linenr - 1] =~ s/(\s*;\s*){2,}$/;/g;
+            }
+        }
+
+# check for case / default statements not preceeded by break/fallthrough/switch
+        if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) {
+            my $has_break = 0;
+            my $has_statement = 0;
+            my $count = 0;
+            my $prevline = $linenr;
+            while ($prevline > 1 && $count < 3 && !$has_break) {
+                $prevline--;
+                my $rline = $rawlines[$prevline - 1];
+                my $fline = $lines[$prevline - 1];
+                last if ($fline =~ /^\@\@/);
+                next if ($fline =~ /^\-/);
+                next if ($fline =~ /^.(?:\s*(?:case\s+(?:$Ident|$Constant)[\s$;]*|default):[\s$;]*)*$/);
+                $has_break = 1 if ($rline =~ /fall[\s_-]*(through|thru)/i);
+                next if ($fline =~ /^.[\s$;]*$/);
+                $has_statement = 1;
+                $count++;
+                $has_break = 1 if ($fline =~ /\bswitch\b|\b(?:break\s*;[\s$;]*$|return\b|goto\b|continue\b)/);
+            }
+            if (!$has_break && $has_statement) {
+                WARN("MISSING_BREAK",
+                     "Possible switch case/default not preceeded by break or fallthrough comment\n" . $herecurr);
+            }
+        }
+
+# check for switch/default statements without a break;
+        if ($^V && $^V ge 5.10.0 &&
+            defined $stat &&
+            $stat =~ /^\+[$;\s]*(?:case[$;\s]+\w+[$;\s]*:[$;\s]*|)*[$;\s]*\bdefault[$;\s]*:[$;\s]*;/g) {
+            my $ctx = '';
+            my $herectx = $here . "\n";
+            my $cnt = statement_rawlines($stat);
+            for (my $n = 0; $n < $cnt; $n++) {
+                $herectx .= raw_line($linenr, $n) . "\n";
+            }
+            WARN("DEFAULT_NO_BREAK",
+                 "switch default: should use break\n" . $herectx);
+        }
+
+# check for comparisons against true and false
+        if ($line =~ /\+\s*(.*?)\b(true|false|$Lval)\s*(==|\!=)\s*(true|false|$Lval)\b(.*)$/i) {
+            my $lead = $1;
+            my $arg = $2;
+            my $test = $3;
+            my $otype = $4;
+            my $trail = $5;
+            my $op = "!";
+
+            ($arg, $otype) = ($otype, $arg) if ($arg =~ /^(?:true|false)$/i);
+
+            my $type = lc($otype);
+            if ($type =~ /^(?:true|false)$/) {
+                if (("$test" eq "==" && "$type" eq "true") ||
+                    ("$test" eq "!=" && "$type" eq "false")) {
+                    $op = "";
+                }
+
+                CHK("BOOL_COMPARISON",
+                    "Using comparison to $otype is error prone\n" . $herecurr);
+            }
+        }
+
+# check for semaphores initialized locked
+        if ($line =~ /^.\s*sema_init.+,\W?0\W?\)/) {
+            WARN("CONSIDER_COMPLETION",
+                 "consider using a completion\n" . $herecurr);
+        }
+
+# check for %L{u,d,i} in strings
+        my $string;
+        while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
+            $string = substr($rawline, $-[1], $+[1] - $-[1]);
+            $string =~ s/%%/__/g;
+            if ($string =~ /(?<!%)%L[udi]/) {
+                WARN("PRINTF_L",
+                     "\%Ld/%Lu are not-standard C, use %lld/%llu\n" . $herecurr);
+                last;
+            }
+        }
+
+
+# Mode permission misuses where it seems decimal should be octal
+# This uses a shortcut match to avoid unnecessary uses of a slow foreach loop
+        if ($^V && $^V ge 5.10.0 &&
+            $line =~ /$mode_perms_search/) {
+            foreach my $entry (@mode_permission_funcs) {
+                my $func = $entry->[0];
+                my $arg_pos = $entry->[1];
+
+                my $skip_args = "";
+                if ($arg_pos > 1) {
+                    $arg_pos--;
+                    $skip_args = "(?:\\s*$FuncArg\\s*,\\s*){$arg_pos,$arg_pos}";
+                }
+                my $test = "\\b$func\\s*\\(${skip_args}([\\d]+)\\s*[,\\)]";
+                if ($line =~ /$test/) {
+                    my $val = $1;
+                    $val = $6 if ($skip_args ne "");
+
+                    if ($val !~ /^0$/ &&
+                        (($val =~ /^$Int$/ && $val !~ /^$Octal$/) ||
+                         length($val) ne 4)) {
+                        ERROR("NON_OCTAL_PERMISSIONS",
+                              "Use 4 digit octal (0777) not decimal permissions\n" . $herecurr);
+                    }
+                }
+            }
+        }
+    }
+
+    # If we have no input at all, then there is nothing to report on
+    # so just keep quiet.
+    if ($#rawlines == -1) {
+        exit(0);
+    }
+
+    # In mailback mode only produce a report in the negative, for
+    # things that appear to be patches.
+    if ($mailback && ($clean == 1 || !$is_patch)) {
+        exit(0);
+    }
+
+    # This is not a patch, and we are are in 'no-patch' mode so
+    # just keep quiet.
+    if (!$chk_patch && !$is_patch) {
+        exit(0);
+    }
+
+    if (!$is_patch) {
+        ERROR("NOT_UNIFIED_DIFF",
+              "Does not appear to be a unified-diff format patch\n");
+    }
+    if ($is_patch && $subject_trailing_dot != 0) {
+        ERROR("SUBJECT_TRAILING_DOT",
+              "The subject of the patch should not end with a dot.\n");
+    }
+    if ($is_patch && $chk_signoff && $signoff == 0) {
+        ERROR("MISSING_SIGN_OFF",
+              "Missing Signed-off-by: line(s)\n");
+    }
+
+    print report_dump();
+    if ($summary && !($clean == 1 && $quiet == 1)) {
+        print "$filename " if ($summary_file);
+        if ($cnt_error > 0) {
+            print "Patch not according to coding guidelines! please fix.\n";
+            print "total: $cnt_error errors, $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n"; exit 1;
+        } else {
+            print "total: $cnt_warn warnings, " .
+                (($check)? "$cnt_chk checks, " : "") .
+                "$cnt_lines lines checked\n";
+            print "Patch found to have warnings, please fix if necessary.\n" if ($cnt_warn > 0);
+            exit 2;
+        }
+        print "\n" if ($quiet == 0);
+    }
+
+    if ($quiet == 0) {
+
+        if ($^V lt 5.10.0) {
+            print("NOTE: perl $^V is not modern enough to detect all possible issues.\n");
+            print("An upgrade to at least perl v5.10.0 is suggested.\n\n");
+        }
+
+        # If there were whitespace errors which cleanpatch can fix
+        # then suggest that.
+        if ($rpt_cleaners) {
+            print "NOTE: whitespace errors detected, you may wish to use scripts/cleanpatch or\n";
+            print "      scripts/cleanfile\n\n";
+            $rpt_cleaners = 0;
+        }
+    }
+
+    hash_show_words(\%use_type, "Used");
+    hash_show_words(\%ignore_type, "Ignored");
+
+    if ($clean == 0 && $fix && "@rawlines" ne "@fixed") {
+        my $newfile = $filename;
+        $newfile .= ".EXPERIMENTAL-checkpatch-fixes" if (!$fix_inplace);
+        my $linecount = 0;
+        my $f;
+
+        open($f, '>', $newfile)
+            or die "$P: Can't open $newfile for write\n";
+        foreach my $fixed_line (@fixed) {
+            $linecount++;
+            if ($file) {
+                if ($linecount > 3) {
+                    $fixed_line =~ s/^\+//;
+                    print $f $fixed_line. "\n";
+                }
+            } else {
+                print $f $fixed_line . "\n";
+            }
+        }
+        close($f);
+
+        if (!$quiet) {
+            print << "EOM";
+Wrote EXPERIMENTAL --fix correction(s) to '$newfile'
+
+Do _NOT_ trust the results written to this file.
+Do _NOT_ submit these changes without inspecting them for correctness.
+
+This EXPERIMENTAL file is simply a convenience to help rewrite patches.
+No warranties, expressed or implied...
+EOM
+        }
+    }
+
+    if ($clean == 1 && $quiet == 0) {
+        print "$vname has no obvious style problems and is ready for submission.\n"
+    }
+    if ($clean == 0 && $quiet == 0) {
+        print << "EOM";
+$vname has style problems, please review.
+
+If any of these errors are false positives, please report
+them to the maintainer, see MAINTAINERS
+EOM
+    }
+
+    return $clean;
+}
diff --git a/build-aux/config.guess.dist b/build-aux/config.guess.dist
new file mode 100755
index 00000000000..881ba7a0438
--- /dev/null
+++ b/build-aux/config.guess.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.guess script
+in the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/config.sub.dist b/build-aux/config.sub.dist
new file mode 100755
index 00000000000..c5a0dbad282
--- /dev/null
+++ b/build-aux/config.sub.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.sub script in
+the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/pkg-version b/build-aux/pkg-version
new file mode 100755
index 00000000000..17ceab70c03
--- /dev/null
+++ b/build-aux/pkg-version
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# To override version/release from git,
+# create VERSION file containing text with version/release
+# eg. v3.4.0-1
+
+# One thing to note, If one does 'git clone --depth N glusterfs.git',
+# the git describe command doesn't work. Hence you notice below that
+# we have added timestamp as version (YYYY.MM.DD) and release (HH.mmss)
+PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*" 2>/dev/null`
+
+get_version()
+{
+    # tags and output versions:
+    #   - v3.4.0   => 3.4.0 (upstream clean)
+    #   - v3.4.0-1 => 3.4.0 (downstream clean)
+    #   - v3.4.0-2-g34e62f   => 3.4.0 (upstream dirty)
+    #   - v3.4.0-1-2-g34e62f => 3.4.0 (downstream dirty)
+    AWK_VERSION='
+    BEGIN { FS="-" }
+    /^v[0-9]/ {
+      sub(/^v/,"") ; print $1
+    }'
+
+    version=$(echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].')
+    if [ "x${version}" == "x" ] ; then
+       version=$(date +%Y.%m.%d | tr -d '\n')
+    fi
+    echo $version | tr -d '\n'
+}
+
+get_release()
+{
+    # tags and output releases:
+    #   - v3.4.0   => 0 (upstream clean)
+    #   - v3.4.0-1 => 1 (downstream clean)
+    #   - v3.4.0-2-g34e62f1   => 2.git34e62f1 (upstream dirty)
+    #   - v3.4.0-1-2-g34e62f1 => 1.2.git34e62f1 (downstream dirty)
+    AWK_RELEASE='
+    BEGIN { FS="-"; OFS="." }
+    /^v[0-9]/ {
+      if (NF == 1) print 0
+      else if (NF == 2) print $2
+      else if (NF == 3) print $2, "git" substr($3, 2)
+      else if (NF == 4) print $2, $3, "git" substr($4, 2)
+    }'
+
+    release=$(echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].')
+    if [ "x${release}" == "x" ] ; then
+       release=$(date +%H.%M%S | tr -d '\n')
+    fi
+    echo $release | tr -d '\n'
+}
+
+if test "x$1" = "x--full"; then
+    echo -n "v$(get_version)-$(get_release)"
+elif test "x$1" = "x--version"; then
+    get_version
+elif test "x$1" = "x--release"; then
+    get_release
+else
+    echo "usage: $0 [--full|--version|--release]"
+    exit 1
+fi
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index 595cfad9df2..16063f27c7f 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -1,24 +1,37 @@
 sbin_PROGRAMS = gluster
 
-gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c \
-	 cli-cmd-volume.c cli-cmd-peer.c cli3_1-cops.c cli-cmd-parser.c\
-	 cli-cmd-system.c cli-cmd-misc.c
+gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c cli-cmd-global.c \
+	 cli-cmd-volume.c cli-cmd-peer.c cli-rpc-ops.c cli-cmd-parser.c\
+	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c cli-cmd-snapshot.c
 
-gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\
+gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) \
+		$(top_builddir)/libglusterd/src/libglusterd.la \
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		$(XML_LIBS)
 
-gluster_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
-noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h
+gluster_LDFLAGS = $(GF_LDFLAGS)
+noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h cli-quotad-client.h
 
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
-	-I$(top_srcdir)/rpc/xdr/src\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/libglusterd/src \
+	-I$(top_builddir)/rpc/xdr/src \
 	-DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS)
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DGLFSHEAL_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE)
 
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
+
+$(top_builddir)/libglusterd/src/libglusterd.la:
+	$(MAKE) -C $(top_builddir)/libglusterd/src/ all
+
+install-data-hook:
+	$(mkdir_p) $(DESTDIR)$(localstatedir)/run/gluster
diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
new file mode 100644
index 00000000000..2c9a5f01bb1
--- /dev/null
+++ b/cli/src/cli-cmd-global.c
@@ -0,0 +1,195 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include <sys/socket.h>
+#include <netdb.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
+
+#include "cli.h"
+#include "cli-cmd.h"
+#include "cli-mem-types.h"
+#include "cli1-xdr.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount);
+
+struct cli_cmd global_cmds[] = {
+    {
+        "global help",
+        cli_cmd_global_help_cbk,
+        "list global commands",
+    },
+    {
+        "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+        "[file <filename>]] [detail|volumeoptions]",
+        cli_cmd_get_state_cbk,
+        "Get local state representation of mentioned daemon",
+    },
+    {
+        "nfs-ganesha {enable| disable} ",
+        cli_cmd_ganesha_cbk,
+        "Enable/disable NFS-Ganesha support",
+    },
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *global_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(global_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, global_cmds, sizeof(global_cmds));
+    count = (sizeof(global_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster global commands");
+    cli_out("========================\n");
+    for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
+        if (_gf_false == global_cmd->disable)
+            cli_out("%s - %s", global_cmd->pattern, global_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_global_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+    for (cmd = global_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
+
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
+
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Setting global option failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
+
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    ret = cli_cmd_get_state_parse(state, words, wordcount, &options,
+                                  &op_errstr);
+
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            cli_usage_out(word->pattern);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting daemon state failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 1e356c63bea..e961d88da86 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -1,105 +1,117 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 extern struct cli_cmd volume_cmds[];
+extern struct cli_cmd bitrot_cmds[];
+extern struct cli_cmd quota_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
+extern struct cli_cmd cli_bd_cmds[];
+extern struct cli_cmd snapshot_cmds[];
+extern struct cli_cmd global_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
-cli_cmd_quit_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quit_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                 const char **words, int wordcount)
 {
-        exit (0);
+    exit(0);
 }
 
-int
-cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+static gf_boolean_t
+cli_is_help_command(const char *pattern)
 {
-        struct cli_cmd        *cmd = NULL;
-
-        for (cmd = volume_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
-
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
-
-        /*
-         * commands for internal usage, don't expose
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
-         */
+    /* FixFixFix
+     * This is not the best way to determine whether
+     * this is a help command
+     */
+    if (strstr(pattern, "help"))
+        return _gf_true;
+
+    return _gf_false;
+}
 
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+int
+cli_cmd_display_help(struct cli_state *state, struct cli_cmd_word *in_word,
+                     const char **words, int wordcount)
+{
+    static struct cli_cmd *cmd[] = {
+        cli_misc_cmds, cli_probe_cmds, volume_cmds, bitrot_cmds,
+        quota_cmds,    snapshot_cmds,  global_cmds, NULL};
+    struct cli_cmd *cmd_ind = NULL;
+    int i = 0;
+    gf_boolean_t list_all = _gf_false;
+
+    /* cli_system_cmds commands for internal usage
+      they are not exposed
+    */
+
+    /* If "help all" */
+    if (wordcount == 2)
+        list_all = _gf_true;
+
+    for (i = 0; cmd[i] != NULL; i++) {
+        for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++) {
+            if ((_gf_false == cmd_ind->disable) &&
+                cli_is_help_command(cmd_ind->pattern)) {
+                if (list_all && (cmd_ind->cbk)) {
+                    cmd_ind->cbk(state, in_word, words, wordcount);
+                } else {
+                    cli_out(" %-25s- %s", cmd_ind->pattern, cmd_ind->desc);
+                }
+            }
+        }
+    }
 
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
-struct cli_cmd cli_misc_cmds[] = {
-        { "quit",
-          cli_cmd_quit_cbk,
-          "quit"},
+struct cli_cmd cli_help_cmds[] = {
+    {"help [all]", cli_cmd_display_help, "display help for command classes"},
 
-        { "help",
-           cli_cmd_display_help,
-           "display command options"},
+    {NULL, NULL, NULL}};
 
-        { NULL, NULL, NULL }
-};
+struct cli_cmd cli_misc_cmds[] = {{"quit", cli_cmd_quit_cbk, "quit"},
+                                  {"exit", cli_cmd_quit_cbk, "exit"},
 
+                                  {NULL, NULL, NULL}};
 
 int
-cli_cmd_misc_register (struct cli_state *state)
+cli_cmd_misc_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd->pattern, cmd->cbk,
-                                        cmd->desc);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = cli_help_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 7c02e1f0562..34620b4a31b 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1,1013 +1,5946 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <fnmatch.h>
+#include <time.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
 
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
-int32_t
-cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options)
+#define MAX_SNAP_DESCRIPTION_LEN 1024
+
+static struct snap_config_opt_vals_ snap_confopt_vals[] = {
+    {.op_name = "snap-max-hard-limit",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new limit.\n"
+                 "Do you want to continue?"},
+    {.op_name = "snap-max-soft-limit",
+     .question = "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to change the snap-max-soft-limit?"},
+    {.op_name = "both",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new snapshot-max-hard-limit.\n"
+                 "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to continue?"},
+    {
+        .op_name = NULL,
+    }};
+
+enum cli_snap_config_set_types {
+    GF_SNAP_CONFIG_SET_HARD = 0,
+    GF_SNAP_CONFIG_SET_SOFT = 1,
+    GF_SNAP_CONFIG_SET_BOTH = 2,
+};
+typedef enum cli_snap_config_set_types cli_snap_config_set_types;
+
+typedef struct _cli_brick {
+    struct list_head list;
+    const char *name;
+    int32_t len;
+} cli_brick_t;
+
+int
+cli_cmd_validate_volume(char *volname);
+
+static const char *
+id_sel(void *wcon)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        int     brick_count = 0, brick_index = 0;
-        int     brick_list_size = 1;
-        char    brick_list[120000] = {0,};
-        int     i = 0;
-        char    *tmp_list = NULL;
-        char    *tmpptr = NULL;
-        int     j = 0;
-        char    *host_name = NULL;
-        char    *tmp = NULL;
-        char    *freeptr = NULL;
-        char    *trans_type = NULL;
-        int32_t index = 0;
-        char    *free_list_ptr = NULL;
-        char    *bricks = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    return (const char *)wcon;
+}
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "create")) == 0);
+static char *
+str_getunamb(const char *tok, char **opwords)
+{
+    return (char *)cli_getunamb(tok, (void **)opwords, id_sel);
+}
 
-        dict = dict_new ();
+int32_t
+cli_cmd_ta_brick_parse(const char **words, int wordcount, char **ta_brick)
+{
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *delimiter = NULL;
+    cli_brick_t *brick = NULL;
+    int ret = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+
+    if (validate_brick_name((char *)words[wordcount - 1])) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words[wordcount - 1]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words[wordcount - 1], ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret)
+            goto out;
+    }
+
+    tmp_host = gf_strdup((char *)words[wordcount - 1]);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to retrieve "
+               "hostname");
+        goto out;
+    }
+
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+    }
+
+    brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+    if (brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+
+    brick->name = words[wordcount - 1];
+    brick->len = strlen(words[wordcount - 1]);
+    *ta_brick = GF_MALLOC(brick->len + 3, gf_common_mt_char);
+    if (*ta_brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+
+    strcat(*ta_brick, " ");
+    strcat(*ta_brick, brick->name);
+    strcat(*ta_brick, " ");
+out:
+    if (tmp_host) {
+        GF_FREE(tmp_host);
+        tmp_host = NULL;
+    }
+    if (brick) {
+        GF_FREE(brick);
+        brick = NULL;
+    }
+
+    return ret;
+}
 
-        if (!dict)
+int32_t
+cli_cmd_bricks_parse(const char **words, int wordcount, int brick_index,
+                     char **bricks, int *brick_count)
+{
+    int ret = 0;
+    char *delimiter = NULL;
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *bricks_str = NULL;
+    int len = 0;
+    int brick_list_len = 1; /* For initial space */
+    struct list_head brick_list = {
+        0,
+    };
+    cli_brick_t *brick = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+    GF_ASSERT(bricks);
+    GF_ASSERT(brick_index > 0);
+    GF_ASSERT(brick_index < wordcount);
+
+    INIT_LIST_HEAD(&brick_list);
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "Wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
                 goto out;
+        }
 
-        if (wordcount < 3)
+        tmp_host = gf_strdup((char *)words[brick_index]);
+        if (!tmp_host) {
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            ret = -1;
+            goto out;
+        }
+        get_host_name(tmp_host, &host_name);
+        if (!host_name) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to allocate "
+                   "memory");
+            GF_FREE(tmp_host);
+            goto out;
+        }
+
+        if (!(strcmp(host_name, "localhost") &&
+              strcmp(host_name, "127.0.0.1") && strncmp(host_name, "0.", 2))) {
+            cli_err(
+                "Please provide a valid hostname/ip other "
+                "than localhost, 127.0.0.1 or loopback "
+                "address (0.0.0.0 to 0.255.255.255).");
+            ret = -1;
+            GF_FREE(tmp_host);
+            goto out;
+        }
+        if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+            cli_err(
+                "internet address '%s' does not conform to "
+                "standards",
+                host_name);
+        }
+        GF_FREE(tmp_host);
+        list_for_each_entry(brick, &brick_list, list)
+        {
+            if (strcmp(brick->name, words[brick_index]) == 0) {
+                ret = -1;
+                cli_err("Found duplicate exports %s", words[brick_index]);
                 goto out;
+            }
+        }
 
-        volname = (char *)words[2];
+        brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+        if (brick == NULL) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            goto out;
+        }
+        len = strlen(words[brick_index]);
+        brick->name = words[brick_index];
+        brick->len = len;
+        list_add_tail(&brick->list, &brick_list);
+
+        brick_list_len += len + 1; /* Brick name + space */
+        ++(*brick_count);
+        ++brick_index;
+    }
+
+    /* If brick count is not valid exit here */
+    if (!*brick_count) {
+        cli_err("No bricks specified");
+        ret = -1;
+        goto out;
+    }
+
+    brick_list_len++; /* For terminating null char */
+
+    bricks_str = GF_MALLOC(brick_list_len, gf_common_mt_char);
+    if (bricks_str == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+    *bricks = bricks_str;
+    *bricks_str = ' ';
+    bricks_str++;
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        memcpy(bricks_str, brick->name, brick->len);
+        bricks_str[brick->len] = ' ';
+        bricks_str += brick->len + 1;
+        GF_FREE(brick);
+    }
+    *bricks_str = 0;
 
-        GF_ASSERT (volname);
+out:
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        GF_FREE(brick);
+    }
 
-        /* Validate the volume name here itself */
-        {
-                if (volname[0] == '-')
-                        goto out;
+    return ret;
+}
 
-                if (!strcmp (volname, "all")) {
-                        cli_out ("\"all\" cannot be the name of a volume.");
-                        goto out;
-                }
+int32_t
+cli_cmd_create_disperse_check(struct cli_state *state, int *disperse,
+                              int *redundancy, int *data, int count)
+{
+    int i = 0;
+    int tmp = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char question[128];
+
+    const char *question1 =
+        "There isn't an optimal redundancy value "
+        "for this configuration. Do you want to "
+        "create the volume with redundancy 1 ?";
+
+    const char *question2 =
+        "The optimal redundancy for this "
+        "configuration is %d. Do you want to create "
+        "the volume with this value ?";
+
+    const char *question3 =
+        "This configuration is not optimal on most "
+        "workloads. Do you want to use it ?";
+
+    const char *question4 =
+        "Redundancy for this configuration is %d. "
+        "Do you want to create "
+        "the volume with this value ?";
+
+    if (*data > 0) {
+        if (*disperse > 0 && *redundancy > 0) {
+            if (*disperse != (*data + *redundancy)) {
+                cli_err(
+                    "Disperse count(%d) should be equal "
+                    "to sum of disperse-data count(%d) and "
+                    "redundancy count(%d)",
+                    *disperse, *data, *redundancy);
+                return -1;
+            }
+        } else if (*redundancy > 0) {
+            *disperse = *data + *redundancy;
+        } else if (*disperse > 0) {
+            *redundancy = *disperse - *data;
+        } else {
+            if ((count - *data) >= *data) {
+                cli_err(
+                    "Please provide redundancy count "
+                    "along with disperse-data count");
+                return -1;
+            } else {
+                sprintf(question, question4, count - *data);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+                *redundancy = count - *data;
+                *disperse = count;
+            }
+        }
+    }
 
-                if (strchr (volname, '/'))
-                        goto out;
+    if (*disperse <= 0) {
+        if (count < 3) {
+            cli_err(
+                "number of bricks must be greater "
+                "than 2");
+
+            return -1;
+        }
+        *disperse = count;
+    }
+
+    if (*redundancy == -1) {
+        tmp = *disperse - 1;
+        for (i = tmp / 2; (i > 0) && ((tmp & -tmp) != tmp); i--, tmp--)
+            ;
+
+        if (i == 0) {
+            answer = cli_cmd_get_confirmation(state, question1);
+            if (answer == GF_ANSWER_NO)
+                return -1;
 
-                if (strlen (volname) > 512)
+            *redundancy = 1;
+        } else {
+            *redundancy = *disperse - tmp;
+            if (*redundancy > 1) {
+                sprintf(question, question2, *redundancy);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+            }
+        }
+
+        tmp = 0;
+    } else {
+        tmp = *disperse - *redundancy;
+    }
+
+    if ((*redundancy < 1) || (*redundancy > (*disperse - 1) / 2)) {
+        cli_err(
+            "redundancy must be greater than or equal to 1 and "
+            "less than %d for a disperse %d volume",
+            (*disperse + 1) / 2, *disperse);
+
+        return -1;
+    }
+
+    if ((tmp & -tmp) != tmp) {
+        answer = cli_cmd_get_confirmation(state, question3);
+        if (answer == GF_ANSWER_NO)
+            return -1;
+    }
+
+    return 0;
+}
+
+static int32_t
+cli_validate_disperse_volume(char *word, gf1_cluster_type type,
+                             const char **words, int32_t wordcount,
+                             int32_t index, int32_t *disperse_count,
+                             int32_t *redundancy_count, int32_t *data_count)
+{
+    int ret = -1;
+
+    switch (type) {
+        case GF_CLUSTER_TYPE_NONE:
+        case GF_CLUSTER_TYPE_DISPERSE:
+            if (strcmp(word, "disperse") == 0) {
+                if (*disperse_count >= 0) {
+                    cli_err("disperse option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], disperse_count);
+                if (ret == -1 && errno == EINVAL) {
+                    *disperse_count = 0;
+                    ret = 1;
+                } else if (ret == -1) {
+                    goto out;
+                } else {
+                    if (*disperse_count < 3) {
+                        cli_err(
+                            "disperse count must "
+                            "be greater than 2");
                         goto out;
+                    }
+                    ret = 2;
+                }
+            } else if (strcmp(word, "disperse-data") == 0) {
+                if (*data_count >= 0) {
+                    cli_err("disperse-data option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], data_count);
+                if (ret == -1 || *data_count < 2) {
+                    cli_err("disperse-data must be greater than 1");
+                    goto out;
+                }
+                ret = 2;
+            } else if (strcmp(word, "redundancy") == 0) {
+                if (*redundancy_count >= 0) {
+                    cli_err("redundancy option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], redundancy_count);
+                if (ret == -1 || *redundancy_count < 1) {
+                    cli_err("redundancy must be greater than 0");
+                    goto out;
+                }
+                ret = 2;
+            }
+            break;
+        case GF_CLUSTER_TYPE_REPLICATE:
+            cli_err(
+                "replicated-dispersed volume is not "
+                "supported");
+            goto out;
+        default:
+            cli_err("Invalid type given");
+            break;
+    }
+out:
+    return ret;
+}
 
-                for (i = 0; i < strlen (volname); i++)
-                        if (!isalnum (volname[i]) && (volname[i] != '_') && (volname[i] != '-'))
-                                goto out;
+int32_t
+cli_validate_volname(const char *volname)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    int volname_len;
+    static const char *const invalid_volnames[] = {"volume",
+                                                   "type",
+                                                   "subvolumes",
+                                                   "option",
+                                                   "end-volume",
+                                                   "all",
+                                                   "volume_not_in_ring",
+                                                   "description",
+                                                   "force",
+                                                   "snap-max-hard-limit",
+                                                   "snap-max-soft-limit",
+                                                   "auto-delete",
+                                                   "activate-on-create",
+                                                   NULL};
+
+    if (volname[0] == '-')
+        goto out;
+
+    for (i = 0; invalid_volnames[i]; i++) {
+        if (!strcmp(volname, invalid_volnames[i])) {
+            cli_err("\"%s\" cannot be the name of a volume.", volname);
+            goto out;
         }
+    }
+
+    if (strchr(volname, '/'))
+        goto out;
+
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volume name exceeds %d characters.", GD_VOLUME_NAME_MAX);
+        goto out;
+    }
+
+    for (i = 0; i < volname_len; i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            goto out;
+        }
+    }
 
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
+int32_t
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **brick_list)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int sub_count = 1;
+    int brick_index = 0;
+    char *trans_type = NULL;
+    int32_t index = 0;
+    char *bricks = NULL;
+    char *ta_brick = NULL;
+    int32_t brick_count = 0;
+    static char *opwords[] = {"replica",  "stripe",       "transport",
+                              "disperse", "redundancy",   "disperse-data",
+                              "arbiter",  "thin-arbiter", NULL};
+
+    char *w = NULL;
+    int op_count = 0;
+    int32_t replica_count = 1;
+    int32_t arbiter_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t stripe_count = 1;
+    int32_t disperse_count = -1;
+    int32_t redundancy_count = -1;
+    int32_t disperse_data_count = -1;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount < 3)
+        goto out;
+
+    volname = (char *)words[2];
+
+    GF_ASSERT(volname);
+
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+
+    type = GF_CLUSTER_TYPE_NONE;
+    index = 3;
+
+    while (op_count < 3) {
+        ret = -1;
+        w = str_getunamb(words[index], opwords);
+        if (!w) {
+            break;
+        } else if ((strcmp(w, "replica")) == 0) {
+            switch (type) {
+                case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                case GF_CLUSTER_TYPE_REPLICATE:
+                    cli_err("replica option given twice");
+                    goto out;
+                case GF_CLUSTER_TYPE_NONE:
+                    type = GF_CLUSTER_TYPE_REPLICATE;
+                    break;
+                case GF_CLUSTER_TYPE_STRIPE:
+                    cli_err("stripe option not supported");
+                    goto out;
+                case GF_CLUSTER_TYPE_DISPERSE:
+                    cli_err(
+                        "replicated-dispersed volume is not "
+                        "supported");
+                    goto out;
+                default:
+                    cli_err("Invalid type given");
+                    goto out;
+            }
+
+            if (wordcount < (index + 2)) {
+                ret = -1;
                 goto out;
+            }
 
-        if (wordcount < 4) {
+            replica_count = strtol(words[index + 1], NULL, 0);
+            if (replica_count < 2) {
+                cli_err(
+                    "replica count should be greater"
+                    " than 1");
                 ret = -1;
                 goto out;
-        }
-        if ((strcasecmp (words[3], "replica")) == 0) {
-                type = GF_CLUSTER_TYPE_REPLICATE;
-                if (wordcount < 5) {
+            }
+
+            index += 2;
+            if (words[index]) {
+                if (!strcmp(words[index], "arbiter")) {
+                    ret = gf_string2int(words[index + 1], &arbiter_count);
+                    if ((ret == -1) || (arbiter_count != 1)) {
+                        cli_err(
+                            "For arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the arbiter");
                         ret = -1;
                         goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_out ("replica count should be greater than 1");
-                        ret = -1;
+                    }
+                    ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+                    if (ret)
                         goto out;
-                }
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
-        } else if ((strcasecmp (words[3], "stripe")) == 0) {
-                type = GF_CLUSTER_TYPE_STRIPE;
-                if (wordcount < 5) {
+                    index += 2;
+                } else if (!strcmp(words[index], "thin-arbiter")) {
+                    ret = gf_string2int(words[index + 1], &thin_arbiter_count);
+                    if ((ret == -1) || (thin_arbiter_count != 1) ||
+                        (replica_count != 2)) {
+                        cli_err(
+                            "For thin-arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and thin-arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the thin-arbiter brick");
                         ret = -1;
                         goto out;
+                    }
+                    ret = dict_set_int32(dict, "thin-arbiter-count",
+                                         thin_arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
                 }
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_out ("stripe count should be greater than 1");
+            }
+
+            /* Do this to keep glusterd happy with sending
+               "replica 3 arbiter 1" options to server */
+            if ((arbiter_count == 1) && (replica_count == 2))
+                replica_count += arbiter_count;
+
+            if (replica_count == 2 && thin_arbiter_count == 0) {
+                if (strcmp(words[wordcount - 1], "force")) {
+                    question =
+                        "Replica 2 volumes are prone"
+                        " to split-brain. Use "
+                        "Arbiter or Replica 3 to "
+                        "avoid this. See: "
+                        "http://docs.gluster.org/en/latest/"
+                        "Administrator%20Guide/"
+                        "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                        "\nDo you still want to "
+                        "continue?\n";
+                    answer = cli_cmd_get_confirmation(state, question);
+                    if (GF_ANSWER_NO == answer) {
+                        gf_log("cli", GF_LOG_ERROR,
+                               "Volume create "
+                               "cancelled, exiting");
                         ret = -1;
                         goto out;
+                    }
                 }
-                ret = dict_set_int32 (dict, "stripe-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
+            }
+            ret = dict_set_int32(dict, "replica-count", replica_count);
+            if (ret)
+                goto out;
+
+        } else if ((strcmp(w, "stripe")) == 0) {
+            cli_err("stripe option not supported");
+            goto out;
+        } else if ((strcmp(w, "transport")) == 0) {
+            if (trans_type) {
+                cli_err(
+                    "'transport' option given more"
+                    " than one time");
+                goto out;
+            }
+            if ((strcasecmp(words[index + 1], "tcp") == 0)) {
+                trans_type = gf_strdup("tcp");
+            } else if ((strcasecmp(words[index + 1], "rdma") == 0)) {
+                trans_type = gf_strdup("rdma");
+            } else if ((strcasecmp(words[index + 1], "tcp,rdma") == 0) ||
+                       (strcasecmp(words[index + 1], "rdma,tcp") == 0)) {
+                trans_type = gf_strdup("tcp,rdma");
+            } else {
+                gf_log("", GF_LOG_ERROR,
+                       "incorrect transport"
+                       " protocol specified");
+                ret = -1;
+                goto out;
+            }
+            index += 2;
+
+        } else if ((strcmp(w, "disperse") == 0) ||
+                   (strcmp(w, "redundancy") == 0) ||
+                   (strcmp(w, "disperse-data") == 0)) {
+            ret = cli_validate_disperse_volume(
+                w, type, words, wordcount, index, &disperse_count,
+                &redundancy_count, &disperse_data_count);
+            if (ret < 0)
+                goto out;
+            index += ret;
+            type = GF_CLUSTER_TYPE_DISPERSE;
+        } else if ((strcmp(w, "arbiter") == 0)) {
+            cli_err(
+                "arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else if ((strcmp(w, "thin-arbiter") == 0)) {
+            cli_err(
+                "thin-arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
         } else {
-                type = GF_CLUSTER_TYPE_NONE;
-                brick_index = 3;
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+        op_count++;
+    }
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret)
-                goto out;
+    if (!trans_type)
+        trans_type = gf_strdup("tcp");
+
+    if (index >= wordcount) {
+        ret = -1;
+        goto out;
+    }
 
-        if (type)
-                index = 5;
+    brick_index = index;
+
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
+
+    // Exclude the thin-arbiter-brick i.e. last brick in the bricks list
+    if (thin_arbiter_count == 1) {
+        ret = cli_cmd_bricks_parse(words, wc - 1, brick_index, &bricks,
+                                   &brick_count);
+        if (ret)
+            goto out;
+
+        ret = cli_cmd_ta_brick_parse(words, wc, &ta_brick);
+
+    } else {
+        ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks,
+                                   &brick_count);
+    }
+
+    if (ret)
+        goto out;
+
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = cli_cmd_create_disperse_check(state, &disperse_count,
+                                            &redundancy_count,
+                                            &disperse_data_count, brick_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "disperse-count", disperse_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "redundancy-count", redundancy_count);
+        if (ret)
+            goto out;
+
+        sub_count = disperse_count;
+    } else
+        sub_count = stripe_count * replica_count;
+
+    if (brick_count % sub_count) {
+        if (type == GF_CLUSTER_TYPE_STRIPE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "stripe count");
+        else if (type == GF_CLUSTER_TYPE_REPLICATE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "replica count");
+        else if (type == GF_CLUSTER_TYPE_DISPERSE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "disperse count");
         else
-                index = 3;
+            cli_err(
+                "number of bricks given doesn't match "
+                "required count");
+
+        ret = -1;
+        goto out;
+    }
+
+    /* Everything is parsed fine. start setting info in dict */
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret)
+        goto out;
+
+    ret = dict_set_dynstr(dict, "transport", trans_type);
+    if (ret)
+        goto out;
+    trans_type = NULL;
+
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
+
+    if (thin_arbiter_count == 1) {
+        ret = dict_set_dynstr(dict, "ta-brick", ta_brick);
+        if (ret)
+            goto out;
+    }
 
-        if (wordcount < (index + 1)) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_int32(dict, "count", brick_count);
+    if (ret)
+        goto out;
 
-        if (strcasecmp(words[index], "transport") == 0) {
-                brick_index = index+2;
-                if (wordcount < (index + 2)) {
-                        ret = -1;
-                        goto out;
-                }
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-                if ((strcasecmp (words[index+1], "tcp") == 0)) {
-                        trans_type = gf_strdup ("tcp");
-                } else if ((strcasecmp (words[index+1], "rdma") == 0)) {
-                        trans_type = gf_strdup ("rdma");
-                } else {
-                        gf_log ("", GF_LOG_ERROR, "incorrect transport"
-                                       " protocol specified");
-                        ret = -1;
-                        goto out;
-                }
+    *options = dict;
+    *brick_list = bricks;
+out:
+    if (ret) {
+        GF_FREE(bricks);
+        GF_FREE(ta_brick);
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
+        if (dict)
+            dict_unref(dict);
+    }
+
+    GF_FREE(trans_type);
+
+    return ret;
+}
+
+int32_t
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount < 3)
+        goto out;
+
+    if (wordcount > 5)
+        goto out;
+
+    volname = (char *)words[2];
+
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (wordcount == 3) {
+        ret = dict_set_str(dict, "key", "all");
+        if (ret)
+            goto out;
+    }
+
+    if (wordcount >= 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "key", "all");
+            if (ret)
+                goto out;
         } else {
-                trans_type = gf_strdup ("tcp");
+            ret = dict_set_str(dict, "key", (char *)words[3]);
+            if (ret)
+                goto out;
         }
+    }
 
-        ret = dict_set_dynstr (dict, "transport", trans_type);
-        if (ret)
+    if (wordcount == 5) {
+        if (strcmp("force", (char *)words[4])) {
+            ret = -1;
+            goto out;
+        } else {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
                 goto out;
+        }
+    }
+
+    *options = dict;
+
+out:
+    if (ret && dict) {
+        dict_unref(dict);
+    }
 
-        strcpy (brick_list, " ");
-        while (brick_index < wordcount) {
-                delimiter = strchr (words[brick_index], ':');
-                if (!delimiter || delimiter == words[brick_index]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
+    return ret;
+}
+
+int32_t
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *odir = NULL;
+    char *filename = NULL;
+    char *daemon_name = NULL;
+    int count = 0;
+    uint32_t cmd = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", options, out);
+    GF_VALIDATE_OR_GOTO("cli", words, out);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (wordcount < 1 || wordcount > 7) {
+        *op_errstr = gf_strdup(
+            "Problem parsing arguments."
+            " Check usage.");
+        goto out;
+    }
+
+    if (wordcount >= 1) {
+        gf_asprintf(&daemon_name, "%s", "glusterd");
+
+        for (count = 1; count < wordcount; count++) {
+            if (strcmp(words[count], "odir") == 0 ||
+                strcmp(words[count], "file") == 0) {
+                if (strcmp(words[count], "odir") == 0) {
+                    if (++count < wordcount) {
+                        odir = (char *)words[count];
+                        continue;
+                    } else {
                         ret = -1;
                         goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
-                }
-                if ((brick_list_size + strlen (words[brick_index]) + 1) > 120000) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "total brick list is larger than a request "
-                                "can take (brick_count %d)", brick_count);
+                    }
+                } else if (strcmp(words[count], "file") == 0) {
+                    if (++count < wordcount) {
+                        filename = (char *)words[count];
+                        continue;
+                    } else {
                         ret = -1;
                         goto out;
+                    }
                 }
-
-                host_name = gf_strdup(words[brick_index]);
-                if (!host_name) {
+            } else {
+                if (count > 1) {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    } else {
+                        *op_errstr = gf_strdup(
+                            "Problem"
+                            " parsing arguments. "
+                            "Check usage.");
                         ret = -1;
-                        gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                               "memory");
                         goto out;
+                    }
                 }
-                freeptr = host_name;
+                if (strcmp(words[count], "glusterd") == 0) {
+                    continue;
+                } else {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    }
 
-                strtok_r(host_name, ":", &tmp);
-                if (!(strcmp(host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1"))) {
-                        cli_out ("Please provide a valid hostname/ip other "
-                                 "than localhost or 127.0.0.1");
-                        ret = -1;
-                        GF_FREE(freeptr);
-                        goto out;
+                    *op_errstr = gf_strdup(
+                        "glusterd is "
+                        "the only supported daemon.");
+                    ret = -1;
+                    goto out;
                 }
-                GF_FREE (freeptr);
-                tmp_list = strdup(brick_list+1);
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(( brick_count != 0) && (j < brick_count)) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_out ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                if (free_list_ptr)
-                                        free (free_list_ptr);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_size += (strlen (words[brick_index]) + 1);
-                ++brick_count;
-                ++brick_index;
-                /*
-                  char    key[50];
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
-
-                if (ret)
-                        goto out;
-                */
-                if (free_list_ptr)
-                        free (free_list_ptr);
+            }
         }
 
-        /* If brick-count is not valid when replica or stripe is
-           given, exit here */
-        if (!brick_count) {
-                cli_out ("No bricks specified");
-                ret = -1;
+        ret = dict_set_dynstr(dict, "daemon", daemon_name);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Command failed. Please check "
+                " log file for more details.");
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Setting daemon name to dictionary failed");
+            goto out;
+        }
+        daemon_name = NULL;
+
+        if (odir) {
+            ret = dict_set_str(dict, "odir", odir);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting output directory to"
+                       "dictionary failed");
                 goto out;
+            }
         }
 
-        if (brick_count % count) {
-                if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_out ("number of bricks is not a multiple of "
-                                 "stripe count");
-                else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_out ("number of bricks is not a multiple of "
-                                 "replica count");
-                ret = -1;
+        if (filename) {
+            ret = dict_set_str(dict, "filename", filename);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more  details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting filename to dictionary failed");
                 goto out;
+            }
         }
 
-        bricks = gf_strdup (brick_list);
-        if (!bricks) {
-                ret = -1;
+        if (cmd) {
+            ret = dict_set_uint32(dict, "getstate-cmd", cmd);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting "
+                       "get-state command type to dictionary "
+                       "failed");
                 goto out;
+            }
         }
+    }
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+out:
+    if (dict)
+        *options = dict;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
-        if (ret)
-                goto out;
+    if (ret && dict)
+        dict_unref(dict);
 
-        *options = dict;
+    GF_FREE(daemon_name);
 
+    return ret;
+}
+
+int32_t
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    if (wordcount != 4)
+        goto out;
+
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
+
+    if (strcmp(words[3], "enable") != 0) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
-        return ret;
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int i = -1;
+    char key[20] = {
+        0,
+    };
+    int64_t value = 0;
+    gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE;
+    static char *opwords[] = {"enable",
+                              "disable",
+                              "limit-usage",
+                              "remove",
+                              "list",
+                              "alert-time",
+                              "soft-timeout",
+                              "hard-timeout",
+                              "default-soft-limit",
+                              "limit-objects",
+                              "list-objects",
+                              "remove-objects",
+                              NULL};
+    char *w = NULL;
+    uint32_t time = 0;
+    double percent = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    if (wordcount < 4) {
+        if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+            ret = 1;
+        }
+        goto out;
+    }
+
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (strcmp(w, "limit-usage") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+    } else if (strcmp(w, "limit-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
+        if (wordcount < 6 || wordcount > 7) {
+            ret = -1;
+            goto out;
+        }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "reset")) == 0);
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret)
+            goto out;
 
-        dict = dict_new ();
+        if (!words[5]) {
+            cli_err("Please enter the limit value to be set");
+            ret = -1;
+            goto out;
+        }
 
-        if (!dict)
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = gf_string2bytesize_int64(words[5], &value);
+            if (ret != 0 || value <= 0) {
+                if (errno == ERANGE || value <= 0) {
+                    ret = -1;
+                    cli_err(
+                        "Please enter an integer "
+                        "value in the range of "
+                        "(1 - %" PRId64 ")",
+                        INT64_MAX);
+                } else
+                    cli_err(
+                        "Please enter a correct "
+                        "value");
                 goto out;
-
-        if (wordcount < 3)
+            }
+        } else {
+            errno = 0;
+            limit = strtol(words[5], &end_ptr, 10);
+            if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+                strcmp(end_ptr, "") != 0) {
+                ret = -1;
+                cli_err(
+                    "Please enter an integer value in "
+                    "the range 1 - %" PRId64,
+                    INT64_MAX);
                 goto out;
+            }
+        }
 
-        volname = (char *)words[2];
+        ret = dict_set_str(dict, "hard-limit", (char *)words[5]);
+        if (ret < 0)
+            goto out;
 
-        if (!volname) {
+        if (wordcount == 7) {
+            ret = gf_string2percent(words[6], &percent);
+            if (ret != 0 || percent > 100) {
                 ret = -1;
+                cli_err(
+                    "Please enter a correct value "
+                    "in the range of 0 to 100");
+                goto out;
+            }
+
+            ret = dict_set_str(dict, "soft-limit", (char *)words[6]);
+            if (ret < 0)
                 goto out;
         }
 
-        ret = dict_set_str (dict, "volname", volname);
+        goto set_type;
+    }
 
-        if (ret)
+    if (strcmp(w, "remove") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+
+        type = GF_QUOTA_OPTION_TYPE_REMOVE;
+
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "remove-objects") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+
+        type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
+
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST;
+
+        if (words[4] && words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
+
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0)
                 goto out;
+        }
 
-        *options = dict;
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0)
+            goto out;
+
+        goto set_type;
+    }
+
+    if (strcmp(w, "list-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
+
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
+
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set "
+                       "quota patch in request dictionary");
+                goto out;
+            }
+        }
+
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set quota "
+                   "limit count in request dictionary");
+            goto out;
+        }
+
+        goto set_type;
+    }
+
+    if (strcmp(w, "alert-time") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
 
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "soft-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
+
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+
+    if (strcmp(w, "hard-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
+
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+    if (strcmp(w, "default-soft-limit") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
+
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-                if (ret) {
+    if (ret < 0) {
         if (dict)
-                dict_destroy (dict);
-                }
+            dict_unref(dict);
+    }
+
+    return ret;
+}
+
+static gf_boolean_t
+cli_is_key_spl(char *key)
+{
+    return (strcmp(key, "group") == 0);
+}
+
+static int32_t
+cli_add_key_group_value(dict_t *dict, const char *name, const char *value,
+                        int32_t id, char **op_errstr)
+{
+    char *key = NULL;
+    char *data = NULL;
+    int32_t ret = -1;
+
+    ret = gf_asprintf(&key, "%s%d", name, id);
+    if (ret < 0) {
+        goto out;
+    }
+    data = gf_strdup(value);
+    if (data == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to allocate memory for data");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(dict, key, data);
+    if (ret == 0) {
+        data = NULL;
+    }
+
+out:
+    GF_FREE(key);
+    GF_FREE(data);
+
+    if ((ret != 0) && (op_errstr != NULL)) {
+        *op_errstr = gf_strdup("Failed to allocate memory");
+    }
 
-                return ret;
+    return ret;
+}
+
+static int
+cli_add_key_group(dict_t *dict, char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char *saveptr = NULL;
+    char *tok_key = NULL;
+    char *tok_val = NULL;
+    char *tagpath = NULL;
+    char line[PATH_MAX + 256] = {
+        0,
+    };
+    FILE *fp = NULL;
+
+    ret = gf_asprintf(&tagpath, "%s/groups/%s", GLUSTERD_DEFAULT_WORKDIR,
+                      value);
+    if (ret == -1) {
+        tagpath = NULL;
+        goto out;
+    }
+
+    fp = fopen(tagpath, "r");
+    if (!fp) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr,
+                        "Unable to open file '%s'. "
+                        "Error: %s",
+                        tagpath, strerror(errno));
+        }
+        goto out;
+    }
+
+    opt_count = 0;
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (strlen(line) >= sizeof(line) - 1) {
+            ret = -1;
+            if (op_errstr != NULL) {
+                *op_errstr = gf_strdup("Line too long");
+            }
+            goto out;
+        }
+
+        /* Treat line that start with "#" as comments */
+        if ('#' == line[0])
+            continue;
+
+        opt_count++;
+        tok_key = strtok_r(line, "=", &saveptr);
+        tok_val = strtok_r(NULL, "\r\n", &saveptr);
+        if (!tok_key || !tok_val) {
+            ret = -1;
+            if (op_errstr) {
+                gf_asprintf(op_errstr,
+                            "'%s' file format "
+                            "not valid.",
+                            tagpath);
+            }
+            goto out;
+        }
+
+        ret = cli_add_key_group_value(dict, "key", tok_key, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+        ret = cli_add_key_group_value(dict, "value", tok_val, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (!opt_count) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr, "'%s' file format not valid.", tagpath);
+        }
+        goto out;
+    }
+    ret = dict_set_int32(dict, "count", opt_count);
+out:
+
+    GF_FREE(tagpath);
+
+    if (fp)
+        fclose(fp);
+
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     count = 0;
-        char    *key = NULL;
-        char    *value = NULL;
-        int     i = 0;
-        char    str[50] = {0,};
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int count = 0;
+    char *key = NULL;
+    char *value = NULL;
+    int i = 0;
+    char str[50] = {
+        0,
+    };
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "set")) == 0);
+    dict = dict_new();
 
-        dict = dict_new ();
+    if (!dict)
+        goto out;
 
-        if (!dict)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        if (wordcount < 4)
-                goto out;
+    volname = (char *)words[2];
 
-        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        GF_ASSERT (volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (ret)
+        goto out;
 
+    if (!strcmp(volname, "all")) {
+        ret = dict_set_str(dict, "globalname", "All");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
+
+        ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
+    }
+
+    if ((!strcmp(volname, "help") || !strcmp(volname, "help-xml")) &&
+        wordcount == 3) {
+        ret = dict_set_str(dict, volname, volname);
         if (ret)
-                goto out;
+            goto out;
+
+    } else if (wordcount < 5) {
+        ret = -1;
+        goto out;
+
+    } else if (wordcount == 5 && cli_is_key_spl((char *)words[3])) {
+        key = (char *)words[3];
+        value = (char *)words[4];
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
+        ret = gf_strip_whitespace(value, strlen(value));
+        if (ret == -1)
+            goto out;
 
-        for (i = 3; i < wordcount; i+=2) {
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-		key = (char *) words[i];
-		value = (char *) words[i+1];
+        ret = cli_add_key_group(dict, key, value, op_errstr);
+        if (ret == 0)
+            *options = dict;
+        goto out;
+    }
 
-                if ( key && !value ) {
-                        if ( !strcmp (key, "history")) {
-                                ret = dict_set_str (dict, key, "history");
-                                if (ret)
-                                        goto out;
-                                ret = dict_set_int32 (dict, "count", 1);
-                                if (ret)
-                                        goto out;
-                                *options = dict;
-                                goto out;
-                        }
-                }
+    for (i = 3; i < wordcount; i += 2) {
+        key = (char *)words[i];
+        value = (char *)words[i + 1];
 
-		if ( !key || !value) {
-			ret = -1;
-			goto out;
-        	}
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                count++;
+        count++;
 
-                sprintf (str, "key%d", count);
-                ret = dict_set_str (dict, str, key);
-                if (ret)
-                        goto out;
+        if (fnmatch("user.*", key, FNM_NOESCAPE) != 0) {
+            ret = gf_strip_whitespace(value, strlen(value));
+            if (ret == -1)
+                goto out;
+        }
 
-                sprintf (str, "value%d", count);
-                ret = dict_set_str (dict, str, value);
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-                if (ret)
-                        goto out;
+        if (cli_is_key_spl(key)) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "count", wordcount-3);
+        sprintf(str, "key%d", count);
+        ret = dict_set_str(dict, str, key);
+        if (ret)
+            goto out;
+
+        sprintf(str, "value%d", count);
+        ret = dict_set_str(dict, str, value);
 
         if (ret)
+            goto out;
+
+        if ((!strcmp(key, "cluster.enable-shared-storage")) &&
+            (!strcmp(value, "disable"))) {
+            question =
+                "Disabling cluster.enable-shared-storage "
+                "will delete the shared storage volume"
+                "(gluster_shared_storage), which is used "
+                "by snapshot scheduler, geo-replication "
+                "and NFS-Ganesha. Do you still want to "
+                "continue?";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
                 goto out;
+            }
+        }
+        if ((!strcmp(key, "nfs.disable")) && (!strcmp(value, "off"))) {
+            question =
+                "Gluster NFS is being deprecated in favor "
+                "of NFS-Ganesha Enter \"yes\" to continue "
+                "using Gluster NFS";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
+    }
 
-        *options = dict;
+    ret = dict_set_int32(dict, "count", wordcount - 3);
+
+    if (ret)
+        goto out;
+
+    *options = dict;
 
 out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options)
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *ret_type)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        int     brick_count = 0, brick_index = 0;
-        int     brick_list_size = 1;
-        char    brick_list[120000] = {0,};
-        int     j = 0;
-        char    *tmp_list = NULL;
-        char    *tmpptr = NULL;
-        char    *host_name = NULL;
-        char    *tmp = NULL;
-        char    *freeptr = NULL;
-        char    *free_list_ptr = NULL;
-        char    *bricks = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int brick_count = 0, brick_index = 0;
+    char *bricks = NULL;
+    static char *opwords_cl[] = {"replica", "stripe", NULL};
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int count = 1;
+    int arbiter_count = 0;
+    char *w = NULL;
+    int index;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount < 3)
+        goto out;
+
+    volname = (char *)words[2];
+
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+
+    if (ret)
+        goto out;
+
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+    if (wordcount < 6) {
+        /* seems no options are given, go directly to the parse_brick */
+        brick_index = 3;
+        type = GF_CLUSTER_TYPE_NONE;
+        goto parse_bricks;
+    }
+
+    w = str_getunamb(words[3], opwords_cl);
+    if (!w) {
+        type = GF_CLUSTER_TYPE_NONE;
+        index = 3;
+    } else if ((strcmp(w, "replica")) == 0) {
+        type = GF_CLUSTER_TYPE_REPLICATE;
+        count = strtol(words[4], NULL, 0);
+        if (!count || (count < 2)) {
+            cli_err("replica count should be greater than 1");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        index = 5;
+        if (words[index] && !strcmp(words[index], "arbiter")) {
+            arbiter_count = strtol(words[6], NULL, 0);
+            if (arbiter_count != 1 || count != 3) {
+                cli_err(
+                    "For arbiter configuration, replica "
+                    "count must be 3 and arbiter count "
+                    "must be 1. The 3rd brick of the "
+                    "replica will be the arbiter");
+                ret = -1;
+                goto out;
+            }
+            ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+            if (ret)
+                goto out;
+            index = 7;
+        }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                question =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or "
+                    "Replica 3 to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, question);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Add brick"
+                           " cancelled, exiting");
+                    ret = -1;
+                    goto out;
+                }
+            }
+        }
+    } else if ((strcmp(w, "stripe")) == 0) {
+        cli_err("stripe option not supported");
+        goto out;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+        ret = -1;
+        goto out;
+    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "add-brick")) == 0);
+    brick_index = index;
 
-        dict = dict_new ();
+parse_bricks:
 
-        if (!dict)
-                goto out;
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        if (wordcount < 3)
-                goto out;
+    ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks, &brick_count);
+    if (ret)
+        goto out;
 
-        volname = (char *)words[2];
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "count", brick_count);
+
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-        GF_ASSERT (volname);
+    *options = dict;
 
-        ret = dict_set_str (dict, "volname", volname);
+out:
+    if (ret_type)
+        *ret_type = type;
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
+    return ret;
+}
+
+int32_t
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *comm)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *delimiter = NULL;
+    int ret = -1;
+    char key[50];
+    int brick_index = 0;
+    int32_t tmp_index = 0;
+    int32_t j = 0;
+    char *tmp_brick = NULL;
+    char *tmp_brick1 = NULL;
+    static char *type_opword[] = {"replica", NULL};
+    static char *opwords[] = {"start",  "commit", "stop",
+                              "status", "force",  NULL};
+    char *w = NULL;
+    int32_t command = GF_OP_CMD_NONE;
+    long count = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *ques = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount < 5)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[2];
+
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    brick_index = 3;
+    w = str_getunamb(words[3], type_opword);
+    if (w && !strcmp("replica", w)) {
+        if (wordcount < 6) {
+            ret = -1;
+            goto out;
+        }
+        count = strtol(words[4], NULL, 0);
+        if (count < 1) {
+            cli_err(
+                "replica count should be greater than 0 in "
+                "case of remove-brick");
+            ret = -1;
+            goto out;
+        }
+
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                ques =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or Replica 3 "
+                    "to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, ques);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Remove "
+                           "brick cancelled, exiting");
+                    ret = -1;
+                    goto out;
+                }
+            }
+        }
+
+        ret = dict_set_int32(dict, "replica-count", count);
         if (ret)
+            goto out;
+        brick_index = 5;
+    } else if (w) {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+    w = str_getunamb(words[wordcount - 1], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    } else {
+        /* handled this option */
+        wordcount--;
+        if (!strcmp("start", w)) {
+            command = GF_OP_CMD_START;
+            if (question)
+                *question = 1;
+        } else if (!strcmp("commit", w)) {
+            command = GF_OP_CMD_COMMIT;
+        } else if (!strcmp("stop", w)) {
+            command = GF_OP_CMD_STOP;
+        } else if (!strcmp("status", w)) {
+            command = GF_OP_CMD_STATUS;
+        } else if (!strcmp("force", w)) {
+            command = GF_OP_CMD_COMMIT_FORCE;
+            if (question)
+                *question = 1;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "command", command);
+    if (ret)
+        gf_log("cli", GF_LOG_INFO, "failed to set 'command' %d", command);
+
+    tmp_index = brick_index;
+    tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
+
+    if (!tmp_brick) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
+
+    if (!tmp_brick1) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
                 goto out;
+        }
 
-        if (wordcount < 4) {
+        j = tmp_index;
+        strcpy(tmp_brick, words[brick_index]);
+        while (j < brick_index) {
+            strcpy(tmp_brick1, words[j]);
+            if (!(strcmp(tmp_brick, tmp_brick1))) {
+                gf_log("", GF_LOG_ERROR,
+                       "Duplicate bricks"
+                       " found %s",
+                       words[brick_index]);
+                cli_err("Duplicate bricks found %s", words[brick_index]);
                 ret = -1;
                 goto out;
+            }
+            j++;
         }
+        snprintf(key, 50, "brick%d", ++(*brick_count));
+        ret = dict_set_str(dict, key, (char *)words[brick_index++]);
 
-        brick_index = 3;
-        strcpy (brick_list, " ");
-        while (brick_index < wordcount) {
-                delimiter = strchr (words[brick_index], ':');
-                if (!delimiter || delimiter == words[brick_index]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
-                }
+        if (ret)
+            goto out;
+    }
 
-                if ((brick_list_size + strlen (words[brick_index]) + 1) > 120000) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "total brick list is larger than a request "
-                                "can take (brick_count %d)", brick_count);
-                        ret = -1;
-                        goto out;
-                }
+    if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
+        ret = dict_set_int32(dict, "count", *brick_count);
+        if (ret)
+            goto out;
+    }
 
-                host_name = gf_strdup(words[brick_index]);
-                if (!host_name) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "unable to allocate "
-                                "memory");
-                        goto out;
-                }
-                freeptr = host_name;
-                strtok_r(host_name, ":", &tmp);
-                if (!(strcmp(host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1"))) {
-                        cli_out ("Please provide a valid hostname/ip other "
-                                 "localhost or 127.0.0.1");
-                        ret = -1;
-                        GF_FREE (freeptr);
-                        goto out;
-                }
-                GF_FREE (freeptr);
-
-                tmp_list = strdup(brick_list+1);
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(( brick_count != 0) && (j < brick_count)) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_out ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                if (free_list_ptr)
-                                        free (free_list_ptr);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_size += (strlen (words[brick_index]) + 1);
-                ++brick_count;
-                ++brick_index;
-                /*
-                  char    key[50];
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
+    *options = dict;
 
-                if (ret)
-                        goto out;
-                */
-                if (free_list_ptr)
-                        free (free_list_ptr);
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
+
+    GF_FREE(tmp_brick);
+    GF_FREE(tmp_brick1);
+
+    *comm = command;
+
+    return ret;
+}
+
+int32_t
+cli_cmd_brick_op_validate_bricks(const char **words, dict_t *dict, int src,
+                                 int dst)
+{
+    int ret = -1;
+    char *delimiter = NULL;
+
+    if (validate_brick_name((char *)words[src])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[3]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[src], '/');
+        ret = gf_canonicalize_path(delimiter);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_set_str(dict, "src-brick", (char *)words[src]);
+    if (ret)
+        goto out;
+
+    if (dst == -1) {
+        ret = 0;
+        goto out;
+    }
+
+    if (validate_brick_name((char *)words[dst])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[dst]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[dst], '/');
+        ret = gf_canonicalize_path(delimiter);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_set_str(dict, "dst-brick", (char *)words[dst]);
+    if (ret)
+        goto out;
+    ret = 0;
+out:
+    return ret;
+}
+
+int32_t
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options)
+{
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount < 5 || wordcount > 7)
+        goto out;
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[2];
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (wordcount == 5) {
+        if (strcmp(words[4], "start")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[4]);
+            ret = -1;
+            goto out;
         }
 
-        bricks = gf_strdup (brick_list);
-        if (!bricks) {
-                ret = -1;
-                goto out;
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, -1);
+        if (ret)
+            goto out;
+
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_START");
+        if (ret)
+            goto out;
+    } else if (wordcount == 6) {
+        if (strcmp(words[5], "commit")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5]);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
         if (ret)
-                goto out;
+            goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT");
+        if (ret)
+            goto out;
+    } else if (wordcount == 7) {
+        if (strcmp(words[5], "commit") || strcmp(words[6], "force")) {
+            cli_err(
+                "Invalid option '%s %s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5], words[6]);
+            ret = -1;
+            goto out;
+        }
 
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
         if (ret)
-                goto out;
+            goto out;
 
-        *options = dict;
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT_FORCE");
+        if (ret)
+            goto out;
+    }
+
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
                                    dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        char    key[50];
-        int     brick_count = 0, brick_index = 0;
-        int32_t tmp_index = 0;
-        int32_t j = 0;
-        char    *tmp_brick = NULL;
-        char    *tmp_brick1 = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount != 7) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
+        goto out;
+    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "remove-brick")) == 0);
+    volname = (char *)words[2];
 
-        dict = dict_new ();
+    GF_ASSERT(volname);
 
-        if (!dict)
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+    if (ret)
+        goto out;
+
+    /* commit force option */
+    if (strcmp("commit", words[5]) || strcmp("force", words[6])) {
+        cli_err(
+            "Invalid option '%s' '%s' for replace-brick. Please "
+            "enter valid replace-brick command",
+            words[5], words[6]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
+
+    return ret;
+}
+
+int32_t
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    str = (char *)words[4];
+    if (strchr(str, ':')) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
                 goto out;
+        }
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+        /* Path */
+        str = (char *)words[5];
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    } else {
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    }
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        if (wordcount < 3)
+    return ret;
+}
+
+int32_t
+cli_cmd_log_level_parse(const char **words, int worcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /*
+     * loglevel command format:
+     *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
+     *  > volume log level colon-o posix WARNING
+     *  > volume log level colon-o replicate* DEBUG
+     *  > volume log level coon-o * TRACE
+     */
+
+    GF_ASSERT((strncmp(words[0], "volume", 6) == 0));
+    GF_ASSERT((strncmp(words[1], "log", 3) == 0));
+    GF_ASSERT((strncmp(words[2], "level", 5) == 0));
+
+    ret = glusterd_check_log_level(words[5]);
+    if (ret == -1) {
+        cli_err("Invalid log level [%s] specified", words[5]);
+        cli_err(
+            "Valid values for loglevel: (DEBUG|WARNING|ERROR"
+            "|CRITICAL|NONE|TRACE)");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    GF_ASSERT(words[3]);
+    GF_ASSERT(words[4]);
+
+    ret = dict_set_str(dict, "volname", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(dict, "xlator", (char *)words[4]);
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(dict, "loglevel", (char *)words[5]);
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+int32_t
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
                 goto out;
+        }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        volname = (char *)words[2];
+    *options = dict;
 
-        GF_ASSERT (volname);
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        ret = dict_set_str (dict, "volname", volname);
+    return ret;
+}
+
+int32_t
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (strcmp("rotate", words[3]) == 0)
+        volname = (char *)words[2];
+    GF_ASSERT(volname);
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
         if (ret)
+            goto out;
+    }
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+static gf_boolean_t
+gsyncd_url_check(const char *w)
+{
+    return !!strpbrk(w, ":/");
+}
+
+static gf_boolean_t
+valid_slave_gsyncd_url(const char *w)
+{
+    if (strstr(w, ":::"))
+        return _gf_false;
+    else if (strstr(w, "::"))
+        return _gf_true;
+    else
+        return _gf_false;
+}
+
+static gf_boolean_t
+gsyncd_glob_check(const char *w)
+{
+    return !!strpbrk(w, "*?[");
+}
+
+static int
+config_parse(const char **words, int wordcount, dict_t *dict, unsigned cmdi,
+             unsigned glob)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    char *append_str = NULL;
+    size_t append_len = 0;
+    char *subop = NULL;
+    char *ret_chkpt = NULL;
+    struct tm checkpoint_time;
+    char chkpt_buf[20] = "";
+
+    switch ((wordcount - 1) - cmdi) {
+        case 0:
+            subop = gf_strdup("get-all");
+            break;
+        case 1:
+            if (words[cmdi + 1][0] == '!') {
+                (words[cmdi + 1])++;
+                if (gf_asprintf(&subop, "del%s", glob ? "-glob" : "") == -1)
+                    subop = NULL;
+            } else
+                subop = gf_strdup("get");
+
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
                 goto out;
+            break;
+        default:
+            if (gf_asprintf(&subop, "set%s", glob ? "-glob" : "") == -1)
+                subop = NULL;
+
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+
+            /* join the varargs by spaces to get the op_value */
+
+            for (i = cmdi + 2; i < wordcount; i++)
+                append_len += (strlen(words[i]) + 1);
+            /* trailing strcat will add two bytes, make space for that */
+            append_len++;
 
-        if (wordcount < 4) {
+            /* strcat is used on this allocation and hence expected to be
+             * initiatlized to 0. So GF_CALLOC is used.
+             */
+            append_str = GF_CALLOC(1, append_len, cli_mt_append_str);
+            if (!append_str) {
                 ret = -1;
                 goto out;
+            }
+
+            for (i = cmdi + 2; i < wordcount; i++) {
+                strcat(append_str, words[i]);
+                strcat(append_str, " ");
+            }
+            append_str[append_len - 2] = '\0';
+            /* "checkpoint now" is special: we resolve that "now" */
+            if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                (strcmp(append_str, "now") == 0)) {
+                struct timeval tv = {
+                    0,
+                };
+
+                ret = gettimeofday(&tv, NULL);
+                if (ret == -1)
+                    goto out;
+
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
+                }
+                snprintf(append_str, 300, "%" GF_PRI_SECOND, tv.tv_sec);
+            } else if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                       (strcmp(append_str, "now") != 0)) {
+                memset(&checkpoint_time, 0, sizeof(struct tm));
+                ret_chkpt = strptime(append_str, "%Y-%m-%d %H:%M:%S",
+                                     &checkpoint_time);
+
+                if (ret_chkpt == NULL || *ret_chkpt != '\0') {
+                    ret = -1;
+                    cli_err(
+                        "Invalid Checkpoint label. Use format "
+                        "\"Y-m-d H:M:S\", Example: 2016-10-25 15:30:45");
+                    goto out;
+                }
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
+                }
+                strftime(chkpt_buf, sizeof(chkpt_buf), "%s", &checkpoint_time);
+                snprintf(append_str, 300, "%s", chkpt_buf);
+            }
+
+            ret = dict_set_dynstr(dict, "op_value", append_str);
+            if (ret != 0) {
+                goto out;
+            }
+            append_str = NULL;
+    }
+
+    ret = -1;
+    if (subop) {
+        ret = dict_set_dynstr(dict, "subop", subop);
+        if (!ret)
+            subop = NULL;
+    }
+
+out:
+    GF_FREE(append_str);
+    GF_FREE(subop);
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+/* ssh_port_parse: Parses and validates when ssh_port is given.
+ *                 ssh_index refers to index of ssh_port and
+ *                 type refers to either push-pem or no-verify
+ */
+
+static int32_t
+parse_ssh_port(const char **words, int wordcount, dict_t *dict, unsigned *cmdi,
+               int ssh_index, char *type)
+{
+    int ret = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    if (!strcmp((char *)words[ssh_index], "ssh-port")) {
+        if (strcmp((char *)words[ssh_index - 1], "create")) {
+            ret = -1;
+            goto out;
+        }
+        (*cmdi)++;
+        limit = strtol(words[ssh_index + 1], &end_ptr, 10);
+        if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+            strcmp(end_ptr, "") != 0) {
+            ret = -1;
+            cli_err("Please enter an integer value for ssh_port ");
+            goto out;
         }
 
-        brick_index = 3;
+        ret = dict_set_int32(dict, "ssh_port", limit);
+        if (ret)
+            goto out;
+        (*cmdi)++;
+    } else if (strcmp((char *)words[ssh_index + 1], "create")) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, type, 1);
+    if (ret)
+        goto out;
+    (*cmdi)++;
 
+out:
+    return ret;
+}
+
+static int32_t
+force_push_pem_no_verify_parse(const char **words, int wordcount, dict_t *dict,
+                               unsigned *cmdi)
+{
+    int32_t ret = 0;
+
+    if (!strcmp((char *)words[wordcount - 1], "force")) {
+        if ((strcmp((char *)words[wordcount - 2], "start")) &&
+            (strcmp((char *)words[wordcount - 2], "stop")) &&
+            (strcmp((char *)words[wordcount - 2], "create")) &&
+            (strcmp((char *)words[wordcount - 2], "no-verify")) &&
+            (strcmp((char *)words[wordcount - 2], "push-pem")) &&
+            (strcmp((char *)words[wordcount - 2], "pause")) &&
+            (strcmp((char *)words[wordcount - 2], "resume"))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(dict, "force", SLEN("force"), 1);
         if (ret)
+            goto out;
+        (*cmdi)++;
+
+        if (!strcmp((char *)words[wordcount - 2], "push-pem")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "push_pem");
+            if (ret)
+                goto out;
+        } else if (!strcmp((char *)words[wordcount - 2], "no-verify")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "no_verify");
+            if (ret)
                 goto out;
+        }
+    } else if (!strcmp((char *)words[wordcount - 1], "push-pem")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "push_pem");
+        if (ret)
+            goto out;
+    } else if (!strcmp((char *)words[wordcount - 1], "no-verify")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "no_verify");
+        if (ret)
+            goto out;
+    }
 
-        tmp_index = brick_index;
-        tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        if (!tmp_brick) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
-                goto out;
+int32_t
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **errstr)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_gsync_set type = GF_GSYNC_OPTION_TYPE_NONE;
+    int i = 0;
+    unsigned masteri = 0;
+    unsigned slavei = 0;
+    unsigned glob = 0;
+    unsigned cmdi = 0;
+    static char *opwords[] = {"create",    "status",   "start",  "stop",
+                              "config",    "force",    "delete", "ssh-port",
+                              "no-verify", "push-pem", "detail", "pause",
+                              "resume",    NULL};
+    char *w = NULL;
+    char *save_ptr = NULL;
+    char *slave_temp = NULL;
+    char *token = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    /* new syntax:
+     *
+     * volume geo-replication $m $s create [[ssh-port n] [[no-verify] |
+     * [push-pem]]] [force] volume geo-replication [$m [$s]] status [detail]
+     * volume geo-replication [$m] $s config [[!]$opt [$val]]
+     * volume geo-replication $m $s start|stop [force]
+     * volume geo-replication $m $s delete [reset-sync-time]
+     * volume geo-replication $m $s pause [force]
+     * volume geo-replication $m $s resume [force]
+     */
+
+    if (wordcount < 3)
+        goto out;
+
+    for (i = 2; i <= 3 && i < wordcount - 1; i++) {
+        if (gsyncd_glob_check(words[i]))
+            glob = i;
+        if (gsyncd_url_check(words[i])) {
+            slavei = i;
+            break;
+        }
+    }
+
+    if (glob && !slavei)
+        /* glob is allowed only for config, thus it implies there is a
+         * slave argument; but that might have not been recognized on
+         * the first scan as it's url characteristics has been covered
+         * by the glob syntax.
+         *
+         * In this case, the slave is perforce the last glob-word -- the
+         * upcoming one is neither glob, nor url, so it's definitely not
+         * the slave.
+         */
+        slavei = glob;
+    if (slavei) {
+        cmdi = slavei + 1;
+        if (slavei == 3)
+            masteri = 2;
+    } else if (i <= 4) {
+        if (strtail("detail", (char *)words[wordcount - 1])) {
+            cmdi = wordcount - 2;
+            if (i == 4)
+                masteri = 2;
+        } else {
+            /* no $s, can only be status cmd
+             * (with either a single $m before it or nothing)
+             * -- these conditions imply that i <= 3 after
+             * the iteration and that i is the successor of
+             * the (0 or 1 length) sequence of $m-s.
+             */
+            cmdi = i;
+            if (i == 3)
+                masteri = 2;
+        }
+    } else
+        goto out;
+
+    /* now check if input really complies syntax
+     * (in a somewhat redundant way, in favor
+     * transparent soundness)
+     */
+
+    if (masteri && gsyncd_url_check(words[masteri]))
+        goto out;
+
+    if (slavei && !glob && !valid_slave_gsyncd_url(words[slavei])) {
+        gf_asprintf(errstr, "Invalid slave url: %s", words[slavei]);
+        goto out;
+    }
+
+    w = str_getunamb(words[cmdi], opwords);
+    if (!w)
+        goto out;
+
+    if (strcmp(w, "create") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "status") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STATUS;
+
+        if (slavei && !masteri)
+            goto out;
+    } else if (strcmp(w, "config") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CONFIG;
+
+        if (!slavei)
+            goto out;
+    } else if (strcmp(w, "start") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_START;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "stop") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STOP;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "delete") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_DELETE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "pause") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_PAUSE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "resume") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_RESUME;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = force_push_pem_no_verify_parse(words, wordcount, dict, &cmdi);
+    if (ret)
+        goto out;
+
+    if (strtail("detail", (char *)words[wordcount - 1])) {
+        if (!strtail("status", (char *)words[wordcount - 2])) {
+            ret = -1;
+            goto out;
         }
- 
-        tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
 
-        if (!tmp_brick1) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
+        ret = dict_set_uint32(dict, "status-detail", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
+        !strcmp((char *)words[wordcount - 1], "reset-sync-time")) {
+        if (strcmp((char *)words[wordcount - 2], "delete")) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_uint32(dict, "reset-sync-time", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
+
+    if (type != GF_GSYNC_OPTION_TYPE_CONFIG && (cmdi < wordcount - 1 || glob)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* If got so far, input is valid, assemble the message */
+
+    ret = 0;
+
+    if (masteri) {
+        ret = dict_set_str(dict, "master", (char *)words[masteri]);
+        if (!ret)
+            ret = dict_set_str(dict, "volname", (char *)words[masteri]);
+    }
+    if (!ret && slavei) {
+        /* If geo-rep is created with root user using the syntax
+         * gluster vol geo-rep <mastervol> root@<slavehost> ...
+         * pass down only <slavehost> else pass as it is.
+         */
+        slave_temp = gf_strdup(words[slavei]);
+        if (slave_temp == NULL) {
+            ret = -1;
+            goto out;
+        }
+        token = strtok_r(slave_temp, "@", &save_ptr);
+        if (token && !strcmp(token, "root")) {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei] + 5);
+        } else {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei]);
+        }
+    }
+    if (!ret)
+        ret = dict_set_int32(dict, "type", type);
+    if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        if (!strcmp((char *)words[wordcount - 2], "ignore-deletes") &&
+            !strcmp((char *)words[wordcount - 1], "true")) {
+            question =
+                "There exists ~15 seconds delay for the option to take"
+                " effect from stime of the corresponding brick. Please"
+                " check the log for the time, the option is effective."
+                " Proceed";
+
+            answer = cli_cmd_get_confirmation(state, question);
+
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_INFO,
+                       "Operation "
+                       "cancelled, exiting");
+                *errstr = gf_strdup("Aborted by user.");
                 ret = -1;
                 goto out;
+            }
         }
 
-        while (brick_index < wordcount) {
-                delimiter = strchr(words[brick_index], ':');
-                if (!delimiter || delimiter == words[brick_index]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
-                }
-
-                j = tmp_index;
-                strcpy(tmp_brick, words[brick_index]);
-                while ( j < brick_index) {
-                        strcpy(tmp_brick1, words[j]);
-                        if (!(strcmp (tmp_brick, tmp_brick1))) {
-                                gf_log("",GF_LOG_ERROR, "Duplicate bricks"
-                                       " found %s", words[brick_index]);
-                                cli_out("Duplicate bricks found %s",
-                                        words[brick_index]);
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
+        ret = config_parse(words, wordcount, dict, cmdi, glob);
+    }
 
-                if (ret)
-                        goto out;
+out:
+    if (slave_temp)
+        GF_FREE(slave_temp);
+    if (ret && dict)
+        dict_unref(dict);
+    else
+        *options = dict;
+
+    return ret;
+}
+
+int32_t
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+    gf_boolean_t is_peek = _gf_false;
+
+    static char *opwords[] = {"start", "stop", "info", NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (wordcount < 4)
+        goto out;
+
+    volname = (char *)words[2];
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((strcmp(w, "start") == 0 || strcmp(w, "stop") == 0) && wordcount > 5) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "info") == 0 && wordcount > 7) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "start") == 0) {
+        op = GF_CLI_STATS_START;
+    } else if (strcmp(w, "stop") == 0) {
+        op = GF_CLI_STATS_STOP;
+    } else if (strcmp(w, "info") == 0) {
+        op = GF_CLI_STATS_INFO;
+        info_op = GF_CLI_INFO_ALL;
+        if (wordcount > 4) {
+            if (strcmp(words[4], "incremental") == 0) {
+                info_op = GF_CLI_INFO_INCREMENTAL;
+                if (wordcount > 5 && strcmp(words[5], "peek") == 0) {
+                    is_peek = _gf_true;
+                }
+            } else if (strcmp(words[4], "cumulative") == 0) {
+                info_op = GF_CLI_INFO_CUMULATIVE;
+            } else if (strcmp(words[4], "clear") == 0) {
+                info_op = GF_CLI_INFO_CLEAR;
+            } else if (strcmp(words[4], "peek") == 0) {
+                is_peek = _gf_true;
+            }
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "info-op", (int32_t)info_op);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    ret = dict_set_int32(dict, "peek", is_peek);
+    if (ret)
+        goto out;
 
+    if (!strcmp(words[wordcount - 1], "nfs")) {
+        ret = dict_set_int32(dict, "nfs", _gf_true);
+        if (ret)
+            goto out;
+    }
+
+    *options = dict;
 out:
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
+}
+
+int32_t
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char *key = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    int32_t list_cnt = -1;
+    int index = 0;
+    int perf = 0;
+    int32_t blk_size = 0;
+    int count = 0;
+    gf_boolean_t nfs = _gf_false;
+    char *delimiter = NULL;
+    static char *opwords[] = {"open",       "read",    "write",
+                              "opendir",    "readdir", "read-perf",
+                              "write-perf", "clear",   NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (wordcount < 4)
+        goto out;
+
+    volname = (char *)words[2];
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
+
+    op = GF_CLI_STATS_TOP;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+    if (strcmp(w, "open") == 0) {
+        top_op = GF_CLI_TOP_OPEN;
+    } else if (strcmp(w, "read") == 0) {
+        top_op = GF_CLI_TOP_READ;
+    } else if (strcmp(w, "write") == 0) {
+        top_op = GF_CLI_TOP_WRITE;
+    } else if (strcmp(w, "opendir") == 0) {
+        top_op = GF_CLI_TOP_OPENDIR;
+    } else if (strcmp(w, "readdir") == 0) {
+        top_op = GF_CLI_TOP_READDIR;
+    } else if (strcmp(w, "read-perf") == 0) {
+        top_op = GF_CLI_TOP_READ_PERF;
+        perf = 1;
+    } else if (strcmp(w, "write-perf") == 0) {
+        top_op = GF_CLI_TOP_WRITE_PERF;
+        perf = 1;
+    } else if (strcmp(w, "clear") == 0) {
+        ret = dict_set_int32(dict, "clear-stats", 1);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
+            gf_log("cli", GF_LOG_ERROR, "Could not set clear-stats in dict");
+            goto out;
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
+    ret = dict_set_int32(dict, "top-op", (int32_t)top_op);
+    if (ret)
+        goto out;
+
+    if ((wordcount > 4) && !strcmp(words[4], "nfs")) {
+        nfs = _gf_true;
+        ret = dict_set_int32(dict, "nfs", nfs);
+        if (ret)
+            goto out;
+        index = 5;
+    } else {
+        index = 4;
+    }
+
+    for (; index < wordcount; index += 2) {
+        key = (char *)words[index];
+        value = (char *)words[index + 1];
+
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(key, "brick")) {
+            delimiter = strchr(value, ':');
+            if (!delimiter || delimiter == value || *(delimiter + 1) != '/') {
+                cli_err(
+                    "wrong brick type: %s, use <HOSTNAME>:"
+                    "<export-dir-abs-path>",
+                    value);
+                ret = -1;
+                goto out;
+            } else {
+                ret = gf_canonicalize_path(delimiter + 1);
+                if (ret)
+                    goto out;
+            }
+            ret = dict_set_str(dict, "brick", value);
+
+        } else if (!strcmp(key, "list-cnt")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                list_cnt = atoi(value);
+            if (ret || (list_cnt < 0) || (list_cnt > 100)) {
+                cli_err("list-cnt should be between 0 to 100");
+                ret = -1;
+                goto out;
+            }
+        } else if (perf && !nfs && !strcmp(key, "bs")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                blk_size = atoi(value);
+            if (ret || (blk_size <= 0)) {
+                if (blk_size < 0)
+                    cli_err(
+                        "block size is an invalid"
+                        " number");
+                else
+                    cli_err(
+                        "block size should be an "
+                        "integer greater than zero");
+                ret = -1;
+                goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-size", (uint32_t)blk_size);
+        } else if (perf && !nfs && !strcmp(key, "count")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                count = atoi(value);
+            if (ret || (count <= 0)) {
+                if (count < 0)
+                    cli_err("count is an invalid number");
+                else
+                    cli_err(
+                        "count should be an integer "
+                        "greater than zero");
 
-        if (tmp_brick)
-                GF_FREE (tmp_brick);
-        if (tmp_brick1)
-                GF_FREE (tmp_brick1);
-
-        return ret;
+                ret = -1;
+                goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-cnt", count);
+        } else {
+            ret = -1;
+            goto out;
+        }
+        if (ret) {
+            gf_log("", GF_LOG_WARNING,
+                   "Dict set failed for "
+                   "key %s",
+                   key);
+            goto out;
+        }
+    }
+    if (list_cnt == -1)
+        list_cnt = 100;
+    ret = dict_set_int32(dict, "list-cnt", list_cnt);
+    if (ret) {
+        gf_log("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        goto out;
+    }
+
+    if ((blk_size > 0) ^ (count > 0)) {
+        cli_err("Need to give both 'bs' and 'count'");
+        ret = -1;
+        goto out;
+    } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+        cli_err("'bs * count' value %" PRIu64
+                " is greater than "
+                "maximum allowed value of 10GB",
+                ((uint64_t)blk_size * count));
+        ret = -1;
+        goto out;
+    }
+
+    *options = dict;
+out:
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
+uint32_t
+cli_cmd_get_statusop(const char *arg)
+{
+    int i = 0;
+    uint32_t ret = GF_CLI_STATUS_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"detail",   "mem",   "clients",     "fd", "inode",
+                              "callpool", "tasks", "client-list", NULL};
+    static struct {
+        char *opname;
+        uint32_t opcode;
+    } optable[] = {{"detail", GF_CLI_STATUS_DETAIL},
+                   {"mem", GF_CLI_STATUS_MEM},
+                   {"clients", GF_CLI_STATUS_CLIENTS},
+                   {"fd", GF_CLI_STATUS_FD},
+                   {"inode", GF_CLI_STATUS_INODE},
+                   {"callpool", GF_CLI_STATUS_CALLPOOL},
+                   {"tasks", GF_CLI_STATUS_TASKS},
+                   {"client-list", GF_CLI_STATUS_CLIENT_LIST},
+                   {NULL}};
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Not a status op  %s", arg);
+        goto out;
+    }
+
+    for (i = 0; optable[i].opname; i++) {
+        if (!strcmp(w, optable[i].opname)) {
+            ret = optable[i].opcode;
+            break;
+        }
+    }
 
-int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
-                                   dict_t **options)
+out:
+    return ret;
+}
+
+int
+cli_cmd_volume_status_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        char    *op = NULL;
-        int     op_index = 0;
-        char    *delimiter = NULL;
-        gf1_cli_replace_op replace_op = GF_REPLACE_OP_NONE;
+    dict_t *dict = NULL;
+    int ret = -1;
+    uint32_t cmd = 0;
+
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    switch (wordcount) {
+        case 2:
+            cmd = GF_CLI_STATUS_ALL;
+            ret = 0;
+            break;
+
+        case 3:
+            if (!strcmp(words[2], "all")) {
+                cmd = GF_CLI_STATUS_ALL;
+                ret = 0;
+
+            } else {
+                cmd = GF_CLI_STATUS_VOL;
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+            }
+
+            break;
+
+        case 4:
+            cmd = cli_cmd_get_statusop(words[3]);
+
+            if (!strcmp(words[2], "all")) {
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    cli_err("%s is not a valid status option", words[3]);
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_ALL;
+                ret = 0;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+            } else {
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+                if (ret)
+                    goto out;
+
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    if (!strcmp(words[3], "nfs")) {
+                        cmd |= GF_CLI_STATUS_NFS;
+                    } else if (!strcmp(words[3], "shd")) {
+                        cmd |= GF_CLI_STATUS_SHD;
+                    } else if (!strcmp(words[3], "quotad")) {
+                        cmd |= GF_CLI_STATUS_QUOTAD;
+                    } else if (!strcmp(words[3], "snapd")) {
+                        cmd |= GF_CLI_STATUS_SNAPD;
+                    } else if (!strcmp(words[3], "bitd")) {
+                        cmd |= GF_CLI_STATUS_BITD;
+                    } else if (!strcmp(words[3], "scrub")) {
+                        cmd |= GF_CLI_STATUS_SCRUB;
+                    } else {
+                        cmd = GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str(dict, "brick", (char *)words[3]);
+                    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "replace-brick")) == 0);
+                } else {
+                    cmd |= GF_CLI_STATUS_VOL;
+                    ret = 0;
+                }
+            }
+
+            break;
 
-        dict = dict_new ();
+        case 5:
+            if (!strcmp(words[2], "all")) {
+                cli_err("Cannot specify brick/nfs for \"all\"");
+                ret = -1;
+                goto out;
+            }
 
-        if (!dict)
+            cmd = cli_cmd_get_statusop(words[4]);
+            if (cmd == GF_CLI_STATUS_NONE) {
+                cli_err("%s is not a valid status option", words[4]);
+                ret = -1;
                 goto out;
+            }
 
-        if (wordcount < 3)
+            ret = dict_set_str(dict, "volname", (char *)words[2]);
+            if (ret)
                 goto out;
 
-        volname = (char *)words[2];
+            if (!strcmp(words[3], "nfs")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_DETAIL ||
+                    cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Tasks status not available"
+                        " for NFS Servers");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_NFS;
+            } else if (!strcmp(words[3], "shd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Clients/Tasks status not "
+                        "available for Self-heal Daemons");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SHD;
+            } else if (!strcmp(words[3], "quotad")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for Quota Daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_QUOTAD;
+            } else if (!strcmp(words[3], "snapd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for snap daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SNAPD;
+            } else {
+                if (cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Tasks status not available for "
+                        "bricks");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_BRICK;
+                ret = dict_set_str(dict, "brick", (char *)words[3]);
+            }
+            break;
 
-        GF_ASSERT (volname);
+        default:
+            goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+gf_boolean_t
+cli_cmd_validate_dumpoption(const char *arg, char **option)
+{
+    static char *opwords[] = {"all",   "nfs",    "mem",   "iobuf",   "callpool",
+                              "priv",  "fd",     "inode", "history", "inodectx",
+                              "fdctx", "quotad", NULL};
+    char *w = NULL;
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Unknown statedump option  %s", arg);
+        return _gf_false;
+    }
+    *option = w;
+    return _gf_true;
+}
 
-        if (wordcount < 4) {
+int
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options)
+{
+    int ret = 0;
+    int i = 0;
+    dict_t *dict = NULL;
+    int option_cnt = 0;
+    char *option = NULL;
+    char *option_str = NULL;
+    char *tmp_str = NULL;
+    char *tmp = NULL;
+    char *ip_addr = NULL;
+    char *pid = NULL;
+
+    if ((wordcount >= 5) && ((strcmp(words[3], "client")) == 0)) {
+        tmp = gf_strdup(words[4]);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        ip_addr = strtok(tmp, ":");
+        pid = strtok(NULL, ":");
+        if (valid_internet_address(ip_addr, _gf_true, _gf_false) && pid &&
+            gf_valid_pid(pid, strlen(pid))) {
+            ret = gf_asprintf(&option_str, "%s %s %s", words[3], ip_addr, pid);
+            if (ret < 0) {
+                goto out;
+            }
+            option_cnt = 3;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    } else {
+        for (i = 3; i < wordcount; i++, option_cnt++) {
+            if (!cli_cmd_validate_dumpoption(words[i], &option)) {
                 ret = -1;
                 goto out;
+            }
+            tmp_str = option_str;
+            option_str = NULL;
+            ret = gf_asprintf(&option_str, "%s%s ", tmp_str ? tmp_str : "",
+                              option);
+            GF_FREE(tmp_str);
+            if (ret < 0) {
+                goto out;
+            }
+        }
+        if (option_str && (strstr(option_str, "nfs")) &&
+            strstr(option_str, "quotad")) {
+            ret = -1;
+            goto out;
         }
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    /* dynamic string in dict is freed up when dict is freed up, and hence
+    if option_str is NULL pass in an duplicate empty string to the same */
+    ret = dict_set_dynstr(dict, "options",
+                          (option_str ? option_str : gf_strdup("")));
+    if (ret)
+        goto out;
+    option_str = NULL;
+
+    ret = dict_set_int32(dict, "option_cnt", option_cnt);
+    if (ret)
+        goto out;
+
+    *options = dict;
+out:
+    GF_FREE(tmp);
+    GF_FREE(option_str);
+    if (ret && dict)
+        dict_unref(dict);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
+    return ret;
+}
 
-        delimiter = strchr ((char *)words[3], ':');
-        if (!delimiter || delimiter == words[3]
-            || *(delimiter+1) != '/') {
-                cli_out ("wrong brick type: %s, use "
-                        "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
+int
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options)
+{
+    int ret = -1;
+    int i = 0;
+    dict_t *dict = NULL;
+    char *kind_opts[4] = {"blocked", "granted", "all", NULL};
+    char *types[4] = {"inode", "entry", "posix", NULL};
+    char *free_ptr = NULL;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (strcmp(words[4], "kind"))
+        goto out;
+
+    for (i = 0; kind_opts[i]; i++) {
+        if (!strcmp(words[5], kind_opts[i])) {
+            free_ptr = gf_strdup(words[5]);
+            ret = dict_set_dynstr(dict, "kind", free_ptr);
+            if (ret)
                 goto out;
-        } else {
-                cli_path_strip_trailing_slashes (delimiter + 1);
+            free_ptr = NULL;
+            break;
+        }
+    }
+    if (i == 3)
+        goto out;
+
+    ret = -1;
+    for (i = 0; types[i]; i++) {
+        if (!strcmp(words[6], types[i])) {
+            free_ptr = gf_strdup(words[6]);
+            ret = dict_set_dynstr(dict, "type", free_ptr);
+            if (ret)
+                goto out;
+            free_ptr = NULL;
+            break;
         }
-        ret = dict_set_str (dict, "src-brick", (char *)words[3]);
+    }
+    if (i == 3)
+        goto out;
 
+    if (wordcount == 8) {
+        free_ptr = gf_strdup(words[7]);
+        ret = dict_set_dynstr(dict, "opts", free_ptr);
         if (ret)
-                goto out;
+            goto out;
+        free_ptr = NULL;
+    }
+
+    ret = 0;
+    *options = dict;
+out:
+    if (ret) {
+        GF_FREE(free_ptr);
+        dict_unref(dict);
+    }
 
-        if (wordcount < 5) {
+    return ret;
+}
+
+static int
+extract_hostname_path_from_token(const char *tmp_words, char **hostname,
+                                 char **path)
+{
+    int ret = 0;
+    char *delimiter = NULL;
+    char *tmp_host = NULL;
+    char *host_name = NULL;
+    char *words = NULL;
+    int str_len = 0;
+    *hostname = NULL;
+    *path = NULL;
+
+    str_len = strlen(tmp_words) + 1;
+    words = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!words) {
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(words, str_len, "%s", tmp_words);
+
+    if (validate_brick_name(words)) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words, ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret) {
+            goto out;
+        } else {
+            str_len = strlen(delimiter + 1) + 1;
+            *path = GF_MALLOC(str_len, gf_common_mt_char);
+            if (!*path) {
                 ret = -1;
                 goto out;
+            }
+            snprintf(*path, str_len, "%s", delimiter + 1);
         }
+    }
+
+    tmp_host = gf_strdup(words);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to allocate "
+               "memory");
+        goto out;
+    }
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+        ret = -1;
+        goto out;
+    }
+
+    str_len = strlen(host_name) + 1;
+    *hostname = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!*hostname) {
+        ret = -1;
+        goto out;
+    }
+    snprintf(*hostname, str_len, "%s", host_name);
+    ret = 0;
 
-        delimiter = strchr ((char *)words[4], ':');
-        if (!delimiter || delimiter == words[4]
-            || *(delimiter+1) != '/') {
-                cli_out ("wrong brick type: %s, use "
-                        "<HOSTNAME>:<export-dir-abs-path>", words[4]);
-                ret = -1;
+out:
+    GF_FREE(words);
+    GF_FREE(tmp_host);
+    return ret;
+}
+
+static int
+set_hostname_path_in_dict(const char *token, dict_t *dict, int heal_op)
+{
+    char *hostname = NULL;
+    char *path = NULL;
+    int ret = 0;
+
+    ret = extract_hostname_path_from_token(token, &hostname, &path);
+    if (ret)
+        goto out;
+
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_set_dynstr(dict, "heal-source-hostname", hostname);
+            if (ret)
                 goto out;
-        } else {
-                cli_path_strip_trailing_slashes (delimiter + 1);
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "heal-source-brickpath", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            ret = dict_set_dynstr(dict, "per-replica-cmd-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-path", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
+        default:
+            ret = -1;
+            break;
+    }
+
+out:
+    GF_FREE(hostname);
+    GF_FREE(path);
+    return ret;
+}
+
+static int
+heal_command_type_get(const char *command)
+{
+    int i = 0;
+    /* subcommands are set as NULL */
+    char *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = NULL,
+        [GF_SHD_OP_HEAL_INDEX] = NULL,
+        [GF_SHD_OP_HEAL_FULL] = "full",
+        [GF_SHD_OP_INDEX_SUMMARY] = "info",
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL,
+        [GF_SHD_OP_STATISTICS] = "statistics",
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK] = NULL,
+        [GF_SHD_OP_HEAL_ENABLE] = "enable",
+        [GF_SHD_OP_HEAL_DISABLE] = "disable",
+    };
+
+    for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
+        if (heal_cmds[i] && (strcmp(heal_cmds[i], command) == 0))
+            return i;
+    }
+
+    return GF_SHD_OP_INVALID;
+}
+
+int
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options)
+{
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set volname");
+        goto out;
+    }
+
+    if (wordcount == 3) {
+        ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
+        goto done;
+    }
+
+    if (wordcount == 4) {
+        op = heal_command_type_get(words[3]);
+        if (op == GF_SHD_OP_INVALID) {
+            ret = -1;
+            goto out;
         }
 
+        ret = dict_set_int32(dict, "heal-op", op);
+        goto done;
+    }
 
-        ret = dict_set_str (dict, "dst-brick", (char *)words[4]);
+    if (wordcount == 5) {
+        if (strcmp(words[3], "info") && strcmp(words[3], "statistics") &&
+            strcmp(words[3], "granular-entry-heal")) {
+            ret = -1;
+            goto out;
+        }
 
-        if (ret)
-                goto out;
+        if (!strcmp(words[3], "info")) {
+            if (!strcmp(words[4], "split-brain")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_SPLIT_BRAIN_FILES);
+                goto done;
+            }
+            if (!strcmp(words[4], "summary")) {
+                ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_SUMMARY);
+                goto done;
+            }
+        }
 
-        op_index = 5;
-        if ((wordcount < (op_index + 1))) {
-                ret = -1;
+        if (!strcmp(words[3], "statistics")) {
+            if (!strcmp(words[4], "heal-count")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_STATISTICS_HEAL_COUNT);
+                goto done;
+            }
+        }
+
+        if (!strcmp(words[3], "granular-entry-heal")) {
+            if (!strcmp(words[4], "enable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE);
+                goto done;
+            } else if (!strcmp(words[4], "disable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE);
+                goto done;
+            }
+        }
+
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 6) {
+        if (strcmp(words[3], "split-brain")) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(words[4], "bigger-file")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[4], "latest-mtime")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
                 goto out;
+            goto done;
         }
+        if (!strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 7) {
+        if (!strcmp(words[3], "statistics") &&
+            !strcmp(words[4], "heal-count") && !strcmp(words[5], "replica")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(
+                words[6], dict, GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[3], "split-brain") &&
+            !strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[6]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+    }
+    ret = -1;
+    goto out;
+done:
+    *options = dict;
+out:
+    if (ret && dict) {
+        dict_unref(dict);
+        *options = NULL;
+    }
 
-        op = (char *) words[op_index];
+    return ret;
+}
 
-        if (!strcasecmp ("start", op)) {
-                replace_op = GF_REPLACE_OP_START;
-        } else if (!strcasecmp ("commit", op)) {
-                replace_op = GF_REPLACE_OP_COMMIT;
-        } else if (!strcasecmp ("pause", op)) {
-                replace_op = GF_REPLACE_OP_PAUSE;
-        } else if (!strcasecmp ("abort", op)) {
-                replace_op = GF_REPLACE_OP_ABORT;
-        } else if (!strcasecmp ("status", op)) {
-                replace_op = GF_REPLACE_OP_STATUS;
+int
+cli_cmd_volume_defrag_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *option = NULL;
+    char *volname = NULL;
+    char *command = NULL;
+    gf_cli_defrag_type cmd = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if (!((wordcount == 4) || (wordcount == 5)))
+        goto out;
+
+    if (wordcount == 4) {
+        if (strcmp(words[3], "start") && strcmp(words[3], "stop") &&
+            strcmp(words[3], "status"))
+            goto out;
+    } else {
+        if (strcmp(words[3], "fix-layout") && strcmp(words[3], "start"))
+            goto out;
+    }
+
+    volname = (char *)words[2];
+
+    if (wordcount == 4) {
+        command = (char *)words[3];
+    }
+    if (wordcount == 5) {
+        if ((strcmp(words[3], "fix-layout") || strcmp(words[4], "start")) &&
+            (strcmp(words[3], "start") || strcmp(words[4], "force"))) {
+            ret = -1;
+            goto out;
         }
+        command = (char *)words[3];
+        option = (char *)words[4];
+    }
+
+    if (strcmp(command, "start") == 0) {
+        cmd = GF_DEFRAG_CMD_START;
+        if (option && strcmp(option, "force") == 0) {
+            cmd = GF_DEFRAG_CMD_START_FORCE;
+        }
+        goto done;
+    }
+
+    if (strcmp(command, "fix-layout") == 0) {
+        cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+        goto done;
+    }
+    if (strcmp(command, "stop") == 0) {
+        cmd = GF_DEFRAG_CMD_STOP;
+        goto done;
+    }
+    if (strcmp(command, "status") == 0) {
+        cmd = GF_DEFRAG_CMD_STATUS;
+    }
+
+done:
+    ret = dict_set_str(dict, "volname", volname);
+
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "rebalance-command", (int32_t)cmd);
+
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
+
+    *options = dict;
 
-        /* commit force option */
-        op_index = 6;
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        if (wordcount > (op_index + 1)) {
-                ret = -1;
-                goto out;
+    return ret;
+}
+
+int32_t
+cli_snap_create_desc_parse(dict_t *dict, const char **words, size_t wordcount,
+                           int32_t desc_opt_loc)
+{
+    int32_t ret = -1;
+    char *desc = NULL;
+    int32_t desc_len = 0;
+    int len;
+
+    desc = GF_MALLOC(MAX_SNAP_DESCRIPTION_LEN + 1, gf_common_mt_char);
+    if (!desc) {
+        ret = -1;
+        goto out;
+    }
+
+    len = strlen(words[desc_opt_loc]);
+    if (len >= MAX_SNAP_DESCRIPTION_LEN) {
+        cli_out(
+            "snapshot create: description truncated: "
+            "Description provided is longer than 1024 characters");
+        desc_len = MAX_SNAP_DESCRIPTION_LEN;
+    } else {
+        desc_len = len;
+    }
+
+    snprintf(desc, desc_len + 1, "%s", words[desc_opt_loc]);
+    /* Calculating the size of the description as given by the user */
+
+    ret = dict_set_dynstr(dict, "description", desc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to save snap "
+               "description");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret && desc)
+        GF_FREE(desc);
+
+    return ret;
+}
+
+/* Function to check whether the Volume name is repeated */
+int
+cli_check_if_volname_repeated(const char **words, unsigned int start_index,
+                              uint64_t cur_index)
+{
+    uint64_t i = -1;
+    int ret = 0;
+
+    GF_ASSERT(words);
+
+    for (i = start_index; i < cur_index; i++) {
+        if (strcmp(words[i], words[cur_index]) == 0) {
+            ret = -1;
+            goto out;
         }
+    }
+out:
+    return ret;
+}
 
-        if (wordcount == (op_index + 1)) {
-                op = (char *) words[op_index];
-                if (!strcasecmp ("force", op)) {
-                        replace_op = GF_REPLACE_OP_COMMIT_FORCE;
-                }
+/* snapshot clone <clonename> <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_clone_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    char *clonename = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount == cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR,
+               "Invalid number of  words for snap clone command");
+        goto out;
+    }
+
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot clone: failed: clonename cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Clone name too long");
+
+        goto out;
+    }
+
+    clonename = (char *)words[cmdi];
+    for (i = 0; i < strlen(clonename); i++) {
+        /* Following volume name convention */
+        if (!isalnum(clonename[i]) &&
+            (clonename[i] != '_' && (clonename[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Clonename can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
         }
+    }
+
+    ret = dict_set_int32(dict, "volcount", 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "clonename", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save clone "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling snap name in the dictionary */
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi + 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not "
+               "save snap name(%s)",
+               (char *)words[cmdi + 1]);
+        goto out;
+    }
+
+    ret = 0;
 
-        if (replace_op == GF_REPLACE_OP_NONE) {
-                ret = -1;
-                goto out;
+out:
+    return ret;
+}
+
+/* snapshot create <snapname> <vol-name(s)> [description <description>]
+ *                                           [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_create_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    uint64_t volcount = 0;
+    char key[PATH_MAX] = "";
+    char *snapname = NULL;
+    unsigned int cmdi = 2;
+    int flags = 0;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount <= cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Too less words for snap create command");
+        goto out;
+    }
+
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot create: failed: snapname cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Snapname too long");
+
+        goto out;
+    }
+
+    snapname = (char *)words[cmdi];
+    for (i = 0; i < strlen(snapname); i++) {
+        /* Following volume name convention */
+        if (!isalnum(snapname[i]) &&
+            (snapname[i] != '_' && (snapname[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Snapname can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save snap "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling volume name in the dictionary */
+    for (i = cmdi + 1;
+         i < wordcount && (strcmp(words[i], "description")) != 0 &&
+         (strcmp(words[i], "force") != 0) &&
+         (strcmp(words[i], "no-timestamp") != 0);
+         i++) {
+        volcount++;
+        /* volume index starts from 1 */
+        ret = snprintf(key, sizeof(key), "volname%" PRIu64, volcount);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "operation", (int32_t) replace_op);
+        ret = dict_set_str(dict, key, (char *)words[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not "
+                   "save volume name(%s)",
+                   (char *)words[i]);
+            goto out;
+        }
 
-        if (ret)
-                goto out;
+        if (i >= cmdi + 2) {
+            ret = -1;
+            cli_err(
+                "Creating multiple volume snapshot is not "
+                "supported as of now");
+            goto out;
+        }
+        /* TODO : remove this above condition check once
+         * multiple volume snapshot is supported */
+    }
+
+    if (volcount == 0) {
+        ret = -1;
+        cli_err("Please provide the volume name");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "volcount", volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    /* Verify how we got out of "for" loop,
+     * if it is by reaching wordcount limit then goto "out",
+     * because we need not parse for "description","force" and
+     * "no-timestamp" after this.
+     */
+    if (i == wordcount) {
+        goto out;
+    }
+
+    if (strcmp(words[i], "no-timestamp") == 0) {
+        ret = dict_set_int32n(dict, "no-timestamp", SLEN("no-timestamp"), 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "time-stamp option");
+        }
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+    }
+
+    if ((strcmp(words[i], "description")) == 0) {
+        ++i;
+        if (i > (wordcount - 1)) {
+            ret = -1;
+            cli_err("Please provide a description");
+            gf_log("cli", GF_LOG_ERROR, "Description not provided");
+            goto out;
+        }
 
+        ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save snap "
+                   "description");
+            goto out;
+        }
 
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+        /* point the index to next word.
+         * As description might be follwed by force option.
+         * Before that, check if wordcount limit is reached
+         */
+    }
+
+    if (strcmp(words[i], "force") == 0) {
+        flags = GF_CLI_FLAG_OP_FORCE;
+
+    } else {
+        ret = -1;
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    /* Check if the command has anything after "force" keyword */
+    if (++i < wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = 0;
 
+out:
+    if (ret == 0) {
+        /*Adding force flag in either of the case i.e force set
+         * or unset*/
+        ret = dict_set_int32(dict, "flags", flags);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "snap force option");
+        }
+    }
+    return ret;
+}
 
-        *options = dict;
+/* snapshot list [volname]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_list_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount < 2 || wordcount > 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    if (wordcount == 2) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save volname in dictionary");
+        goto out;
+    }
+out:
+    return ret;
+}
+
+/* snapshot info [(snapname |  volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_info_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
+
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
+
+    /* If 3rd word is not "volume", then it must
+     * be snapname.
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[cmdi]);
+            goto out;
+        }
 
+        /* Once snap name is parsed, if we encounter any other
+         * word then fail it. Invalid Syntax.
+         * example : snapshot info <snapname> word
+         */
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        cmd = GF_SNAP_INFO_TYPE_SNAP;
+        ret = 0;
+        goto out;
+        /* No need to continue the parsing once we
+         * get the snapname
+         */
+    }
+
+    /* If 3rd word is "volume", then check if next word
+     * is present. As, "snapshot info volume" is an
+     * invalid command.
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_INFO_TYPE_VOL;
 out:
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "type of snapshot info");
         }
+    }
+    return ret;
+}
 
-        return ret;
+/* snapshot restore <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_restore_parse(dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount != 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    question =
+        "Restore operation will replace the "
+        "original volume with the snapshotted volume. "
+        "Do you still want to continue?";
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "User cancelled a snapshot "
+               "restore operation for snap %s",
+               (char *)words[2]);
+        goto out;
+    }
+out:
+    return ret;
 }
 
-int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
+/* snapshot activate <snapname> [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_activate_parse(dict_t *dict, const char **words, int wordcount)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    int ret = -1;
+    int flags = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            flags = GF_CLI_FLAG_OP_FORCE;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "Invalid option");
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save force option");
+        goto out;
+    }
+out:
+    return ret;
+}
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+/* snapshot deactivate <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancelled the request
+ */
+int
+cli_snap_deactivate_parse(dict_t *dict, const char **words, int wordcount,
+                          struct cli_state *state)
+{
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Deactivating snap will make its "
+        "data inaccessible. Do you want to "
+        "continue?";
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount != 3)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled "
+               "snapshot deactivate operation");
+        goto out;
+    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "log")) == 0);
-        GF_ASSERT ((strcmp (words[2], "filename")) == 0);
+out:
+    return ret;
+}
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+/* snapshot delete (all | snapname | volume <volname>)
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancel the operation
+ */
+int
+cli_snap_delete_parse(dict_t *dict, const char **words, int wordcount,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    int32_t cmd = -1;
+    unsigned int cmdi = 2;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount <= cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    question =
+        "Deleting snap will erase all the information about "
+        "the snap. Do you still want to continue?";
+
+    if (strcmp(words[cmdi], "all") == 0) {
+        ret = 0;
+        cmd = GF_SNAP_DELETE_TYPE_ALL;
+    } else if (strcmp(words[cmdi], "volume") == 0) {
+        if (++cmdi == wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "volume name %s",
+                   words[wordcount - 1]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_VOL;
+    } else {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[2]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    }
+
+    if ((cmdi + 1) != wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot delete operation for snap %s",
+                   (char *)words[2]);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "sub-cmd", cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "type of snapshot delete");
+    }
+out:
+    return ret;
+}
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+/* snapshot status [(snapname | volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_status_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
+    int32_t cmd = GF_SNAP_STATUS_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
+
+    /* if 3rd word is not "volume", then it must be "snapname"
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Count not save "
+                   "snap name %s",
+                   words[cmdi]);
+            goto out;
+        }
 
-        str = (char *)words[4];
-        if (strchr (str, ':')) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
-                }
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
-                /* Path */
-                str = (char *)words[5];
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
-        } else {
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        *options = dict;
+        ret = 0;
+        cmd = GF_SNAP_STATUS_TYPE_SNAP;
+        goto out;
+    }
+
+    /* If 3rd word is "volume", then check if next word is present.
+     * As, "snapshot info volume" is an invalid command
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Count not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_STATUS_TYPE_VOL;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save cmd "
+                   "of snapshot status");
+        }
+    }
 
-        return ret;
+    return ret;
+}
+
+/* return value:
+ *      -1  in case of failure.
+ *       0  in case of success.
+ */
+int32_t
+cli_snap_config_limit_parse(const char **words, dict_t *dict,
+                            unsigned int wordcount, unsigned int index,
+                            char *key)
+{
+    int ret = -1;
+    int limit = 0;
+    char *end_ptr = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
+
+    if (index >= wordcount) {
+        ret = -1;
+        cli_err("Please provide a value for %s.", key);
+        gf_log("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+        goto out;
+    }
+
+    limit = strtol(words[index], &end_ptr, 10);
+
+    if (limit <= 0 || strcmp(end_ptr, "") != 0) {
+        ret = -1;
+        cli_err(
+            "Please enter an integer value "
+            "greater than zero for %s",
+            key);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, key, limit);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not set "
+               "%s in dictionary",
+               key);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "globalname", "All");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global key");
+        goto out;
+    }
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global locks");
+        goto out;
+    }
+
+out:
+    return ret;
 }
 
+/* function cli_snap_config_parse
+ * Config Syntax : gluster snapshot config [volname]
+ *                                         [snap-max-hard-limit <count>]
+ *                                         [snap-max-soft-limit <count>]
+ *
+   return value: <0  on failure
+                  1  if user cancels the operation, or limit value is out of
+                                                                      range
+                  0  on success
+
+  NOTE : snap-max-soft-limit can only be set for system.
+*/
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
+cli_snap_config_parse(const char **words, int wordcount, dict_t *dict,
+                      struct cli_state *state)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    gf_boolean_t vol_presence = _gf_false;
+    struct snap_config_opt_vals_ *conf_vals = NULL;
+    int8_t hard_limit = 0;
+    int8_t soft_limit = 0;
+    int8_t config_type = -1;
+    const char *question = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(state);
+
+    if ((wordcount < 2) || (wordcount > 7)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid wordcount(%d)", wordcount);
+        goto out;
+    }
+
+    if (wordcount == 2) {
+        config_type = GF_SNAP_CONFIG_DISPLAY;
+        ret = 0;
+        goto set;
+    }
+
+    /* auto-delete cannot be a volume name */
+    /* Check whether the 3rd word is volname */
+    if (strcmp(words[cmdi], "snap-max-hard-limit") != 0 &&
+        strcmp(words[cmdi], "snap-max-soft-limit") != 0 &&
+        strcmp(words[cmdi], "auto-delete") != 0 &&
+        strcmp(words[cmdi], "activate-on-create") != 0) {
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set volname");
+            goto out;
+        }
+        cmdi++;
+        vol_presence = _gf_true;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+        if (cmdi == wordcount) {
+            config_type = GF_SNAP_CONFIG_DISPLAY;
+            ret = 0;
+            goto set;
+        }
+    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "log")) == 0);
-        GF_ASSERT ((strcmp (words[2], "locate")) == 0);
+    config_type = GF_SNAP_CONFIG_TYPE_SET;
 
-        dict = dict_new ();
-        if (!dict)
+    if (strcmp(words[cmdi], "snap-max-hard-limit") == 0) {
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-hard-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config hard limit");
+            goto out;
+        }
+        hard_limit = 1;
+
+        if (++cmdi == wordcount) {
+            ret = 0;
+            goto set;
+        }
+    }
+
+    if (strcmp(words[cmdi], "snap-max-soft-limit") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "Soft limit cannot be set to individual "
+                "volumes.");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Soft limit cannot be "
+                   "set to volumes");
+            goto out;
+        }
+
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-soft-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config soft limit");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+        soft_limit = 1;
+    }
+
+    if (hard_limit || soft_limit)
+        goto set;
+
+    if (strcmp(words[cmdi], "auto-delete") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, auto-delete option cannot be set "
+                "to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "auto-delete option "
+                   "cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "auto-delete", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set "
+                   "value of auto-delete in request "
+                   "dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, activate-on-create option "
+                "cannot be set to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "activate-on-create "
+                   "option cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "snap-activate-on-create",
+                           (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set value "
+                   "of activate-on-create in request dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = 0; /* Success */
+
+set:
+    ret = dict_set_int32(dict, "config-command", config_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to set "
+               "config-command");
+        goto out;
+    }
+
+    if (config_type == GF_SNAP_CONFIG_TYPE_SET && (hard_limit || soft_limit)) {
+        conf_vals = snap_confopt_vals;
+        if (hard_limit && soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+        } else if (soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+        } else if (hard_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+        }
+
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot config operation");
+        }
+    }
+
+out:
+    return ret;
+}
+
+int
+validate_op_name(const char *op, const char *opname, char **opwords)
+{
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(opname);
+    GF_ASSERT(opwords);
+
+    for (i = 0; opwords[i] != NULL; i++) {
+        if (strcmp(opwords[i], opname) == 0) {
+            cli_out("\"%s\" cannot be a %s", opname, op);
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+int32_t
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_snapshot type = GF_SNAP_OPTION_TYPE_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"create",     "delete", "restore", "activate",
+                              "deactivate", "list",   "status",  "config",
+                              "info",       "clone",  NULL};
+    static char *invalid_snapnames[] = {"description", "force", "volume", "all",
+                                        NULL};
+    static char *invalid_volnames[] = {"volume",
+                                       "type",
+                                       "subvolumes",
+                                       "option",
+                                       "end-volume",
+                                       "all",
+                                       "volume_not_in_ring",
+                                       "description",
+                                       "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit",
+                                       "auto-delete",
+                                       "activate-on-create",
+                                       NULL};
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+    GF_ASSERT(state);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    /* Lowest wordcount possible */
+    if (wordcount < 2) {
+        gf_log("", GF_LOG_ERROR, "Invalid command: Not enough arguments");
+        goto out;
+    }
+
+    w = str_getunamb(words[1], opwords);
+    if (!w) {
+        /* Checks if the operation is a valid operation */
+        gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+        goto out;
+    }
+
+    if (!strcmp(w, "create")) {
+        type = GF_SNAP_OPTION_TYPE_CREATE;
+    } else if (!strcmp(w, "list")) {
+        type = GF_SNAP_OPTION_TYPE_LIST;
+    } else if (!strcmp(w, "info")) {
+        type = GF_SNAP_OPTION_TYPE_INFO;
+    } else if (!strcmp(w, "delete")) {
+        type = GF_SNAP_OPTION_TYPE_DELETE;
+    } else if (!strcmp(w, "config")) {
+        type = GF_SNAP_OPTION_TYPE_CONFIG;
+    } else if (!strcmp(w, "restore")) {
+        type = GF_SNAP_OPTION_TYPE_RESTORE;
+    } else if (!strcmp(w, "status")) {
+        type = GF_SNAP_OPTION_TYPE_STATUS;
+    } else if (!strcmp(w, "activate")) {
+        type = GF_SNAP_OPTION_TYPE_ACTIVATE;
+    } else if (!strcmp(w, "deactivate")) {
+        type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
+    } else if (!strcmp(w, "clone")) {
+        type = GF_SNAP_OPTION_TYPE_CLONE;
+    }
+
+    if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
+        type != GF_SNAP_OPTION_TYPE_STATUS) {
+        ret = dict_set_int32(dict, "hold_snap_locks", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to set hold-snap-locks value "
+                   "as _gf_true");
+            goto out;
+        }
+    }
+
+    /* Following commands does not require volume locks  */
+    if (type == GF_SNAP_OPTION_TYPE_STATUS ||
+        type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
+        type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
+        ret = dict_set_int32(dict, "hold_vol_locks", _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Setting volume lock "
+                   "flag failed");
+            goto out;
+        }
+    }
+
+    /* Check which op is intended */
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            /* Syntax :
+             * gluster snapshot create <snapname> <vol-name(s)>
+             *                         [no-timestamp]
+             *                         [description <description>]
+             *                         [force]
+             */
+            /* In cases where the snapname is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
                 goto out;
+            }
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+            ret = validate_op_name("snapname", words[2], invalid_snapnames);
+            if (ret) {
+                goto out;
+            }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
+            ret = cli_snap_create_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "create command parsing failed.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            /* Syntax :
+             * gluster snapshot clone <clonename> <snapname>
+             */
+            /* In cases where the clonename is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
                 goto out;
+            }
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
+            ret = validate_op_name("clonename", words[2], invalid_volnames);
+            if (ret) {
+                goto out;
+            }
+
+            ret = cli_snap_clone_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "clone command parsing failed.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_INFO:
+            /* Syntax :
+             * gluster snapshot info [(snapname] | [vol <volname>)]
+             */
+            ret = cli_snap_info_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot info command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_LIST:
+            /* Syntax :
+             * gluster snaphsot list [volname]
+             */
+
+            ret = cli_snap_list_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot list command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            /* Syntax :
+             * snapshot delete (all | snapname | volume <volname>)
+             */
+            ret = cli_snap_delete_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse "
+                           "snapshot delete command");
                 }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            /* snapshot config [volname]  [snap-max-hard-limit <count>]
+             *                            [snap-max-soft-limit <percent>] */
+            ret = cli_snap_config_parse(words, wordcount, dict, state);
+            if (ret) {
+                if (ret < 0)
+                    gf_log("cli", GF_LOG_ERROR,
+                           "config command parsing failed.");
+                goto out;
+            }
+
+            ret = dict_set_int32(dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Unable to set "
+                       "config type");
+                ret = -1;
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS: {
+            /* Syntax :
+             * gluster snapshot status [(snapname |
+             *                         volume <volname>)]
+             */
+            ret = cli_snap_status_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot status command");
+                goto out;
+            }
+            break;
         }
 
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            /* Syntax:
+             * snapshot restore <snapname>
+             */
+            ret = cli_snap_restore_parse(dict, words, wordcount, state);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "restore command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            /* Syntax:
+             * snapshot activate <snapname> [force]
+             */
+            ret = cli_snap_activate_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "start command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            /* Syntax:
+             * snapshot deactivate <snapname>
+             */
+            ret = cli_snap_deactivate_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse deactivate "
+                           "command");
+                }
+                goto out;
+            }
+            break;
+
+        default:
+            ret = -1;
+            gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+            goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Failed to set type.");
+        goto out;
+    }
+    /* If you got so far, input is valid */
+    ret = 0;
+out:
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+    } else
         *options = dict;
 
-out:
-        if (ret && dict)
-                dict_destroy (dict);
+    return ret;
+}
+
+int
+cli_cmd_validate_volume(char *volname)
+{
+    int i = 0;
+    int ret = -1;
+    int volname_len;
+
+    if (volname[0] == '-')
+        return ret;
+
+    if (!strcmp(volname, "all")) {
+        cli_err("\"all\" cannot be the name of a volume.");
+        return ret;
+    }
 
+    if (strchr(volname, '/')) {
+        cli_err("Volume name should not contain \"/\" character.");
         return ret;
+    }
+
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volname can not exceed %d characters.", GD_VOLUME_NAME_MAX);
+        return ret;
+    }
+
+    for (i = 0; i < volname_len; i++)
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            return ret;
+        }
+
+    ret = 0;
+
+    return ret;
 }
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    int32_t ret = -1;
+    char *w = NULL;
+    char *volname = NULL;
+    static char *opwords[] = {"enable",          "disable", "scrub-throttle",
+                              "scrub-frequency", "scrub",   "signing-time",
+                              "signer-threads",  NULL};
+    static char *scrub_throt_values[] = {"lazy", "normal", "aggressive", NULL};
+    static char *scrub_freq_values[] = {
+        "hourly", "daily", "weekly", "biweekly", "monthly", "minute", NULL};
+    static char *scrub_values[] = {"pause", "resume", "status", "ondemand",
+                                   NULL};
+    dict_t *dict = NULL;
+    gf_bitrot_type type = GF_BITROT_OPTION_TYPE_NONE;
+    int32_t expiry_time = 0;
+    int32_t signer_th_count = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /* Hack to print out bitrot help properly */
+    if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+        ret = 1;
+        return ret;
+    }
+
+    if (wordcount < 4 || wordcount > 5) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = cli_cmd_validate_volume(volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to validate volume name");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        cli_out("Failed to set volume name in dictionary ");
+        goto out;
+    }
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid bit rot option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!strcmp(w, "scrub-throttle")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing scrub-throttle value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_throt_values);
+            if (!w) {
+                cli_err(
+                    "Invalid scrub-throttle option for "
+                    "bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
+                ret = dict_set_str(dict, "scrub-throttle-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set scrub-throttle "
+                        "value in the dict");
+                    goto out;
+                }
+                goto set_type;
+            }
+        }
+    }
 
-        GF_ASSERT ((strcmp (words[0], "volume")) == 0);
-        GF_ASSERT ((strcmp (words[1], "log")) == 0);
-        GF_ASSERT ((strcmp (words[2], "rotate")) == 0);
+    if (!strcmp(words[3], "scrub-frequency")) {
+        if (!words[4]) {
+            cli_err("Missing scrub-frequency value");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_freq_values);
+            if (!w) {
+                cli_err("Invalid frequency option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
+                ret = dict_set_str(dict, "scrub-frequency-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
+                }
+                goto set_type;
+            }
+        }
+    }
+
+    if (!strcmp(words[3], "scrub")) {
+        if (!words[4]) {
+            cli_err("Missing scrub value for bitrot option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_values);
+            if (!w) {
+                cli_err("Invalid scrub option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                if (strcmp(words[4], "status") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_STATUS;
+                } else if (strcmp(words[4], "ondemand") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_ONDEMAND;
+                } else {
+                    type = GF_BITROT_OPTION_TYPE_SCRUB;
+                }
+                ret = dict_set_str(dict, "scrub-value", (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
+                }
+                goto set_type;
+            }
+        }
+    }
+
+    if (!strcmp(words[3], "signing-time")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signing-time value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
 
-        dict = dict_new ();
-        if (!dict)
+            expiry_time = strtol(words[4], NULL, 0);
+            if (expiry_time < 1) {
+                cli_err(
+                    "Expiry time  value should not be less"
+                    " than 1");
+                ret = -1;
                 goto out;
+            }
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+            ret = dict_set_uint32(dict, "expiry-time",
+                                  (unsigned int)expiry_time);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
+                goto out;
+            }
+            goto set_type;
+        }
+    } else if (!strcmp(words[3], "signer-threads")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signer-thread value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_SIGNER_THREADS;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
+            signer_th_count = strtol(words[4], NULL, 0);
+            if (signer_th_count < 1) {
+                cli_err("signer-thread count should not be less than 1");
+                ret = -1;
                 goto out;
+            }
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        cli_path_strip_trailing_slashes (delimiter + 1);
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+            ret = dict_set_uint32(dict, "signer-threads",
+                                  (unsigned int)signer_th_count);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
+                goto out;
+            }
+            goto set_type;
         }
+    } else {
+        cli_err(
+            "Invalid option %s for bitrot. Please enter valid "
+            "bitrot option",
+            words[3]);
+        ret = -1;
+        goto out;
+    }
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 
-        *options = dict;
+out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
+        if (dict)
+            dict_unref(dict);
+    }
 
+    return ret;
+}
+
+/* Parsing global option for NFS-Ganesha config
+ *  gluster nfs-ganesha enable/disable */
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char *w = NULL;
+    static char *opwords[] = {"enable", "disable", NULL};
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount != 2)
+        goto out;
+
+    key = (char *)words[0];
+    value = (char *)words[1];
+
+    if (!key || !value) {
+        cli_out("Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_strip_whitespace(value, strlen(value));
+    if (ret == -1)
+        goto out;
+
+    if (strcmp(key, "nfs-ganesha")) {
+        gf_asprintf(op_errstr,
+                    "Global option: error: ' %s '"
+                    "is not a valid global option.",
+                    key);
+        ret = -1;
+        goto out;
+    }
+
+    w = str_getunamb(value, opwords);
+    if (!w) {
+        cli_out(
+            "Invalid global option \n"
+            "Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        question =
+            "Enabling NFS-Ganesha requires Gluster-NFS to be "
+            "disabled across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else if (strcmp(value, "disable") == 0) {
+        question =
+            "Disabling NFS-Ganesha will tear down the entire "
+            "ganesha cluster across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else {
+        ret = -1;
+        goto out;
+    }
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Global operation "
+               "cancelled, exiting");
+        ret = -1;
+        goto out;
+    }
+    cli_out("This will take a few minutes to complete. Please wait ..");
+
+    ret = dict_set_str(dict, "key", key);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "value", value);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "globalname", "All");
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global"
+               " key failed.");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global key "
+               "failed.");
+        goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index ca89592d613..084998701d8 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -1,226 +1,317 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
+#include "cli1-xdr.h"
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+int
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount);
 
 int
-cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_peer_probe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (!(wordcount == 3)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!(wordcount == 3)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = valid_internet_address((char *)words[2], _gf_false, _gf_false);
+    if (ret == 1) {
+        ret = 0;
+    } else {
+        cli_out("%s is an invalid address", words[2]);
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_PROBE];
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer probe failed");
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    CLI_STACK_DESTROY(frame);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    if (ret == 0) {
+        gf_event(EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
+    }
 
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+int
+cli_cmd_peer_deprobe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int flags = 0;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    question =
+        "All clients mounted through the peer which is getting detached need "
+        "to be remounted using one of the other active peers in the trusted "
+        "storage pool to ensure client gets notification on any changes done "
+        "on the gluster configuration and if the same has been done do you "
+        "want to proceed?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
+
+    dict = dict_new();
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3]))
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret)
+        goto out;
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer probe failed");
-        }
-        return ret;
-}
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer detach failed");
+    }
 
+    CLI_STACK_DESTROY(frame);
+
+    if (ret == 0) {
+        gf_event(EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
+    }
+
+    return ret;
+}
 
 int
-cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_peer_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                   ret   = -1;
-        rpc_clnt_procedure_t *proc  = NULL;
-        call_frame_t         *frame = NULL;
-        dict_t               *dict  = NULL;
-        int                  sent = 0;
-        int                  parse_error = 0;
-
-        if (!(wordcount == 3) ) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_DEPROBE];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        dict = dict_new ();
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_PEERS);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer detach failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer status failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_pool_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, (char *)words[1] );
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_POOL_NODES);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer status failed");
-        }
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("pool list: command execution failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 struct cli_cmd cli_probe_cmds[] = {
-        { "peer probe <HOSTNAME>",
-          cli_cmd_peer_probe_cbk,
-          "probe peer specified by <HOSTNAME>"},
+    {"peer probe { <HOSTNAME> | <IP-address> }", cli_cmd_peer_probe_cbk,
+     "probe peer specified by <HOSTNAME>"},
 
-        { "peer detach <HOSTNAME>",
-          cli_cmd_peer_deprobe_cbk,
-          "detach peer specified by <HOSTNAME>"},
+    {"peer detach { <HOSTNAME> | <IP-address> } [force]",
+     cli_cmd_peer_deprobe_cbk, "detach peer specified by <HOSTNAME>"},
 
-        { "peer status",
-          cli_cmd_peer_status_cbk,
-          "list status of peers"},
+    {"peer status", cli_cmd_peer_status_cbk, "list status of peers"},
 
-	{ "peer help",
-           cli_cmd_peer_help_cbk,
-           "Help command for peer "},
+    {"peer help", cli_cmd_peer_help_cbk, "display help for peer commands"},
 
-        { NULL, NULL, NULL }
-};
+    {"pool list", cli_cmd_pool_list_cbk,
+     "list all the nodes in the pool (including localhost)"},
+
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *probe_cmd = NULL;
+    int count = 0;
+
+    cli_out("\ngluster peer commands");
+    cli_out("======================\n");
 
+    cmd = GF_MALLOC(sizeof(cli_probe_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_probe_cmds, sizeof(cli_probe_cmds));
+    count = (sizeof(cli_probe_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
+    for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
+        cli_out("%s - %s", probe_cmd->pattern, probe_cmd->desc);
 
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    GF_FREE(cmd);
 
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
 int
-cli_cmd_probe_register (struct cli_state *state)
+cli_cmd_probe_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd->pattern, cmd->cbk,
-                                        cmd->desc);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
new file mode 100644
index 00000000000..859d6b2e40d
--- /dev/null
+++ b/cli/src/cli-cmd-snapshot.c
@@ -0,0 +1,135 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include "cli.h"
+#include "cli-cmd.h"
+#include "cli-mem-types.h"
+
+int
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount);
+
+int
+cli_cmd_snapshot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
+{
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SNAP];
+
+    /* Parses the command entered by the user */
+    ret = cli_cmd_snapshot_parse(words, wordcount, &options, state);
+    if (ret) {
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+        } else {
+            /* User might have cancelled the snapshot operation */
+            ret = 0;
+        }
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+
+out:
+    if (ret && parse_err == 0)
+        cli_out("Snapshot command failed");
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+struct cli_cmd snapshot_cmds[] = {
+    {"snapshot help", cli_cmd_snapshot_help_cbk,
+     "display help for snapshot commands"},
+    {"snapshot create <snapname> <volname> [no-timestamp] "
+     "[description <description>] [force]",
+     cli_cmd_snapshot_cbk, "Snapshot Create."},
+    {"snapshot clone <clonename> <snapname>", cli_cmd_snapshot_cbk,
+     "Snapshot Clone."},
+    {"snapshot restore <snapname>", cli_cmd_snapshot_cbk, "Snapshot Restore."},
+    {"snapshot status [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Status."},
+    {"snapshot info [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Info."},
+    {"snapshot list [volname]", cli_cmd_snapshot_cbk, "Snapshot List."},
+    {"snapshot config [volname] ([snap-max-hard-limit <count>] "
+     "[snap-max-soft-limit <percent>]) "
+     "| ([auto-delete <enable|disable>])"
+     "| ([activate-on-create <enable|disable>])",
+     cli_cmd_snapshot_cbk, "Snapshot Config."},
+    {"snapshot delete (all | snapname | volume <volname>)",
+     cli_cmd_snapshot_cbk, "Snapshot Delete."},
+    {"snapshot activate <snapname> [force]", cli_cmd_snapshot_cbk,
+     "Activate snapshot volume."},
+    {"snapshot deactivate <snapname>", cli_cmd_snapshot_cbk,
+     "Deactivate snapshot volume."},
+    {NULL, NULL, NULL}};
+
+int
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *snap_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(snapshot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, snapshot_cmds, sizeof(snapshot_cmds));
+    count = (sizeof(snapshot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster snapshot commands");
+    cli_out("=========================\n");
+
+    for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
+        if (_gf_false == snap_cmd->disable)
+            cli_out("%s - %s", snap_cmd->pattern, snap_cmd->desc);
+    cli_out("\n");
+
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_snapshot_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index e92376cef8a..801e8f4efed 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -1,205 +1,624 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-#define GETSPEC_SYNTAX "system:: getspec <VOLID>"
-#define BRICKTOPORT_SYNTAX "system:: portmap brick2port <BRICK>"
-
-extern struct rpc_clnt *global_rpc;
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-extern rpc_clnt_prog_t *cli_rpc_prog;
+int
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
-int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                             const char **words, int wordcount);
+int
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount);
 
 int
-cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_getspec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volid", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+out:
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[2]);
+    }
 
-        if (wordcount != 3) {
-                cli_out ("Usage: " GETSPEC_SYNTAX);
-                goto out;
-        }
+    if (dict)
+        dict_unref(dict);
 
-        ret = dict_set_str (dict, "volid", (char *)words[2]);
-        if (ret)
-                goto out;
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_GETSPEC];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+int
+cli_cmd_pmap_b2p_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "brick", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[2]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[3]);
+    }
 
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_pmap_b2p_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_fsm_log_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *name = "";
+
+    if ((wordcount != 4) && (wordcount != 3)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    if (wordcount == 4)
+        name = (char *)words[3];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, (void *)name);
+    }
+out:
+    return ret;
+}
 
-        frame = create_frame (THIS, THIS->ctx->pool);
+int
+cli_cmd_getwd_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
         if (!frame)
-                goto out;
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
+out:
+    return ret;
+}
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+static dict_t *
+make_seq_dict(int argc, char **argv)
+{
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    int len;
+    int ret = 0;
+    dict_t *dict = dict_new();
+
+    if (!dict)
+        return NULL;
+
+    for (i = 0; i < argc; i++) {
+        len = snprintf(index, sizeof(index), "%d", i);
+        ret = dict_set_strn(dict, index, len, argv[i]);
+        if (ret == -1)
+            break;
+    }
+
+    if (ret) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+
+    return dict;
+}
 
-        if (wordcount != 4) {
-                cli_out ("Usage: " BRICKTOPORT_SYNTAX);
-                goto out;
-        }
+int
+cli_cmd_mount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
+{
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    void *dataa[] = {NULL, NULL};
+
+    if (wordcount < 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = make_seq_dict(wordcount - 3, (char **)words + 3);
+    if (!dict)
+        goto out;
+
+    dataa[0] = (void *)words[2];
+    dataa[1] = dict;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, dataa);
+    }
 
-        ret = dict_set_str (dict, "brick", (char *)words[3]);
-        if (ret)
-                goto out;
+out:
+    if (dict)
+        dict_unref(dict);
+
+    if (!proc && ret)
+        cli_out("Mount command failed");
+
+    return ret;
+}
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_PMAP_PORTBYBRICK];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+int
+cli_cmd_umount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                   const char **words, int wordcount)
+{
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!(wordcount == 3 ||
+          (wordcount == 4 && strcmp(words[3], "lazy") == 0))) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "path", (char *)words[2]);
+    if (ret != 0)
+        goto out;
+    ret = dict_set_int32(dict, "lazy", wordcount == 4);
+    if (ret != 0)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[3]);
-        }
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    if (!proc && ret)
+        cli_out("Umount command failed");
+
+    return ret;
 }
 
 int
-cli_cmd_fsm_log (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_uuid_get_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        char                            *name = "";
+    int ret = -1;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    if (proc->fn)
+        ret = proc->fn(frame, this, dict);
 
-        if ((wordcount != 3) && (wordcount != 2)) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid get failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
+
+int
+cli_cmd_uuid_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char *question = NULL;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    question =
+        "Resetting uuid changes the uuid of local glusterd. "
+        "Do you want to continue?";
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+    this = THIS;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    // send NULL as argument since no dictionary is sent to glusterd
+    if (proc->fn) {
+        ret = proc->fn(frame, this, dict);
+    }
 
-        if (wordcount == 3)
-                name = (char*)words[2];
-        proc = &cli_rpc_prog->proctable[GF1_CLI_FSM_LOG];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, (void*)name);
-        }
 out:
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid reset failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
-struct cli_cmd cli_system_cmds[] = {
-        { GETSPEC_SYNTAX,
-          cli_cmd_getspec_cbk,
-          "fetch spec for volume <VOLID>"},
+static struct cli_cmd cli_system_cmds[] = {
+    {"system:: getspec <VOLNAME>", cli_cmd_getspec_cbk,
+     "fetch the volume file for the volume <VOLNAME>"},
+
+    {"system:: portmap brick2port <BRICK>", cli_cmd_pmap_b2p_cbk,
+     "query which port <BRICK> listens on"},
+
+    {"system:: fsm log [<peer-name>]", cli_cmd_fsm_log_cbk,
+     "display fsm transitions"},
+
+    {"system:: getwd", cli_cmd_getwd_cbk, "query glusterd work directory"},
+
+    {"system:: mount <label> <args...>", cli_cmd_mount_cbk, "request a mount"},
+
+    {"system:: umount <path> [lazy]", cli_cmd_umount_cbk, "request an umount"},
 
-        { BRICKTOPORT_SYNTAX,
-          cli_cmd_pmap_b2p_cbk,
-          "query which port <BRICK> listens on"},
+    {"system:: uuid get", cli_cmd_uuid_get_cbk, "get uuid of glusterd"},
 
-        { "system:: help",
-           cli_cmd_system_help_cbk,
-           "display help for system commands"},
+    {"system:: uuid reset", cli_cmd_uuid_reset_cbk,
+     "reset the uuid of glusterd"},
 
-        { "fsm log [<peer-name>]",
-           cli_cmd_fsm_log,
-           "display fsm transitions"},
+    {"system:: help", cli_cmd_system_help_cbk,
+     "display help for system commands"},
 
-        { NULL, NULL, NULL }
-};
+    {"system:: copy file [<filename>]", cli_cmd_copy_file_cbk,
+     "Copy file from current node's $working_dir to "
+     "$working_dir of all cluster nodes"},
+
+    {"system:: execute <command> <args>", cli_cmd_sys_exec_cbk,
+     "Execute the command on all the nodes "
+     "in the cluster and display their output."},
+
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount)
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    char cmd_arg_name[PATH_MAX] = "";
+    char *command = NULL;
+    char *saveptr = NULL;
+    char *tmp = NULL;
+    int ret = -1;
+    int i = -1;
+    int len;
+    int cmd_args_count = 0;
+    int in_cmd_args_count = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if ((wordcount < 3) || (words[2] == NULL)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    command = strtok_r((char *)words[2], " ", &saveptr);
+    if (command == NULL) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to parse command");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    do {
+        tmp = strtok_r(NULL, " ", &saveptr);
+        if (tmp) {
+            in_cmd_args_count++;
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                     in_cmd_args_count);
+            ret = dict_set_str(dict, cmd_arg_name, tmp);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Unable to set "
+                       "%s in dict",
+                       cmd_arg_name);
+                goto out;
+            }
+        }
+    } while (tmp);
+
+    cmd_args_count = wordcount - 3;
+
+    ret = dict_set_str(dict, "command", command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set command in dict");
+        goto out;
+    }
+
+    for (i = 1; i <= cmd_args_count; i++) {
+        in_cmd_args_count++;
+        len = snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                       in_cmd_args_count);
+        ret = dict_set_strn(dict, cmd_arg_name, len, (char *)words[2 + i]);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Unable to set %s in dict", cmd_arg_name);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "cmd_args_count", in_cmd_args_count);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set cmd_args_count in dict");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+
+        /* proc->fn is processed synchronously, which means that the
+         * execution flow won't return here until the operation is
+         * fully processed, including any related callback. For this
+         * reason, it's safe to destroy the stack here, since no one
+         * can still be using it. Additionally, it's not easy to move
+         * the stack destroy to the callback executed after completion
+         * of the operation because there are multiple things than can
+         * fail even before having queued the callback, so we would
+         * still need to destroy the stack if proc->fn returns an
+         * error. */
+        CLI_STACK_DESTROY(frame);
+        dict = NULL;
+    }
+out:
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return ret;
 }
 
 int
-cli_cmd_system_register (struct cli_state *state)
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd->pattern, cmd->cbk,
-                                        cmd->desc);
-                if (ret)
-                        goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *filename = "";
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    filename = (char *)words[3];
+    ret = dict_set_str(dict, "source", filename);
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
         }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+    }
+out:
+    return ret;
+}
+
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *system_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(cli_system_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_system_cmds, sizeof(cli_system_cmds));
+    count = (sizeof(cli_system_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
+        cli_out("%s - %s", system_cmd->pattern, system_cmd->desc);
+
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_system_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 0b6e2dd3680..f238851586e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1,985 +1,3278 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <sys/socket.h>
+#include <netdb.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
+extern rpc_clnt_prog_t cli_quotad_clnt;
 
-extern rpc_clnt_prog_t *cli_rpc_prog;
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...);
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
 int
-cli_cmd_volume_info_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount);
+
+int
+cli_cmd_volume_info_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        cli_cmd_volume_get_ctx_t        ctx = {0,};
-        cli_local_t                     *local = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_cmd_volume_get_ctx_t ctx = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+
+    if ((wordcount == 2) || (wordcount == 3 && !strcmp(words[2], "all"))) {
+        ctx.flags = GF_CLI_GET_NEXT_VOLUME;
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
+    } else if (wordcount == 3) {
+        ctx.flags = GF_CLI_GET_VOLUME;
+        ctx.volname = (char *)words[2];
+        if (strlen(ctx.volname) > GD_VOLUME_NAME_MAX) {
+            cli_out("Invalid volume name");
+            goto out;
+        }
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+    } else {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        return -1;
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_GET_VOLUME];
+    local = cli_local_get();
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    if (!local)
+        goto out;
 
-        if ((wordcount == 2)  || (wordcount == 3 &&
-                                  !strcmp (words[2], "all"))) {
-                ctx.flags = GF_CLI_GET_NEXT_VOLUME;
-                proc = &cli_rpc_prog->proctable[GF1_CLI_GET_NEXT_VOLUME];
-        } else if (wordcount == 3) {
-                ctx.flags = GF_CLI_GET_VOLUME;
-                ctx.volname = (char *)words[2];
-                if (strlen (ctx.volname) > 1024) {
-                        cli_out ("Invalid volume name");
-                        goto out;
-                }
-                proc = &cli_rpc_prog->proctable[GF1_CLI_GET_VOLUME];
-        } else {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                return -1;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        local = cli_local_get ();
+    local->get_vol.flags = ctx.flags;
+    if (ctx.volname)
+        local->get_vol.volname = gf_strdup(ctx.volname);
 
-        if (!local)
-                goto out;
+    frame->local = local;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, &ctx);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting Volume information failed!");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
 
-        local->u.get_vol.flags = ctx.flags;
-        if (ctx.volname)
-                local->u.get_vol.volname = gf_strdup (ctx.volname);
+int
+cli_cmd_sync_volume_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Sync volume may make data "
+        "inaccessible while the sync "
+        "is in progress. Do you want "
+        "to continue?";
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    if ((wordcount == 3) || !strcmp(words[3], "all")) {
+        ret = dict_set_int32(dict, "flags", (int32_t)GF_CLI_SYNC_ALL);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set"
+                   "flag");
+            goto out;
+        }
+    } else {
+        ret = dict_set_str(dict, "volname", (char *)words[3]);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "volume");
+            goto out;
+        }
+    }
 
-        frame->local = local;
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set hostname");
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &ctx);
+    if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume sync failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+int
+cli_cmd_volume_create_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    char *trans_type = NULL;
+    char *bricks = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
+
+    ret = cli_cmd_volume_create_parse(state, words, wordcount, &options,
+                                      &bricks);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = dict_get_str(options, "transport", &trans_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
+        goto out;
+    }
+
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting Volume information failed!");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        return ret;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume create failed");
+    }
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_CREATE, "name=%s;bricks=%s", (char *)words[2],
+                 bricks);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_volume_delete_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        gf1_cli_sync_volume_req req = {0,};
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    question =
+        "Deleting volume will erase all information about the volume. "
+        "Do you want to continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    volname = (char *)words[2];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Deleting the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume delete failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
+    }
 
-        if ((wordcount == 3) || !strcmp(words[3], "all")) {
-                req.flags = GF_CLI_SYNC_ALL;
-                req.volname = "";
+    return ret;
+}
+
+int
+cli_cmd_volume_start_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    int flags = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (!words[2])
+        goto out;
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
         } else {
-                req.volname = (char *)words[3];
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        req.hostname = (char *)words[2];
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_SYNC_VOLUME];
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &req);
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume sync failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume start failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_START, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+gf_answer_t
+cli_cmd_get_confirmation(struct cli_state *state, const char *question)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    char answer[5] = {
+        '\0',
+    };
+    int flush = '\0';
+    size_t len;
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_CREATE_VOLUME];
+    if (state->mode & GLUSTER_MODE_SCRIPT)
+        return GF_ANSWER_YES;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    printf("%s (y/n) ", question);
 
-        ret = cli_cmd_volume_create_parse (words, wordcount, &options);
+    if (fgets(answer, 4, stdin) == NULL) {
+        cli_out("gluster cli read error");
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    len = strlen(answer);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (len && answer[len - 1] == '\n') {
+        answer[--len] = '\0';
+    } else {
+        do {
+            flush = getchar();
+        } while (flush != '\n');
+    }
+
+    if (len > 3)
+        goto out;
+
+    if (!strcasecmp(answer, "y") || !strcasecmp(answer, "yes"))
+        return GF_ANSWER_YES;
+
+    else if (!strcasecmp(answer, "n") || !strcasecmp(answer, "no"))
+        return GF_ANSWER_NO;
 
 out:
-        if (options)
-                dict_unref (options);
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume create failed");
-        }
+    cli_out("Invalid input, please enter y/n");
 
-        return ret;
+    return GF_ANSWER_NO;
 }
 
+int
+cli_cmd_volume_stop_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int flags = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    cli_local_t *local = NULL;
+
+    const char *question =
+        "Stopping volume will make its data inaccessible. "
+        "Do you want to continue?";
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    volname = (char *)words[2];
+
+    dict = dict_new();
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Stopping the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume stop on '%s' failed", volname);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    if (dict)
+        dict_unref(dict);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_STOP, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
+
+    return ret;
+}
 
 int
-cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_rename_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *volname = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        question = "Deleting volume will erase all information about the volume."
-                   "Do you want to continue?";
-        proc = &cli_rpc_prog->proctable[GF1_CLI_DELETE_VOLUME];
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "old-volname", (char *)words[2]);
 
-        answer = cli_cmd_get_confirmation (state, question);
+    if (ret)
+        goto out;
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_str(dict, "new-volname", (char *)words[3]);
 
-        volname = (char *)words[2];
+    if (ret)
+        goto out;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, volname);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
+
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume delete failed");
-        }
+    if (dict)
+        dict_unref(dict);
+
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rename on '%s' failed", (char *)words[2]);
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_volume_defrag_cbk(struct cli_state *state, struct cli_cmd_word *word,
                           const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        gf1_cli_start_vol_req    req = {0,};
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+#endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
 
-        if (wordcount < 3 || wordcount > 4) {
-               cli_usage_out (word->pattern);
-                parse_error = 1;
-               goto out;
-        }
+    ret = cli_cmd_volume_defrag_parse(words, wordcount, &dict);
 
-        req.volname = (char *)words[2];
-        if (!req.volname)
-                goto out;
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        req.flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_START_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &req);
-        }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume start failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rebalance failed");
+    } else {
+#if (USE_EVENTS)
+        if (!(strcmp(words[wordcount - 1], "start")) ||
+            !(strcmp(words[wordcount - 1], "force"))) {
+            event = EVENT_VOLUME_REBALANCE_START;
+        } else if (!strcmp(words[wordcount - 1], "stop")) {
+            event = EVENT_VOLUME_REBALANCE_STOP;
         }
 
-        return ret;
+        if (event != EVENT_LAST)
+            gf_event(event, "volume=%s", (char *)words[2]);
+#endif
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
-gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question)
+int
+cli_cmd_volume_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        char                    answer[5] = {'\0', };
-        char                    flush = '\0';
-	int			len = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    char *tmp_opt = NULL;
+#endif
 
-        if (state->mode & GLUSTER_MODE_SCRIPT)
-                return GF_ANSWER_YES;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
-	printf ("%s (y/n) ", question);
+    ret = cli_cmd_volume_reset_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-	fgets (answer, 4, stdin);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-	len = strlen (answer);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-	if (answer [len - 1] == '\n'){
-		answer [--len] = '\0';
-	} else {
-		do{
-			flush = getchar ();
-		}while (flush != '\n');
-	}
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-	if (len > 3)
-		goto out;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_str(options, "key", &tmp_opt);
+        if (ret1)
+            tmp_opt = "";
+
+        gf_event(EVENT_VOLUME_RESET, "name=%s;option=%s", (char *)words[2],
+                 tmp_opt);
+    }
+#endif
 
-	if (!strcasecmp (answer, "y") || !strcasecmp (answer, "yes"))
-		return GF_ANSWER_YES;
+    CLI_STACK_DESTROY(frame);
 
-	else if (!strcasecmp (answer, "n") || !strcasecmp (answer, "no"))
-		return GF_ANSWER_NO;
+    return ret;
+}
+
+int
+cli_cmd_volume_profile_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
+{
+    int sent = 0;
+    int parse_error = 0;
+
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_profile_parse(words, wordcount, &options);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-	cli_out ("Invalid input, please enter y/n");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume profile failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-	return GF_ANSWER_NO;
+    return ret;
 }
 
 int
-cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     flags   = 0;
-        gf1_cli_stop_vol_req    req = {0,};
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
+
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int i = 1;
+    char dict_key[50] = {
+        0,
+    };
+    char *tmp_opt = NULL;
+    char *opts_str = NULL;
+    int num_options = 0;
+#endif
 
-        const char *question = "Stopping volume will make its data inaccessible. "
-                               "Do you want to Continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_set_parse(state, words, wordcount, &options,
+                                   &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
 
-        if (wordcount < 3 || wordcount > 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+        parse_error = 1;
+        goto out;
+    }
 
-        req.volname = (char *)words[2];
-        if (!req.volname)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        answer = cli_cmd_get_confirmation (state, question);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume set failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0 && strcmp(words[2], "help") != 0) {
+        ret1 = dict_get_int32(options, "count", &num_options);
+        if (ret1) {
+            num_options = 0;
+            goto end;
+        } else {
+            num_options = num_options / 2;
         }
 
-        req.flags = flags;
-        proc = &cli_rpc_prog->proctable[GF1_CLI_STOP_VOLUME];
+        char *free_list_key[num_options];
+        char *free_list_val[num_options];
+        for (i = 0; i < num_options; i++) {
+            free_list_key[i] = NULL;
+            free_list_val[i] = NULL;
+        }
+        /* Initialize opts_str */
+        opts_str = "";
+
+        /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
+        for (i = 1; i <= num_options; i++) {
+            sprintf(dict_key, "key%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
+
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_key[i - 1] = opts_str;
+
+            sprintf(dict_key, "value%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
+
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_val[i - 1] = opts_str;
+        }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &req);
+        gf_event(EVENT_VOLUME_SET, "name=%s;options=%s", (char *)words[2],
+                 opts_str);
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        for (i = 0; i < num_options; i++) {
+            GF_FREE(free_list_key[i]);
+            GF_FREE(free_list_val[i]);
         }
+    }
+#endif
 
-out:
+end:
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+static int
+cli_event_remove_brick_str(dict_t *options, char **event_str,
+                           eventtypes_t *event)
+{
+    int ret = -1;
+    char *bricklist = NULL;
+    char *brick = NULL;
+    char *volname = NULL;
+    char key[256] = {
+        0,
+    };
+    const char *eventstrformat = "volume=%s;bricks=%s";
+    int32_t command = 0;
+    int32_t i = 1;
+    int32_t count = 0;
+    int32_t eventstrlen = 1;
+    int bricklen = 0;
+    char *tmp_ptr = NULL;
+
+    if (!options || !event_str || !event)
+        goto out;
+
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+    /* Get the list of bricks for the event */
+    ret = dict_get_int32(options, "command", &command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch command");
+        ret = -1;
+        goto out;
+    }
+
+    switch (command) {
+        case GF_OP_CMD_START:
+            *event = EVENT_VOLUME_REMOVE_BRICK_START;
+            break;
+        case GF_OP_CMD_COMMIT:
+            *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
+            break;
+        case GF_OP_CMD_COMMIT_FORCE:
+            *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
+            break;
+        case GF_OP_CMD_STOP:
+            *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
+            break;
+        default:
+            *event = EVENT_LAST;
+            break;
+    }
+
+    ret = -1;
+
+    if (*event == EVENT_LAST) {
+        goto out;
+    }
+
+    /* I could just get this from words[] but this is cleaner in case the
+     * format changes  */
+    while (i) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
         if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume stop failed", req.volname);
+            break;
         }
+        eventstrlen += strlen(brick) + 1;
+        i++;
+    }
 
-        return ret;
-}
+    count = --i;
+
+    eventstrlen += 1;
+
+    bricklist = GF_CALLOC(eventstrlen, sizeof(char), gf_common_mt_char);
+    if (!bricklist) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "memory allocation failed for"
+               "bricklist");
+        ret = -1;
+        goto out;
+    }
 
+    tmp_ptr = bricklist;
+
+    i = 1;
+    while (i <= count) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
+        if (ret) {
+            break;
+        }
+        snprintf(tmp_ptr, eventstrlen, "%s ", brick);
+        bricklen = strlen(brick);
+        eventstrlen -= (bricklen + 1);
+        tmp_ptr += (bricklen + 1);
+        i++;
+    }
+
+    if (!ret) {
+        gf_asprintf(event_str, eventstrformat, volname, bricklist);
+    } else {
+        gf_asprintf(event_str, eventstrformat, volname, "<unavailable>");
+    }
+
+    ret = 0;
+out:
+    GF_FREE(bricklist);
+    return ret;
+}
 
 int
-cli_cmd_volume_rename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_add_brick_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+
+#if (USE_EVENTS)
+    char *event_str = NULL;
+    char *bricks = NULL;
+    const char *eventstrformat = "volume=%s;bricks=%s";
+#endif
 
+    const char *question =
+        "Changing the 'stripe count' of the volume is "
+        "not a supported feature. In some cases it may result in data "
+        "loss on the volume. Also there may be issues with regular "
+        "filesystem operations on the volume after the change. Do you "
+        "really want to continue with 'stripe' count option ? ";
+
+    ret = cli_cmd_volume_add_brick_parse(state, words, wordcount, &options, 0);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    /* TODO: there are challenges in supporting changing of
+       stripe-count, until it is properly supported give warning to user */
+    if (dict_get(options, "stripe-count")) {
+        answer = cli_cmd_get_confirmation(state, question);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
+        }
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+#if (USE_EVENTS)
+    /* Get the list of bricks for the event */
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+    ret = dict_get_str(options, "bricks", &bricks);
+
+    if (!ret) {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    &bricks[1] /*Skip leading space*/);
+    } else {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    "<unavailable>");
+    }
+#endif
+
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        ret = dict_set_str (dict, "old-volname", (char *)words[2]);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
 
-        if (ret)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "new-volname", (char *)words[3]);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (ret)
-                goto out;
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_RENAME_VOLUME];
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume add-brick failed");
+    } else {
+#if (USE_EVENTS)
+        gf_event(EVENT_VOLUME_ADD_BRICK, "%s", event_str);
+#endif
+    }
+#if (USE_EVENTS)
+    GF_FREE(event_str);
+#endif
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
+
+int
+cli_get_soft_limit(dict_t *options, const char **words, dict_t *xdata)
+{
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    char *default_sl = NULL;
+    char *default_sl_dup = NULL;
+    int ret = -1;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    // We need a ref on @options to prevent CLI_STACK_DESTROY
+    // from destroying it prematurely.
+    dict_ref(options);
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    ret = proc->fn(frame, THIS, options);
+
+    ret = dict_get_str(options, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get default soft limit");
+        goto out;
+    }
+
+    default_sl_dup = gf_strdup(default_sl);
+    if (!default_sl_dup) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(xdata, "default-soft-limit", default_sl_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set default soft limit");
+        GF_FREE(default_sl_dup);
+        goto out;
+    }
 
 out:
-        if (dict)
-                dict_destroy (dict);
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rename failed", (char *)words[2]);
+/* Checks if at least one limit has been set on the volume
+ *
+ * Returns true if at least one limit is set. Returns false otherwise.
+ */
+gf_boolean_t
+_limits_set_on_volume(char *volname, int type)
+{
+    gf_boolean_t limits_set = _gf_false;
+    int ret = -1;
+    char quota_conf_file[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+    char buf[16] = {
+        0,
+    };
+    float version = 0.0f;
+    char gfid_type_stored = 0;
+    char gfid_type = 0;
+
+    /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
+     * to fetch working directory
+     */
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1)
+        goto out;
+
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
+    else
+        gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
+
+    /* Try to read at least one gfid  of type 'gfid_type' */
+    while (1) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type_stored, version);
+        if (ret <= 0)
+            break;
+
+        if (gfid_type_stored == gfid_type) {
+            limits_set = _gf_true;
+            break;
         }
+    }
+out:
+    if (fd != -1)
+        sys_close(fd);
 
-        return ret;
+    return limits_set;
 }
 
 int
-cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_quota_handle_list_all(const char **words, dict_t *options)
 {
-        int                   ret     = -1;
-        rpc_clnt_procedure_t *proc    = NULL;
-        call_frame_t         *frame   = NULL;
-        dict_t               *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int all_failed = 1;
+    int count = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *xdata = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+    char *volname = NULL;
+    char *volname_dup = NULL;
+    unsigned char buf[16] = {0};
+    int fd = -1;
+    char quota_conf_file[PATH_MAX] = {0};
+    gf_boolean_t xml_err_flag = _gf_false;
+    char err_str[NAME_MAX] = {
+        0,
+    };
+    int32_t type = 0;
+    char gfid_type = 0;
+    float version = 0.0f;
+    int32_t max_count = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
+
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota option type");
+        goto out;
+    }
+
+    ret = dict_set_int32(xdata, "type", type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set type in xdata");
+        goto out;
+    }
+
+    ret = cli_get_soft_limit(options, words, xdata);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to fetch default "
+               "soft-limit");
+        goto out;
+    }
+
+    /* Check if at least one limit is set on volume. No need to check for
+     * quota enabled as cli_get_soft_limit() handles that
+     */
+    if (!_limits_set_on_volume(volname, type)) {
+        snprintf(err_str, sizeof(err_str),
+                 "No%s quota configured on"
+                 " volume %s",
+                 (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode", volname);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            xml_err_flag = _gf_true;
+        } else {
+            cli_out("quota: %s", err_str);
+        }
+        ret = 0;
+        goto out;
+    }
+
+    volname_dup = gf_strdup(volname);
+    if (!volname_dup) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(xdata, "volume-uuid", volname_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
+        GF_FREE(volname_dup);
+        goto out;
+    }
+
+    // TODO: fix hardcoding; Need to perform an RPC call to glusterd
+    // to fetch working directory
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1) {
+        // This may because no limits were yet set on the volume
+        gf_log("cli", GF_LOG_TRACE,
+               "Unable to open "
+               "quota.conf");
+        ret = 0;
+        goto out;
+    }
+
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, xdata);
+    proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        max_count++;
+    }
+    ret = dict_set_int32(xdata, "max_count", max_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set max_count");
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0L, SEEK_SET);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "failed to move offset to "
+               "the beginning: %s",
+               strerror(errno));
+        goto out;
+    }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+        uuid_utoa_r(buf, gfid_str);
+        ret = dict_set_str(xdata, "gfid", gfid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set gfid");
+            goto out;
         }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dict, "command", (char *)words[3]);
-        if (ret)
-                goto out;
+        ret = proc->fn(frame, THIS, xdata);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to get quota "
+                   "limits for %s",
+                   uuid_utoa((unsigned char *)buf));
+        }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_DEFRAG_VOLUME];
+        dict_del(xdata, "gfid");
+        all_failed = all_failed && ret;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_end(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error in printing "
+                   "xml output");
+            goto out;
         }
+    }
 
-out:
-        if (dict)
-                dict_destroy (dict);
+    if (count > 0) {
+        ret = all_failed ? -1 : 0;
+    } else {
+        ret = 0;
+    }
 
+out:
+    if (xml_err_flag) {
+        ret = cli_xml_output_str("volQuota", NULL, -1, 0, err_str);
         if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rebalance failed");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error outputting in "
+                   "xml format");
         }
-
-        return ret;
+    }
+    if (xdata)
+        dict_unref(xdata);
+
+    if (fd != -1) {
+        sys_close(fd);
+    }
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch and display quota"
+               " limits");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_bitrot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                   const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    int parse_err = 0;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+#if (USE_EVENTS)
+    int cmd_type = -1;
+    int ret1 = -1;
+    int event_type = -1;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    char *volname = NULL;
+#endif
 
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
+    ret = cli_cmd_bitrot_parse(words, wordcount, &options);
+    if (ret < 0) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_RESET_VOLUME];
+    if (ret == 1) {
+        /* this is 'volume bitrot help' */
+        cli_cmd_bitrot_help_cbk(state, word, words, wordcount);
+        ret = 0;
+        goto out2;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (options)
-                dict_unref (options);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err(
+                "Bit rot command failed. Please check the cli "
+                "logs for more details");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+        else {
+            ret1 = dict_get_str(options, "volname", &volname);
+            if (ret1)
+                volname = "";
+        }
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset failed");
+        switch (cmd_type) {
+            case GF_BITROT_OPTION_TYPE_ENABLE:
+                event_type = EVENT_BITROT_ENABLE;
+                break;
+            case GF_BITROT_OPTION_TYPE_DISABLE:
+                event_type = EVENT_BITROT_DISABLE;
+                break;
+            case GF_BITROT_CMD_SCRUB_ONDEMAND:
+                event_type = EVENT_BITROT_SCRUB_ONDEMAND;
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+                event_type = EVENT_BITROT_SCRUB_THROTTLE;
+                ret1 = dict_get_str(options, "scrub-throttle-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+                event_type = EVENT_BITROT_SCRUB_FREQ;
+                ret1 = dict_get_str(options, "scrub-frequency-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB:
+                event_type = EVENT_BITROT_SCRUB_OPTION;
+                ret1 = dict_get_str(options, "scrub-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            default:
+                break;
         }
 
-        return ret;
+        if (event_type > -1)
+            gf_event(event_type, "%s", events_str);
 
-}
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
+    CLI_STACK_DESTROY(frame);
+out2:
+    return ret;
+}
 
 int
-cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_quota_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = 0;
+    int parse_err = 0;
+    int32_t type = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+    int sent = 0;
+    char *volname = NULL;
+    const char *question =
+        "Disabling quota will delete all the quota "
+        "configuration. Do you want to continue?";
+
+    // parse **words into options dictionary
+    if (strcmp(words[1], "inode-quota") == 0) {
+        ret = cli_cmd_inode_quota_parse(words, wordcount, &options);
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
+        }
+    } else {
+        ret = cli_cmd_quota_parse(words, wordcount, &options);
 
-	int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
+        if (ret == 1) {
+            cli_cmd_quota_help_cbk(state, word, words, wordcount);
+            ret = 0;
+            goto out;
+        }
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
+        }
+    }
+
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get opcode");
+        goto out;
+    }
+
+    // handle quota-disable and quota-list-all different from others
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+            answer = cli_cmd_get_confirmation(state, question);
+            if (answer == GF_ANSWER_NO)
+                goto out;
+            break;
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            if (wordcount != 4)
+                break;
+            ret = cli_cmd_quota_handle_list_all(words, options);
+            goto out;
+        default:
+            break;
+    }
+
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_SET_VOLUME];
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0 && parse_err == 0)
+            cli_out(
+                "Quota command failed. Please check the cli "
+                "logs for more details");
+    }
+    if (options)
+        dict_unref(options);
+
+    /* Events for Quota */
+    if (ret == 0) {
+        switch (type) {
+            case GF_QUOTA_OPTION_TYPE_ENABLE:
+                gf_event(EVENT_QUOTA_ENABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DISABLE:
+                gf_event(EVENT_QUOTA_DISABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                gf_event(EVENT_QUOTA_SET_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+                gf_event(EVENT_QUOTA_SET_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE:
+                gf_event(EVENT_QUOTA_REMOVE_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                gf_event(EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                gf_event(EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s", volname,
+                         words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                gf_event(EVENT_QUOTA_SOFT_TIMEOUT,
+                         "volume=%s;"
+                         "soft-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                gf_event(EVENT_QUOTA_HARD_TIMEOUT,
+                         "volume=%s;"
+                         "hard-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                gf_event(EVENT_QUOTA_DEFAULT_SOFT_LIMIT,
+                         "volume=%s;"
+                         "default-soft-limit=%s",
+                         volname, words[4]);
+                break;
+        }
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        ret = cli_cmd_volume_set_parse (words, wordcount, &options);
+int
+cli_cmd_volume_remove_brick_cbk(struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int brick_count = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int need_question = 0;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+    char *event_str = NULL;
+    int event_ret = -1;
+#endif
+    int32_t command = GF_OP_CMD_NONE;
+    char *question = NULL;
+
+    ret = cli_cmd_volume_remove_brick_parse(state, words, wordcount, &options,
+                                            &need_question, &brick_count,
+                                            &command);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (command == GF_OP_CMD_COMMIT_FORCE) {
+        question =
+            "Remove-brick force will not migrate files from the "
+            "removed bricks, so they will no longer be available"
+            " on the volume.\nDo you want to continue?";
+    } else if (command == GF_OP_CMD_START) {
+        question =
+            "It is recommended that remove-brick be run with"
+            " cluster.force-migration option disabled to prevent"
+            " possible data corruption. Doing so will ensure that"
+            " files that receive writes during migration will not"
+            " be migrated and will need to be manually copied"
+            " after the remove-brick commit operation. Please"
+            " check the value of the option and update accordingly."
+            " \nDo you want to continue with your current"
+            " cluster.force-migration settings?";
+    }
+
+    if (!brick_count) {
+        cli_err("No bricks specified");
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+
+#if (USE_EVENTS)
+    event_ret = cli_event_remove_brick_str(options, &event_str, &event);
+#endif
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Removing brick from the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+        need_question = _gf_true;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (options)
-                dict_unref (options);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume remove-brick failed");
+    }
+#if (USE_EVENTS)
+    if (!ret && !event_ret)
+        gf_event(event, "%s", event_str);
+    if (event_str)
+        GF_FREE(event_str);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume set failed");
-        }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    if (options)
+        dict_unref(options);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_add_brick_cbk (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
+cli_cmd_volume_reset_brick_cbk(struct cli_state *state,
+                               struct cli_cmd_word *word, const char **words,
+                               int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
+    if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+        ret = dict_set_int32(options, "ignore-partition", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set ignore-"
+                   "partition option");
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_ADD_BRICK];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-out:
-        if (options)
-                dict_unref (options);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume add-brick failed");
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset-brick failed");
+    } else {
+        if (wordcount > 5) {
+            gf_event(EVENT_BRICK_RESET_COMMIT,
+                     "Volume=%s;source-brick=%s;"
+                     "destination-brick=%s",
+                     (char *)words[2], (char *)words[3], (char *)words[4]);
+        } else {
+            gf_event(EVENT_BRICK_RESET_START, "Volume=%s;source-brick=%s",
+                     (char *)words[2], (char *)words[3]);
         }
+    }
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
+int
+cli_cmd_volume_replace_brick_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+
+    ret = cli_cmd_volume_replace_brick_parse(words, wordcount, &options);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume replace-brick failed");
+    } else {
+        gf_event(EVENT_BRICK_REPLACE,
+                 "Volume=%s;source-brick=%s;destination-brick=%s",
+                 (char *)words[2], (char *)words[3], (char *)words[4]);
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
 
 int
-cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
+cli_cmd_volume_set_transport_cbk(struct cli_state *state,
                                  struct cli_cmd_word *word, const char **words,
                                  int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    cli_cmd_broadcast_response(0);
+    return 0;
+}
 
-        const char *question = "Removing brick(s) can result in data loss. "
-                               "Do you want to Continue?";
+int
+cli_cmd_volume_top_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_top_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
 
-        answer = cli_cmd_get_confirmation (state, question);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_REMOVE_BRICK];
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume top failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+int
+cli_cmd_log_rotate_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!((wordcount == 4) || (wordcount == 5))) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (!(strcmp("rotate", words[3]) == 0)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
+
+    ret = cli_cmd_log_rotate_parse(words, wordcount, &options);
+    if (ret)
+        goto out;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume remove-brick failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume log rotate failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+#if (SYNCDAEMON_COMPILE)
+static int
+cli_check_gsync_present()
+{
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (-1 == ret) {
+        gf_log("", GF_LOG_WARNING,
+               "setenv syscall failed, hence could"
+               "not assert if geo-replication is installed");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_log("", GF_LOG_INFO, "geo-replication not installed");
+        goto out;
+    }
+
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        ret = -1;
+        goto out;
+    }
 
-        if (options)
-                dict_unref (options);
-        return ret;
+    ret = runner_end(&runner);
 
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "geo-replication not installed");
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret ? -1 : 0;
 }
 
+void
+cli_cmd_check_gsync_exists_cbk(struct cli_cmd *this)
+{
+    int ret = 0;
+
+    ret = cli_check_gsync_present();
+    if (ret)
+        this->disable = _gf_true;
+}
+#endif
+
 int
-cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words,
-                                  int wordcount)
+cli_cmd_volume_gsync_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char *errstr = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int cmd_type = -1;
+    int tmpi = 0;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    int event_type = -1;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_REPLACE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GSYNC_SET];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_gsync_set_parse(state, words, wordcount, &options, &errstr);
+    if (ret) {
+        if (errstr) {
+            cli_err("%s", errstr);
+            GF_FREE(errstr);
+        } else {
+            cli_usage_out(word->pattern);
+        }
+        parse_err = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+
+out:
+    if (ret && parse_err == 0)
+        cli_out(GEOREP " command failed");
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        events_str = gf_strdup("");
+
+        /* Type of Geo-rep Action - Create, Start etc */
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+
+        /* Only capture Events for modification commands */
+        switch (cmd_type) {
+            case GF_GSYNC_OPTION_TYPE_CREATE:
+                event_type = EVENT_GEOREP_CREATE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_START:
+                event_type = EVENT_GEOREP_START;
+                break;
+            case GF_GSYNC_OPTION_TYPE_STOP:
+                event_type = EVENT_GEOREP_STOP;
+                break;
+            case GF_GSYNC_OPTION_TYPE_PAUSE:
+                event_type = EVENT_GEOREP_PAUSE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_RESUME:
+                event_type = EVENT_GEOREP_RESUME;
+                break;
+            case GF_GSYNC_OPTION_TYPE_DELETE:
+                event_type = EVENT_GEOREP_DELETE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret1 = dict_get_str(options, "subop", &tmp);
+                if (ret1)
+                    tmp = "";
+
+                /* For Config Set additionally capture key and value */
+                /* For Config Reset capture key */
+                if (strcmp(tmp, "set") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_SET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+
+                    ret1 = dict_get_str(options, "op_value", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%svalue=%s;", events_str,
+                                       tmp);
+                } else if (strcmp(tmp, "del") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_RESET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+                }
+                break;
+            default:
+                break;
         }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+        if (event_type > -1) {
+            /* Capture all optional arguments used */
+            ret1 = dict_get_int32(options, "force", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sforce=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "push_pem", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%spush_pem=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "no_verify", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sno_verify=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "ssh_port", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sssh_port=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "reset-sync-time", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sreset_sync_time=%d;",
+                                   events_str, tmpi);
+            }
+            /* Capture Master and Slave Info */
+            ret1 = dict_get_str(options, "master", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%smaster=%s;", events_str, tmp);
+
+            ret1 = dict_get_str(options, "slave", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%sslave=%s", events_str, tmp);
+
+            gf_event(event_type, "%s", events_str);
         }
 
+        /* Allocated by gf_strdup and gf_asprintf */
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+int
+cli_cmd_volume_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    uint32_t cmd = 0;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_status_parse(words, wordcount, &dict);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        /* for one volume or brick */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
+    } else {
+        /* volume status all or all detail */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+    }
+
+    if (!proc->fn) {
+        ret = -1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    ret = proc->fn(frame, THIS, dict);
+
 out:
-        if (options)
-                dict_unref (options);
+    CLI_STACK_DESTROY(frame);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume replace-brick failed");
-        }
+    return ret;
+}
+
+int
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status)
+{
+    uint64_t free = 0;
+    uint64_t total = 0;
+    char key[1024] = {0};
+    int ret = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free", i);
+    ret = dict_get_uint64(dict, key, &free);
+
+    status->free = gf_uint64_2human_readable(free);
+    if (!status->free)
+        goto out;
+
+    snprintf(key, sizeof(key), "brick%d.total", i);
+    ret = dict_get_uint64(dict, key, &total);
+
+    status->total = gf_uint64_2human_readable(total);
+    if (!status->total)
+        goto out;
+
+    snprintf(key, sizeof(key), "brick%d.device", i);
+    ret = dict_get_str(dict, key, &(status->device));
+    if (ret)
+        status->device = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.block_size", i);
+    ret = dict_get_uint64(dict, key, &(status->block_size));
+    if (ret) {
+        ret = 0;
+        status->block_size = 0;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.mnt_options", i);
+    ret = dict_get_str(dict, key, &(status->mount_options));
+    if (ret)
+        status->mount_options = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.fs_name", i);
+    ret = dict_get_str(dict, key, &(status->fs_name));
+    if (ret) {
+        ret = 0;
+        status->fs_name = NULL;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.inode_size", i);
+    ret = dict_get_str(dict, key, &(status->inode_size));
+    if (ret)
+        status->inode_size = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.total_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->total_inodes));
+    if (ret)
+        status->total_inodes = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->free_inodes));
+    if (ret) {
+        ret = 0;
+        status->free_inodes = 0;
+    }
 
-        return ret;
+out:
+    return ret;
 }
 
+void
+cli_print_detailed_status(cli_volume_status_t *status)
+{
+    cli_out("%-20s : %-20s", "Brick", status->brick);
+
+    if (status->online) {
+        cli_out("%-20s : %-20d", "TCP Port", status->port);
+        cli_out("%-20s : %-20d", "RDMA Port", status->rdma_port);
+    } else {
+        cli_out("%-20s : %-20s", "TCP Port", "N/A");
+        cli_out("%-20s : %-20s", "RDMA Port", "N/A");
+    }
+
+    cli_out("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
+    cli_out("%-20s : %-20s", "Pid", status->pid_str);
+
+    if (status->fs_name)
+        cli_out("%-20s : %-20s", "File System", status->fs_name);
+    else
+        cli_out("%-20s : %-20s", "File System", "N/A");
+
+    if (status->device)
+        cli_out("%-20s : %-20s", "Device", status->device);
+    else
+        cli_out("%-20s : %-20s", "Device", "N/A");
+
+    if (status->mount_options) {
+        cli_out("%-20s : %-20s", "Mount Options", status->mount_options);
+    } else {
+        cli_out("%-20s : %-20s", "Mount Options", "N/A");
+    }
+
+    if (status->inode_size) {
+        cli_out("%-20s : %-20s", "Inode Size", status->inode_size);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Size", "N/A");
+    }
+    if (status->free)
+        cli_out("%-20s : %-20s", "Disk Space Free", status->free);
+    else
+        cli_out("%-20s : %-20s", "Disk Space Free", "N/A");
+
+    if (status->total)
+        cli_out("%-20s : %-20s", "Total Disk Space", status->total);
+    else
+        cli_out("%-20s : %-20s", "Total Disk Space", "N/A");
+
+    if (status->total_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Inode Count",
+                status->total_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Count", "N/A");
+    }
+
+    if (status->free_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Free Inodes",
+                status->free_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Free Inodes", "N/A");
+    }
+}
 
 int
-cli_cmd_volume_set_transport_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words, int wordcount)
+cli_print_brick_status(cli_volume_status_t *status)
 {
-        cli_cmd_broadcast_response (0);
-        return 0;
+    int fieldlen = CLI_VOL_STATUS_BRICK_LEN;
+    int bricklen = 0;
+    char *p = NULL;
+    int num_spaces = 0;
+
+    p = status->brick;
+    bricklen = strlen(p);
+    while (bricklen > 0) {
+        if (bricklen > fieldlen) {
+            cli_out("%.*s", fieldlen, p);
+            p += fieldlen;
+            bricklen -= fieldlen;
+        } else {
+            num_spaces = (fieldlen - bricklen) + 1;
+            printf("%s", p);
+            while (num_spaces-- != 0)
+                printf(" ");
+            if (status->port || status->rdma_port) {
+                if (status->online)
+                    cli_out("%-10d%-11d%-8c%-5s", status->port,
+                            status->rdma_port, status->online ? 'Y' : 'N',
+                            status->pid_str);
+                else
+                    cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                            status->online ? 'Y' : 'N', status->pid_str);
+            } else
+                cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                        status->online ? 'Y' : 'N', status->pid_str);
+            bricklen = 0;
+        }
+    }
+
+    return 0;
 }
 
+#define NEEDS_GLFS_HEAL(op)                                                    \
+    ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) ||                         \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||                        \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||                               \
+     (op == GF_SHD_OP_INDEX_SUMMARY) || (op == GF_SHD_OP_SPLIT_BRAIN_FILES) || \
+     (op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||                           \
+     (op == GF_SHD_OP_HEAL_SUMMARY))
+
 int
-cli_cmd_log_filename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_launch_glfs_heal(int heal_op, dict_t *options)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    char buff[PATH_MAX] = {0};
+    runner_t runner = {0};
+    char *filename = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    char *out = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    ret = dict_get_str(options, "volname", &volname);
+    runner_add_args(&runner, GLFSHEAL_PREFIX "/glfsheal", volname, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    switch (heal_op) {
+        case GF_SHD_OP_INDEX_SUMMARY:
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "bigger-file", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "latest-mtime", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str(options, "heal-source-hostname", &hostname);
+            ret = dict_get_str(options, "heal-source-brickpath", &path);
+            runner_add_args(&runner, "source-brick", NULL);
+            runner_argprintf(&runner, "%s:%s", hostname, path);
+            if (dict_get_str(options, "file", &filename) == 0)
+                runner_argprintf(&runner, "%s", filename);
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            runner_add_args(&runner, "split-brain-info", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            runner_add_args(&runner, "granular-entry-heal-op", NULL);
+            break;
+        case GF_SHD_OP_HEAL_SUMMARY:
+            runner_add_args(&runner, "info-summary", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    if (global_state->mode & GLUSTER_MODE_GLFSHEAL_NOLOG)
+        runner_add_args(&runner, "--nolog", NULL);
+    ret = runner_start(&runner);
+    if (ret == -1)
+        goto out;
+    while ((
+        out = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO)))) {
+        printf("%s", out);
+    }
+    ret = runner_end(&runner);
 
-        if (!((wordcount == 5) || (wordcount == 6))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+out:
+    return ret;
+}
+
+int
+cli_cmd_volume_heal_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *options = NULL;
+    xlator_t *this = NULL;
+    cli_local_t *local = NULL;
+    int heal_op = 0;
+
+    this = THIS;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_heal_options_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    ret = dict_get_int32(options, "heal-op", &heal_op);
+    if (ret < 0)
+        goto out;
+    if (NEEDS_GLFS_HEAL(heal_op)) {
+        ret = cli_launch_glfs_heal(heal_op, options);
+        if (ret < 0)
+            goto out;
+        if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+            goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0) &&
+            !(global_state->mode & GLUSTER_MODE_XML)) {
+            cli_out("Volume heal failed.");
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_LOG_FILENAME];
+    if (options)
+        dict_unref(options);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    CLI_STACK_DESTROY(frame);
 
-        ret = cli_cmd_log_filename_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+int
+cli_cmd_volume_statedump_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (wordcount >= 3) {
+        ret = cli_cmd_volume_statedump_options_parse(words, wordcount,
+                                                     &options);
+        if (ret) {
+            parse_error = 1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error parsing "
+                   "statedump options");
+            cli_out("Error parsing options");
+            cli_usage_out(word->pattern);
         }
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (options)
-                dict_destroy (options);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume statedump failed");
+    }
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log filename failed");
-        }
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_log_locate_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_volume_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_LOG_LOCATE];
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0)
+            cli_out("Volume list failed");
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    CLI_STACK_DESTROY(frame);
 
-        ret = cli_cmd_log_locate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+int
+cli_cmd_volume_clearlocks_cbk(struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 7 || wordcount > 8) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_clrlks_opts_parse(words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Error parsing "
+               "clear-locks options");
+        cli_out("Error parsing options");
+        cli_usage_out(word->pattern);
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(options, "path", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (options)
-                dict_destroy (options);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume clear-locks failed");
+    }
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("getting log file location information failed");
-        }
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_barrier_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    options = dict_new();
+    if (!options) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(options, "barrier", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("Volume barrier failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
 
-        proc = &cli_rpc_prog->proctable[GF1_CLI_LOG_ROTATE];
+int
+cli_cmd_volume_getopt_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_err = 0;
+    cli_local_t *local = NULL;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-out:
-        if (options)
-                dict_destroy (options);
+    ret = dict_set_str(options, "key", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log rotate failed");
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        return ret;
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err("Volume get option failed");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
+/* This is a bit of a hack to display the help. The current bitrot cmd
+ * format does not work well when registering the cmds.
+ * Ideally the should have been of the form
+ * gluster volume bitrot <subcommand> <volumename> ...
+ */
+
+struct cli_cmd bitrot_cmds[] = {
+
+    {"volume bitrot help", cli_cmd_bitrot_help_cbk,
+     "display help for volume bitrot commands"},
+
+    {"volume bitrot <VOLNAME> {enable|disable}", NULL, /*cli_cmd_bitrot_cbk,*/
+     "Enable/disable bitrot for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> signing-time <time-in-secs>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Waiting time for an object after last fd is closed to start signing "
+     "process"},
+
+    {"volume bitrot <VOLNAME> signer-threads <count>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Number of signing process threads. Usually set to number of available "
+     "cores"},
+
+    {"volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the speed of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the frequency of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Pause/resume the scrubber for <VOLNAME>. Status displays the status of "
+     "the scrubber. ondemand starts the scrubber immediately."},
+
+    {"volume bitrot <VOLNAME> {enable|disable}\n"
+     "volume bitrot <VOLNAME> signing-time <time-in-secs>\n"
+     "volume bitrot <VOLNAME> signer-threads <count>\n"
+     "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive}\n"
+     "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}\n"
+     "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
+     cli_cmd_bitrot_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
+struct cli_cmd quota_cmds[] = {
+
+    /* Quota commands */
+    {"volume quota help", cli_cmd_quota_help_cbk,
+     "display help for volume quota commands"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}",
+     cli_cmd_quota_cbk, "Enable/disable and configure quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}",
+     cli_cmd_quota_cbk, "Set maximum size for <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}",
+     cli_cmd_quota_cbk,
+     "Set the maximum number of entries allowed in <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, "Set quota timeout for <VOLNAME>"},
+
+    {"volume inode-quota <VOLNAME> enable", cli_cmd_quota_cbk,
+     "Enable/disable inode-quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}\n"
+     "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}\n"
+     "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}\n"
+     "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, NULL},
+
+    {NULL, NULL, NULL}};
 
 struct cli_cmd volume_cmds[] = {
-        { "volume info [all|<VOLNAME>]",
-          cli_cmd_volume_info_cbk,
-          "list information of all volumes"},
+    {"volume help", cli_cmd_volume_help_cbk,
+     "display help for volume commands"},
+
+    {"volume info [all|<VOLNAME>]", cli_cmd_volume_info_cbk,
+     "list information of all volumes"},
 
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [transport <tcp|rdma>] <NEW-BRICK> ...",
-          cli_cmd_volume_create_cbk,
-          "create a new volume of specified type with mentioned bricks"},
+    {"volume create <NEW-VOLNAME> [stripe <COUNT>] "
+     "[[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] "
+     "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
+     "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> <TA-BRICK>"
+     "... [force]",
 
-        { "volume delete <VOLNAME>",
-          cli_cmd_volume_delete_cbk,
-          "delete volume specified by <VOLNAME>"},
+     cli_cmd_volume_create_cbk,
+     "create a new volume of specified type with mentioned bricks"},
 
-        { "volume start <VOLNAME> [force]",
-          cli_cmd_volume_start_cbk,
-          "start volume specified by <VOLNAME>"},
+    {"volume delete <VOLNAME>", cli_cmd_volume_delete_cbk,
+     "delete volume specified by <VOLNAME>"},
 
-        { "volume stop <VOLNAME> [force]",
-          cli_cmd_volume_stop_cbk,
-          "stop volume specified by <VOLNAME>"},
+    {"volume start <VOLNAME> [force]", cli_cmd_volume_start_cbk,
+     "start volume specified by <VOLNAME>"},
 
-        /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
-          cli_cmd_volume_rename_cbk,
-          "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
+    {"volume stop <VOLNAME> [force]", cli_cmd_volume_stop_cbk,
+     "stop volume specified by <VOLNAME>"},
 
-        { "volume add-brick <VOLNAME> <NEW-BRICK> ...",
-          cli_cmd_volume_add_brick_cbk,
-          "add brick to volume <VOLNAME>"},
+    /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
+      cli_cmd_volume_rename_cbk,
+      "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume remove-brick <VOLNAME> <BRICK> ...",
-          cli_cmd_volume_remove_brick_cbk,
-          "remove brick from volume <VOLNAME>"},
+    {"volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
+     "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
+     cli_cmd_volume_add_brick_cbk, "add brick to volume <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> {start|stop|status}",
-          cli_cmd_volume_defrag_cbk,
-          "rebalance operations"},
+    {"volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
+     " <start|stop|status|commit|force>",
+     cli_cmd_volume_remove_brick_cbk, "remove brick from volume <VOLNAME>"},
 
-        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start|pause|abort|status}",
-          cli_cmd_volume_replace_brick_cbk,
-          "replace-brick operations"},
+    {"volume rebalance <VOLNAME> {{fix-layout start} | {start "
+     "[force]|stop|status}}",
+     cli_cmd_volume_defrag_cbk, "rebalance operations"},
 
-        /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ...",
-          cli_cmd_volume_set_transport_cbk,
-          "set transport type for volume <VOLNAME>"},*/
+    {"volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
+     "{commit force}",
+     cli_cmd_volume_replace_brick_cbk, "replace-brick operations"},
 
-        { "volume set <VOLNAME> <KEY> <VALUE>",
-          cli_cmd_volume_set_cbk,
-         "set options for volume <VOLNAME>"},
+    /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>]
+      ...", cli_cmd_volume_set_transport_cbk, "set transport type for volume
+      <VOLNAME>"},*/
 
-        { "volume help",
-          cli_cmd_volume_help_cbk,
-          "display help for the volume command"},
+    {"volume set <VOLNAME> <KEY> <VALUE>", cli_cmd_volume_set_cbk,
+     "set options for volume <VOLNAME>"},
 
-        { "volume log filename <VOLNAME> [BRICK] <PATH>",
-          cli_cmd_log_filename_cbk,
-         "set the log file for corresponding volume/brick"},
+    {"volume set <VOLNAME> group <GROUP>", cli_cmd_volume_set_cbk,
+     "This option can be used for setting multiple pre-defined volume options "
+     "where group_name is a file under /var/lib/glusterd/groups containing one "
+     "key value pair per line"},
 
-        { "volume log locate <VOLNAME> [BRICK]",
-          cli_cmd_log_locate_cbk,
-         "locate the log file for corresponding volume/brick"},
+    {"volume log <VOLNAME> rotate [BRICK]", cli_cmd_log_rotate_cbk,
+     "rotate the log file for corresponding volume/brick"},
 
-        { "volume log rotate <VOLNAME> [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"},
+    {"volume sync <HOSTNAME> [all|<VOLNAME>]", cli_cmd_sync_volume_cbk,
+     "sync the volume information from a peer"},
 
-        { "volume sync <HOSTNAME> [all|<VOLNAME>]",
-          cli_cmd_sync_volume_cbk,
-         "sync the volume information from a peer"},
+    {"volume reset <VOLNAME> [option] [force]", cli_cmd_volume_reset_cbk,
+     "reset all the reconfigured options"},
 
-         { "volume reset <VOLNAME> ",
-         cli_cmd_volume_reset_cbk,
-         "reset all the reconfigured options"},
+#if (SYNCDAEMON_COMPILE)
+    {"volume " GEOREP " [<MASTER-VOLNAME>] [<SLAVE-IP>]::[<SLAVE-VOLNAME>] {"
+     "\\\n create [[ssh-port n] [[no-verify] \\\n | [push-pem]]] [force] \\\n"
+     " | start [force] \\\n | stop [force] \\\n | pause [force] \\\n | resume "
+     "[force] \\\n"
+     " | config [[[\\!]<option>] [<value>]] \\\n | status "
+     "[detail] \\\n | delete [reset-sync-time]} ",
+     cli_cmd_volume_gsync_set_cbk, "Geo-sync operations",
+     cli_cmd_check_gsync_exists_cbk},
+#endif
 
-        { NULL, NULL, NULL }
-};
+    {"volume profile <VOLNAME> {start|info [peek|incremental "
+     "[peek]|cumulative|clear]|stop} [nfs]",
+     cli_cmd_volume_profile_cbk, "volume profile operations"},
+
+    {"volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick "
+     "<brick>] [list-cnt <value>] | "
+     "{read-perf|write-perf} [bs <size> count <count>] "
+     "[brick <brick>] [list-cnt <value>]",
+     cli_cmd_volume_top_cbk, "volume top operations"},
+
+    {"volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
+     " [detail|clients|mem|inode|fd|callpool|tasks|client-list]",
+     cli_cmd_volume_status_cbk,
+     "display status of all or specified volume(s)/brick"},
+
+    {"volume heal <VOLNAME> [enable | disable | full |"
+     "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
+     "info [summary | split-brain] |"
+     "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
+     "source-brick <HOSTNAME:BRICKNAME> [<FILE>]} |"
+     "granular-entry-heal {enable | disable}]",
+     cli_cmd_volume_heal_cbk,
+     "self-heal commands on volume specified by <VOLNAME>"},
+
+    {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+     "priv|fd|inode|history]... | [client <hostname:process-id>]]",
+     cli_cmd_volume_statedump_cbk, "perform statedump on bricks"},
+
+    {"volume list", cli_cmd_volume_list_cbk, "list all volumes in cluster"},
+
+    {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+     "{inode [range]|entry [basename]|posix [range]}",
+     cli_cmd_volume_clearlocks_cbk, "Clear locks held on path"},
+    {"volume barrier <VOLNAME> {enable|disable}", cli_cmd_volume_barrier_cbk,
+     "Barrier/unbarrier file operations on a volume"},
+    {"volume get <VOLNAME|all> <key|all>", cli_cmd_volume_getopt_cbk,
+     "Get the value of the all options or given option for volume <VOLNAME>"
+     " or all option. gluster volume get all all is to get all global "
+     "options"},
+
+    {"volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+     " {<NEW-BRICK> commit}}",
+     cli_cmd_volume_reset_brick_cbk, "reset-brick operations"},
+
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *quota_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(quota_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, quota_cmds, sizeof(quota_cmds));
+    count = (sizeof(quota_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster quota commands");
+    cli_out("=======================\n");
+
+    for (quota_cmd = cmd; quota_cmd->pattern; quota_cmd++)
+        if ((_gf_false == quota_cmd->disable) && (quota_cmd->desc))
+            cli_out("%s - %s", quota_cmd->pattern, quota_cmd->desc);
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++)
-                cli_out ("%s - %s", cmd->pattern, cmd->desc);
+    cli_out("\n");
+    GF_FREE(cmd);
 
-        return 0;
+    return 0;
 }
 
 int
-cli_cmd_volume_register (struct cli_state *state)
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *bitrot_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(bitrot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, bitrot_cmds, sizeof(bitrot_cmds));
+    count = (sizeof(bitrot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster bitrot commands");
+    cli_out("========================\n");
+
+    for (bitrot_cmd = cmd; bitrot_cmd->pattern; bitrot_cmd++)
+        if ((_gf_false == bitrot_cmd->disable) && (bitrot_cmd->desc))
+            cli_out("%s - %s", bitrot_cmd->pattern, bitrot_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
+}
+
+int
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *vol_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(volume_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, volume_cmds, sizeof(volume_cmds));
+    count = (sizeof(volume_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster volume commands");
+    cli_out("========================\n");
+
+    for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
+        if (_gf_false == vol_cmd->disable)
+            cli_out("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_volume_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = volume_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = bitrot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = quota_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd->pattern, cmd->cbk,
-                                        cmd->desc);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
+}
+
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...)
+{
+    va_list arg;
+    int rv = 0;
+    char *tmp = *string_ptr;
+
+    va_start(arg, format);
+    rv = gf_vasprintf(string_ptr, format, arg);
+    va_end(arg);
+
+    if (tmp)
+        GF_FREE(tmp);
+
+    return rv;
 }
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index 2b9aea191df..2d458b16a56 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -1,374 +1,411 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
+#include "protocol-common.h"
 
 #include <fnmatch.h>
 
 static int cmd_done;
 static int cmd_sent;
-static pthread_cond_t      cond  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     cond_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t      conn  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     conn_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t conn = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-int    cli_op_ret = 0;
-int    connected = 0;
+int cli_op_ret = 0;
+static gf_boolean_t connected = _gf_false;
 
-int cli_cmd_log_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
-
-static gf_boolean_t
-cli_cmd_needs_connection (struct cli_cmd_word *word)
+static unsigned
+cli_cmd_needs_connection(struct cli_cmd_word *word)
 {
-        if (!strcasecmp ("quit", word->word))
-                return _gf_false;
+    if (!strcasecmp("quit", word->word))
+        return 0;
 
-        if (!strcasecmp ("help", word->word))
-                return _gf_false;
+    if (!strcasecmp("help", word->word))
+        return 0;
+
+    if (!strcasecmp("getwd", word->word))
+        return 1;
+
+    if (!strcasecmp("exit", word->word))
+        return 0;
 
-        return _gf_true;
+    return cli_default_conn_timeout;
 }
 
 int
-cli_cmd_status_reset (void)
+cli_cmd_status_reset(void)
 {
-        int ret = 0;
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0) {
-                        cmd_sent = 0;
-                        cmd_done = 0;
-                }
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
 
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0) {
+            cmd_sent = 0;
+            cmd_done = 0;
+        }
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_sent_status_get (int *status)
+cli_cmd_sent_status_get(int *status)
 {
-        int ret = 0;
-        GF_ASSERT (status);
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0)
-                        *status = cmd_sent;
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
+    GF_ASSERT(status);
+
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0)
+            *status = cmd_sent;
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_process (struct cli_state *state, int argc, char **argv)
+cli_cmd_process(struct cli_state *state, int argc, char **argv)
 {
-        int                  ret = 0;
-        struct cli_cmd_word *word = NULL;
-        struct cli_cmd_word *next = NULL;
-        int                  i = 0;
-        gf_boolean_t         await_conn = _gf_false;
+    int ret = 0;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    int i = 0;
 
-        word = &state->tree.root;
+    word = &state->tree.root;
 
-        if (!argc)
-                return 0;
+    if (!argc)
+        return 0;
 
-        for (i = 0; i < argc; i++) {
-                next = cli_cmd_nextword (word, argv[i]);
+    for (i = 0; i < argc; i++) {
+        next = cli_cmd_nextword(word, argv[i]);
 
-                word = next;
-                if (!word)
-                        break;
+        word = next;
+        if (!word)
+            break;
 
-                if (word->cbkfn)
-                        break;
-        }
+        if (word->cbkfn)
+            break;
+    }
 
-        if (!word) {
-                cli_out ("unrecognized word: %s (position %d)",
-                         argv[i], i);
-                return -1;
-        }
+    if (!word) {
+        cli_out("unrecognized word: %s (position %d)\n", argv[i], i);
+        usage();
+        return -1;
+    }
 
-        if (!word->cbkfn) {
-                cli_out ("unrecognized command");
-                return -1;
-        }
+    if (!word->cbkfn) {
+        cli_out("unrecognized command\n");
+        usage();
+        return -1;
+    }
 
-	if ( strcmp (word->word,"help")==0 )
-		goto callback;
+    if (strcmp(word->word, "help") == 0)
+        goto callback;
 
-        await_conn = cli_cmd_needs_connection (word);
+    state->await_connected = cli_cmd_needs_connection(word);
 
-        if (await_conn) {
-                ret = cli_cmd_await_connected ();
-                if (ret) {
-                        cli_out ("Connection failed. Please check if gluster "
-                                  "daemon is operational.");
-                        gf_log ("", GF_LOG_NORMAL, "Exiting with: %d", ret);
-                        exit (ret);
-                }
-        }
+    ret = cli_cmd_await_connected(state->await_connected);
+    if (ret) {
+        cli_out(
+            "Connection failed. Please check if gluster "
+            "daemon is operational.");
+        gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+        exit(ret);
+    }
 
 callback:
-        ret = word->cbkfn (state, word, (const char **)argv, argc);
-        (void) cli_cmd_status_reset ();
-        return ret;
+    ret = word->cbkfn(state, word, (const char **)argv, argc);
+    (void)cli_cmd_status_reset();
+    return ret;
 }
 
 int
-cli_cmd_input_token_count (const char *text)
+cli_cmd_input_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-
 int
-cli_cmd_process_line (struct cli_state *state, const char *text)
+cli_cmd_process_line(struct cli_state *state, const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-        int     ret = -1;
-
-        count = cli_cmd_input_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return -1;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
-
-        ret = cli_cmd_process (state, count, tokens);
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+    int ret = -1;
+
+    count = cli_cmd_input_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return -1;
+
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    ret = cli_cmd_process(state, count, tokens);
 out:
-        if (copy)
-                free (copy);
+    free(copy);
 
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmds_register (struct cli_state *state)
+cli_cmds_register(struct cli_state *state)
 {
-        int  ret = 0;
-
-        ret = cli_cmd_volume_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_probe_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_system_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_misc_register (state);
-        if (ret)
-                goto out;
-
+    int ret = 0;
+
+    ret = cli_cmd_volume_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_probe_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_system_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_misc_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_snapshot_register(state);
+    if (ret)
+        goto out;
+    ret = cli_cmd_global_register(state);
+    if (ret)
+        goto out;
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_cond_init ()
+cli_cmd_lock()
 {
-
-       pthread_mutex_init (&cond_mutex, NULL);
-       pthread_cond_init (&cond, NULL);
-
-       pthread_mutex_init (&conn_mutex, NULL);
-       pthread_cond_init (&conn, NULL);
-
-       return 0;
+    pthread_mutex_lock(&cond_mutex);
+    return 0;
 }
 
 int
-cli_cmd_lock ()
+cli_cmd_unlock()
 {
-       pthread_mutex_lock (&cond_mutex);
-       return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
-int
-cli_cmd_unlock ()
+static void
+seconds_from_now(unsigned secs, struct timespec *ts)
 {
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    struct timeval tv = {
+        0,
+    };
+
+    gettimeofday(&tv, NULL);
+
+    ts->tv_sec = tv.tv_sec + secs;
+    ts->tv_nsec = tv.tv_usec * 1000;
 }
 
 int
-cli_cmd_await_response ()
+cli_cmd_await_response(unsigned time)
 {
-        struct  timespec        ts = {0,};
-        int                     ret = 0;
-
-        cli_op_ret = -1;
-
-        time (&ts.tv_sec);
-        ts.tv_sec += CLI_DEFAULT_CMD_TIMEOUT;
-        while (!cmd_done && !ret) {
-                ret = pthread_cond_timedwait (&cond, &cond_mutex,
-                                        &ts);
-        }
-
-        cmd_done = 0;
-
-        cli_cmd_unlock ();
-
-        if (ret)
-                return ret;
-
-        return cli_op_ret;
+    struct timespec ts = {
+        0,
+    };
+    int ret = 0;
+
+    cli_op_ret = -1;
+
+    seconds_from_now(time, &ts);
+    while (!cmd_done && !ret) {
+        ret = pthread_cond_timedwait(&cond, &cond_mutex, &ts);
+    }
+
+    if (!cmd_done) {
+        if (ret == ETIMEDOUT)
+            cli_out("Error : Request timed out");
+        else
+            cli_out("Error : Command returned with error code:%d", ret);
+    }
+    cmd_done = 0;
+
+    return cli_op_ret;
 }
 
+/* This function must be called _only_ after all actions associated with
+ * command processing is complete. Otherwise, gluster process may exit before
+ * reporting results to stdout/stderr. */
 int
-cli_cmd_broadcast_response (int32_t status)
+cli_cmd_broadcast_response(int32_t status)
 {
-
-        pthread_mutex_lock (&cond_mutex);
-        {
-                if (!cmd_sent)
-                        goto out;
-                cmd_done = 1;
-                cli_op_ret = status;
-                pthread_cond_broadcast (&cond);
-        }
-
+    pthread_mutex_lock(&cond_mutex);
+    {
+        if (!cmd_sent)
+            goto out;
+        cmd_done = 1;
+        cli_op_ret = status;
+        pthread_cond_broadcast(&cond);
+    }
 
 out:
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 int32_t
-cli_cmd_await_connected ()
+cli_cmd_await_connected(unsigned conn_timo)
 {
-        int32_t                 ret = 0;
-        struct  timespec        ts = {0,};
-
-        pthread_mutex_lock (&conn_mutex);
-        {
-                time (&ts.tv_sec);
-                ts.tv_sec += CLI_DEFAULT_CONN_TIMEOUT;
-                while (!connected && !ret) {
-                        ret = pthread_cond_timedwait (&conn, &conn_mutex,
-                                                      &ts);
-                }
-        }
-        pthread_mutex_unlock (&conn_mutex);
+    int32_t ret = 0;
+    struct timespec ts = {
+        0,
+    };
 
+    if (!conn_timo)
+        return 0;
+
+    pthread_mutex_lock(&conn_mutex);
+    {
+        seconds_from_now(conn_timo, &ts);
+        while (!connected && !ret) {
+            ret = pthread_cond_timedwait(&conn, &conn_mutex, &ts);
+        }
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_broadcast_connected ()
+cli_cmd_broadcast_connected(gf_boolean_t status)
 {
-        pthread_mutex_lock (&conn_mutex);
-        {
-                connected = 1;
-                pthread_cond_broadcast (&conn);
-        }
+    pthread_mutex_lock(&conn_mutex);
+    {
+        connected = status;
+        pthread_cond_broadcast(&conn);
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return 0;
+}
 
-        pthread_mutex_unlock (&conn_mutex);
+gf_boolean_t
+cli_cmd_connected(void)
+{
+    gf_boolean_t status;
 
-        return 0;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        status = connected;
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return status;
+}
+
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    unsigned timeout = 0;
+
+    if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+        (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
+        (GLUSTER_CLI_GANESHA == procnum))
+        timeout = cli_ten_minutes_timeout;
+    else
+        timeout = cli_default_conn_timeout;
+
+    cli_cmd_lock();
+    cmd_sent = 0;
+    ret = cli_submit_request(rpc, req, frame, prog, procnum, NULL, this, cbkfn,
+                             xdrproc);
+
+    if (!ret) {
+        cmd_sent = 1;
+        ret = cli_cmd_await_response(timeout);
+    }
+
+    cli_cmd_unlock();
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_cmd_submit (void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                cli_serialize_t sfunc, xlator_t *this,
-                fop_cbk_fn_t cbkfn)
+cli_cmd_pattern_cmp(void *a, void *b)
+{
+    struct cli_cmd *ia = NULL;
+    struct cli_cmd *ib = NULL;
+    int ret = 0;
+
+    ia = a;
+    ib = b;
+    if (strcmp(ia->pattern, ib->pattern) > 0)
+        ret = 1;
+    else if (strcmp(ia->pattern, ib->pattern) < 0)
+        ret = -1;
+    else
+        ret = 0;
+    return ret;
+}
+
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count)
 {
-        int     ret = -1;
-
-        cli_cmd_lock ();
-        cmd_sent = 0;
-        ret = cli_submit_request (req, frame, prog,
-                                  procnum, NULL, sfunc,
-                                  this, cbkfn);
-
-        if (!ret) {
-                cmd_sent = 1;
-                ret = cli_cmd_await_response ();
-        } else
-                cli_cmd_unlock ();
-
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_array_insertionsort(cmd, 1, count - 2, sizeof(struct cli_cmd),
+                           cli_cmd_pattern_cmp);
 }
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 4daf7fae39f..c1c068c7085 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -1,80 +1,129 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_CMD_H__
 #define __CLI_CMD_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <netdb.h>
 
 #include "cli.h"
-
-typedef enum {
-        GF_ANSWER_YES = 1,
-        GF_ANSWER_NO  = 2
-} gf_answer_t;
+#include <glusterfs/list.h>
+
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+
+#define CLI_LOCAL_INIT(local, words, frame, dictionary)                        \
+    do {                                                                       \
+        local = cli_local_get();                                               \
+                                                                               \
+        if (local) {                                                           \
+            local->words = words;                                              \
+            if (dictionary)                                                    \
+                local->dict = dictionary;                                      \
+            if (frame)                                                         \
+                frame->local = local;                                          \
+        }                                                                      \
+    } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        if (_frame) {                                                          \
+            if (_frame->local) {                                               \
+                gf_log("cli", GF_LOG_DEBUG,                                    \
+                       "frame->local "                                         \
+                       "is not NULL (%p)",                                     \
+                       _frame->local);                                         \
+                cli_local_wipe(_frame->local);                                 \
+                _frame->local = NULL;                                          \
+            }                                                                  \
+            STACK_DESTROY(_frame->root);                                       \
+        }                                                                      \
+    } while (0);
+
+typedef enum { GF_ANSWER_YES = 1, GF_ANSWER_NO = 2 } gf_answer_t;
 
 struct cli_cmd {
-        const char     *pattern;
-        cli_cmd_cbk_t  *cbk;
-        const char     *desc;
+    const char *pattern;
+    cli_cmd_cbk_t *cbk;
+    const char *desc;
+    cli_cmd_reg_cbk_t *reg_cbk; /* callback to check in runtime if the   *
+                                 * command should be enabled or disabled */
+    gf_boolean_t disable;
 };
 
 struct cli_cmd_volume_get_ctx_ {
-        char            *volname;
-        int             flags;
+    char *volname;
+    int flags;
 };
 
+typedef struct cli_profile_info_ {
+    uint64_t fop_hits;
+    double min_latency;
+    double max_latency;
+    double avg_latency;
+    char *fop_name;
+    double percentage_avg_latency;
+} cli_profile_info_t;
+
 typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
 
-int cli_cmd_volume_register (struct cli_state *state);
+int
+cli_cmd_volume_register(struct cli_state *state);
 
-int cli_cmd_probe_register (struct cli_state *state);
+int
+cli_cmd_probe_register(struct cli_state *state);
+
+int
+cli_cmd_system_register(struct cli_state *state);
+
+int
+cli_cmd_snapshot_register(struct cli_state *state);
 
-int cli_cmd_system_register (struct cli_state *state);
+int
+cli_cmd_global_register(struct cli_state *state);
 
-int cli_cmd_misc_register (struct cli_state *state);
+int
+cli_cmd_misc_register(struct cli_state *state);
 
-struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
-                                       const char *text);
-void cli_cmd_tokens_destroy (char **tokens);
+struct cli_cmd_word *
+cli_cmd_nextword(struct cli_cmd_word *word, const char *text);
+void
+cli_cmd_tokens_destroy(char **tokens);
 
-int cli_cmd_await_response ();
+int
+cli_cmd_await_response(unsigned time);
 
-int cli_cmd_broadcast_response (int32_t status);
+int
+cli_cmd_broadcast_response(int32_t status);
 
-int cli_cmd_cond_init ();
+int
+cli_cmd_lock();
 
-int cli_cmd_lock ();
+int
+cli_cmd_unlock();
 
-int cli_cmd_unlock ();
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 int
-cli_cmd_submit (void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                cli_serialize_t sfunc, xlator_t *this,
-                fop_cbk_fn_t cbkfn);
+cli_cmd_pattern_cmp(void *a, void *b);
+
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count);
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question);
-int cli_cmd_sent_status_get (int *status);
+cli_cmd_get_confirmation(struct cli_state *state, const char *question);
+int
+cli_cmd_sent_status_get(int *status);
+
+gf_boolean_t
+_limits_set_on_volume(char *volname, int type);
+
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 86e346641ff..b42b4dd86c2 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -1,38 +1,30 @@
 /*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define CLI_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum cli_mem_types_ {
-        cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
-        cli_mt_xlator_t,
-        cli_mt_xlator_cmdline_option_t,
-        cli_mt_char,
-        cli_mt_call_pool_t,
-        cli_mt_cli_local_t,
-        cli_mt_cli_get_vol_ctx_t,
-        cli_mt_end
+    cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
+    cli_mt_xlator_t,
+    cli_mt_xlator_cmdline_option_t,
+    cli_mt_char,
+    cli_mt_call_pool_t,
+    cli_mt_cli_local_t,
+    cli_mt_cli_get_vol_ctx_t,
+    cli_mt_append_str,
+    cli_mt_cli_cmd,
+    cli_mt_end
 
 };
 
diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c
new file mode 100644
index 00000000000..772b8f75bd9
--- /dev/null
+++ b/cli/src/cli-quotad-client.c
@@ -0,0 +1,146 @@
+/*
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "cli-quotad-client.h"
+
+int
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
+
+out:
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+
+    return ret;
+}
+
+int
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
+{
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    this = mydata;
+
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
+        }
+
+        case RPC_CLNT_DISCONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            break;
+        }
+
+        default:
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
+
+    return ret;
+}
+
+struct rpc_clnt *
+cli_quotad_clnt_init(xlator_t *this, dict_t *options)
+{
+    struct rpc_clnt *rpc = NULL;
+    int ret = -1;
+
+    ret = dict_set_nstrn(options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport-type", SLEN("transport-type"),
+                         "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport.socket.connect-path",
+                         SLEN("transport.socket.connect-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(rpc, cli_quotad_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    rpc_clnt_start(rpc);
+out:
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
+
+    return rpc;
+}
diff --git a/cli/src/cli-quotad-client.h b/cli/src/cli-quotad-client.h
new file mode 100644
index 00000000000..71a44e5916b
--- /dev/null
+++ b/cli/src/cli-quotad-client.h
@@ -0,0 +1,29 @@
+/*
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include "cli.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include "cli-cmd.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "protocol-common.h"
+#include "cli-mem-types.h"
+
+int
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+
+struct rpc_clnt *
+cli_quotad_clnt_init(xlator_t *this, dict_t *options);
+
+int
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data);
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index 5d1310858ec..7a38a0b882a 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -1,38 +1,23 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "event.h"
+#include <glusterfs/gf-event.h>
 
 #include <fnmatch.h>
 
@@ -42,357 +27,376 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 
-
 int
-cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            i = 0;
-        int            ret = 0;
-
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        printf ("\r");
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        for (i = 0; i <= strlen (state->prompt); i++)
-                printf (" ");
+    printf("\r%*s\r", (int)strlen(state->prompt), "");
 
-        printf ("\r");
+    ret = vprintf(fmt, ap);
 
-        ret = vprintf (fmt, ap);
+    printf("\n");
+    fflush(stdout);
 
-        printf ("\n");
-        fflush(stdout);
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
-
-        return ret;
+    return ret;
 }
 
+int
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap)
+{
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
+
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
+
+    fprintf(stderr, "\r%*s\r", (int)strlen(state->prompt), "");
+
+    ret = vfprintf(stderr, fmt, ap);
+
+    fprintf(stderr, "\n");
+    fflush(stderr);
+
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
+
+    return ret;
+}
 
 void
-cli_rl_process_line (char *line)
+cli_rl_process_line(char *line)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = global_state;
+    state = global_state;
 
-        state->rl_processing = 1;
-        {
-                ret = cli_cmd_process_line (state, line);
-                add_history (line);
-        }
-        state->rl_processing = 0;
-}
+    state->rl_processing = 1;
+    {
+        ret = cli_cmd_process_line(state, line);
+        if (ret)
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to process line");
 
+        add_history(line);
+    }
+    state->rl_processing = 0;
+}
 
-int
-cli_rl_stdin (int fd, int idx, void *data,
-              int poll_out, int poll_in, int poll_err)
+void
+cli_rl_stdin(int fd, int idx, int gen, void *data, int poll_out, int poll_in,
+             int poll_err, char event_thread_died)
 {
-        rl_callback_read_char ();
+    struct cli_state *state = NULL;
 
-        return 0;
-}
+    state = data;
+
+    rl_callback_read_char();
+
+    gf_event_handled(state->ctx->event_pool, fd, idx, gen);
 
+    return;
+}
 
 char *
-cli_rl_autocomplete_entry (const char *text, int times)
+cli_rl_autocomplete_entry(const char *text, int times)
 {
-        struct cli_state  *state = NULL;
-        char              *retp = NULL;
+    struct cli_state *state = NULL;
+    char *retp = NULL;
 
-        state = global_state;
+    state = global_state;
 
-        if (!state->matchesp)
-                return NULL;
+    if (!state->matchesp)
+        return NULL;
 
-        retp = *state->matchesp;
+    retp = *state->matchesp;
 
-        state->matchesp++;
+    state->matchesp++;
 
-        return retp ? strdup (retp) : NULL;
+    return retp ? strdup(retp) : NULL;
 }
 
-
 int
-cli_rl_token_count (const char *text)
+cli_rl_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        if (is_spc)
-                /* what needs to be autocompleted is a full
-                   new word, and not extend the last word
-                */
-                count++;
+    if (is_spc)
+        /* what needs to be autocompleted is a full
+           new word, and not extend the last word
+        */
+        count++;
 
-        return count;
+    return count;
 }
 
-
 char **
-cli_rl_tokenize (const char *text)
+cli_rl_tokenize(const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-
-        count = cli_rl_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return NULL;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    count = cli_rl_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return NULL;
 
-        if (i < count) {
-                /* symoblize that what needs to be autocompleted is
-                   the full set of possible nextwords, and not extend
-                   the last word
-                */
-                *tokenp = strdup ("");
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-        }
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    if (i < count) {
+        /* symbolize that what needs to be autocompleted is
+           the full set of possible nextwords, and not extend
+           the last word
+        */
+        *tokenp = strdup("");
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
 
 out:
-        if (copy)
-                free (copy);
+    free(copy);
 
-        if (i < count) {
-                cli_cmd_tokens_destroy (tokens);
-                tokens = NULL;
-        }
+    if (i < count) {
+        cli_cmd_tokens_destroy(tokens);
+        tokens = NULL;
+    }
 
-        return tokens;
+    return tokens;
 }
 
-
 char **
-cli_rl_get_matches (struct cli_state *state, struct cli_cmd_word *word,
-                    const char *text)
+cli_rl_get_matches(struct cli_state *state, struct cli_cmd_word *word,
+                   const char *text)
 {
-        char                 **matches = NULL;
-        char                 **matchesp = NULL;
-        struct cli_cmd_word  **next = NULL;
-        int                    count = 0;
-        int                    len = 0;
+    char **matches = NULL;
+    char **matchesp = NULL;
+    struct cli_cmd_word **next = NULL;
+    int count = 0;
+    int len = 0;
 
-        len = strlen (text);
+    len = strlen(text);
 
-        if (!word->nextwords)
-                return NULL;
+    if (!word->nextwords)
+        return NULL;
 
-        for (next = word->nextwords; *next; next++)
-                count++;
+    for (next = word->nextwords; *next; next++)
+        count++;
 
-        matches = calloc (count + 1, sizeof (*matches));
-        matchesp = matches;
+    matches = calloc(count + 1, sizeof(*matches));
+    matchesp = matches;
 
-        for (next = word->nextwords; *next; next++) {
-                if ((*next)->match) {
-                        continue;
-                }
+    for (next = word->nextwords; *next; next++) {
+        if ((*next)->match) {
+            continue;
+        }
 
-                if (strncmp ((*next)->word, text, len) == 0) {
-                        *matchesp = strdup ((*next)->word);
-                        matchesp++;
-                }
+        if (strncmp((*next)->word, text, len) == 0) {
+            *matchesp = strdup((*next)->word);
+            matchesp++;
         }
+    }
 
-        return matches;
+    return matches;
 }
 
-
 int
-cli_rl_autocomplete_prepare (struct cli_state *state, const char *text)
+cli_rl_autocomplete_prepare(struct cli_state *state, const char *text)
 {
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-        char                 **tokens = NULL;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        char                 **matches = NULL;
-
-        tokens = cli_rl_tokenize (text);
-        if (!tokens)
-                return 0;
-
-        word = &state->tree.root;
-
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!*(tokenp+1)) {
-                        /* last word */
-                        break;
-                }
-
-                next = cli_cmd_nextword (word, token);
-                word = next;
-                if (!word)
-                        break;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char **matches = NULL;
+
+    tokens = cli_rl_tokenize(text);
+    if (!tokens)
+        return 0;
+
+    word = &state->tree.root;
+
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!*(tokenp + 1)) {
+            /* last word */
+            break;
         }
 
+        next = cli_cmd_nextword(word, token);
+        word = next;
         if (!word)
-                goto out;
+            break;
+    }
+
+    if (!word || !token)
+        goto out;
 
-        matches = cli_rl_get_matches (state, word, token);
+    matches = cli_rl_get_matches(state, word, token);
 
-        state->matches = matches;
-        state->matchesp = matches;
+    state->matches = matches;
+    state->matchesp = matches;
 
 out:
-        cli_cmd_tokens_destroy (tokens);
-        return 0;
+    cli_cmd_tokens_destroy(tokens);
+    return 0;
 }
 
-
 int
-cli_rl_autocomplete_cleanup (struct cli_state *state)
+cli_rl_autocomplete_cleanup(struct cli_state *state)
 {
-        if (state->matches)
-                cli_cmd_tokens_destroy (state->matches);
+    if (state->matches)
+        cli_cmd_tokens_destroy(state->matches);
 
-        state->matches = NULL;
-        state->matchesp = NULL;
+    state->matches = NULL;
+    state->matchesp = NULL;
 
-        return 0;
+    return 0;
 }
 
-
 char **
-cli_rl_autocomplete (const char *text, int start, int end)
+cli_rl_autocomplete(const char *text, int start, int end)
 {
-        struct cli_state  *state = NULL;
-        char             **matches = NULL;
-        char               save = 0;
+    struct cli_state *state = NULL;
+    char **matches = NULL;
+    char save = 0;
 
-        state = global_state;
+    state = global_state;
 
-        /* hack to make the autocompletion code neater */
-        /* fake it as though the cursor is at the end of line */
+    /* hack to make the autocompletion code neater */
+    /* fake it as though the cursor is at the end of line */
 
-        save = rl_line_buffer[rl_point];
-        rl_line_buffer[rl_point] = 0;
+    save = rl_line_buffer[rl_point];
+    rl_line_buffer[rl_point] = 0;
 
-        cli_rl_autocomplete_prepare (state, rl_line_buffer);
+    cli_rl_autocomplete_prepare(state, rl_line_buffer);
 
-        matches = rl_completion_matches (text, cli_rl_autocomplete_entry);
+    matches = rl_completion_matches(text, cli_rl_autocomplete_entry);
 
-        cli_rl_autocomplete_cleanup (state);
+    cli_rl_autocomplete_cleanup(state);
 
-        rl_line_buffer[rl_point] = save;
+    rl_line_buffer[rl_point] = save;
 
-        return matches;
+    return matches;
 }
 
-
 static char *
-complete_none (const char *txt, int times)
+complete_none(const char *txt, int times)
 {
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_rl_input (void *_data)
+cli_rl_input(void *_data)
 {
-        struct cli_state *state = NULL;
-        char             *line = NULL;
+    struct cli_state *state = NULL;
+    char *line = NULL;
 
-        state = _data;
+    state = _data;
 
-        for (;;) {
-                line = readline (state->prompt);
-                if (!line)
-                        break;
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        line = readline(state->prompt);
+        if (!line)
+            exit(0);  // break;
 
-                cli_rl_process_line (line);
+        if (*line)
+            cli_rl_process_line(line);
 
-                free (line);
-        }
+        free(line);
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        int ret = 0;
-
-        rl_pre_input_hook = NULL;
-        rl_attempted_completion_function = cli_rl_autocomplete;
-        rl_completion_entry_function = complete_none;
-
-        if (!state->rl_async) {
-                ret = pthread_create (&state->input, NULL,
-                                      cli_rl_input, state);
-                if (ret == 0)
-                        state->rl_enabled = 1;
-                goto out;
-        }
+    int ret = 0;
+
+    rl_pre_input_hook = NULL;
+    rl_attempted_completion_function = cli_rl_autocomplete;
+    rl_completion_entry_function = complete_none;
 
-        ret = event_register (state->ctx->event_pool, 0, cli_rl_stdin, state,
-                              1, 0);
-        if (ret == -1)
-                goto out;
+    if (!state->rl_async) {
+        ret = pthread_create(&state->input, NULL, cli_rl_input, state);
+        if (ret == 0)
+            state->rl_enabled = 1;
+        goto out;
+    }
 
-        state->rl_enabled = 1;
-        rl_callback_handler_install (state->prompt, cli_rl_process_line);
+    ret = gf_event_register(state->ctx->event_pool, 0, cli_rl_stdin, state, 1,
+                            0, 0);
+    if (ret == -1)
+        goto out;
+
+    state->rl_enabled = 1;
+    rl_callback_handler_install(state->prompt, cli_rl_process_line);
 
 out:
-        return state->rl_enabled;
+    return state->rl_enabled;
 }
 
 #else /* HAVE_READLINE */
 
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        return 0;
+    return 0;
 }
 
 #endif /* HAVE_READLINE */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
new file mode 100644
index 00000000000..9b6b0c7fa50
--- /dev/null
+++ b/cli/src/cli-rpc-ops.c
@@ -0,0 +1,10949 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+/* Widths of various columns in top read/write-perf output
+ * Total width of top read/write-perf should be 80 chars
+ * including one space between column
+ */
+#define VOL_TOP_PERF_FILENAME_DEF_WIDTH 47
+#define VOL_TOP_PERF_FILENAME_ALT_WIDTH 44
+#define VOL_TOP_PERF_SPEED_WIDTH 4
+#define VOL_TOP_PERF_TIME_WIDTH 26
+
+#define INDENT_MAIN_HEAD "%-25s %s "
+
+#define RETURNING "Returning %d"
+#define XML_ERROR "Error outputting to xml"
+#define XDR_DECODE_FAIL "Failed to decode xdr response"
+#define DICT_SERIALIZE_FAIL "Failed to serialize to data to dictionary"
+#define DICT_UNSERIALIZE_FAIL "Failed to unserialize the dictionary"
+
+/* Do not show estimates if greater than this number */
+#define REBAL_ESTIMATE_SEC_UPPER_LIMIT (60 * 24 * 3600)
+#define REBAL_ESTIMATE_START_TIME 600
+
+#include "cli.h"
+#include <glusterfs/compat-errno.h>
+#include "cli-cmd.h"
+#include <sys/uio.h>
+#include <stdlib.h>
+#include <sys/mount.h>
+#include <glusterfs/compat.h>
+#include "cli-mem-types.h"
+#include <glusterfs/syscall.h>
+#include "glusterfs3.h"
+#include "portmap-xdr.h"
+#include <glusterfs/byte-order.h>
+
+#include <glusterfs/run.h>
+#include <glusterfs/events.h>
+
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
+
+rpc_clnt_prog_t cli_quotad_clnt;
+
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data);
+
+char *cli_vol_status_str[] = {
+    "Created",
+    "Started",
+    "Stopped",
+};
+
+char *cli_vol_task_status_str[] = {"not started",
+                                   "in progress",
+                                   "stopped",
+                                   "completed",
+                                   "failed",
+                                   "fix-layout in progress",
+                                   "fix-layout stopped",
+                                   "fix-layout completed",
+                                   "fix-layout failed",
+                                   "unknown"};
+
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data);
+
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data);
+
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref);
+
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict);
+
+static rpc_clnt_prog_t cli_handshake_prog = {
+    .progname = "cli handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+};
+
+static rpc_clnt_prog_t cli_pmap_prog = {
+    .progname = "cli portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+};
+
+static void
+gf_free_xdr_cli_rsp(gf_cli_rsp rsp)
+{
+    if (rsp.dict.dict_val) {
+        free(rsp.dict.dict_val);
+    }
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+}
+
+static void
+gf_free_xdr_getspec_rsp(gf_getspec_rsp rsp)
+{
+    if (rsp.spec) {
+        free(rsp.spec);
+    }
+    if (rsp.xdata.xdata_val) {
+        free(rsp.xdata.xdata_val);
+    }
+}
+
+static void
+gf_free_xdr_fsm_log_rsp(gf1_cli_fsm_log_rsp rsp)
+{
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+    if (rsp.fsm_log.fsm_log_val) {
+        free(rsp.fsm_log.fsm_log_val);
+    }
+}
+
+static int
+gf_cli_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+
+    if (rsp.op_errstr && rsp.op_errstr[0] != '\0') {
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        if (rsp.op_ret) {
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (!rsp.op_ret)
+        cli_out("peer probe: %s", msg);
+    else
+        cli_err("peer probe: failed: %s", msg);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_deprobe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to deprobe");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr[0] != '\0') {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            gf_log("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (!rsp.op_ret)
+        cli_out("peer detach: %s", msg);
+    else
+        cli_err("peer detach: failed: %s", msg);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_output_peer_hostnames(dict_t *dict, int count, const char *prefix)
+{
+    int ret = -1;
+    char key[512] = {
+        0,
+    };
+    int i = 0;
+    char *hostname = NULL;
+
+    cli_out("Other names:");
+    /* Starting from friend.hostname1, as friend.hostname0 will be the same
+     * as friend.hostname
+     */
+    for (i = 1; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
+        if (ret)
+            break;
+        cli_out("%s", hostname);
+        hostname = NULL;
+    }
+
+    return ret;
+}
+
+static int
+gf_cli_output_peer_status(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t i = 1;
+    char key[256] = {
+        0,
+    };
+    int keylen;
+    char *state = NULL;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+    int hostname_count = 0;
+
+    cli_out("Number of Peers: %d", count);
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
+
+        keylen = snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_strn(dict, key, keylen, &state);
+        if (ret)
+            goto out;
+
+        cli_out("\nHostname: %s\nUuid: %s\nState: %s (%s)", hostname_buf,
+                uuid_buf, state, connected_str);
+
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &hostname_count);
+        /* Print other addresses only if there are more than 1.
+         */
+        if ((ret == 0) && (hostname_count > 1)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = gf_cli_output_peer_hostnames(dict, hostname_count, key);
+            if (ret) {
+                gf_log("cli", GF_LOG_WARNING,
+                       "error outputting peer other names");
+                goto out;
+            }
+        }
+        i++;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+gf_cli_output_pool_list(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t hostname_len = 8; /*min len 8 chars*/
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+
+    if (count <= 0)
+        goto out;
+
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
+
+        ret = strlen(hostname_buf);
+        if (ret > hostname_len)
+            hostname_len = ret;
+
+        i++;
+    }
+
+    cli_out("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
+
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
+
+        cli_out("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
+                connected_str);
+        i++;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* function pointer for gf_cli_output_{pool_list,peer_status} */
+typedef int (*cli_friend_output_fn)(dict_t *, int);
+
+static int
+gf_cli_list_friends_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    const char *cmd = NULL;
+    cli_friend_output_fn friend_output_fn;
+    call_frame_t *frame = NULL;
+    unsigned long flags = 0;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    flags = (long)frame->local;
+
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        cmd = "pool list";
+        friend_output_fn = &gf_cli_output_pool_list;
+    } else {
+        cmd = "peer status";
+        friend_output_fn = &gf_cli_output_peer_status;
+    }
+
+    /* 'free' the flags set by gf_cli_list_friends */
+    frame->local = NULL;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to list: %d", rsp.op_ret);
+
+    if (!rsp.op_ret) {
+        if (!rsp.friends.friends_len) {
+            snprintf(msg, sizeof(msg), "%s: No peers present", cmd);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                                 msg);
+                if (ret)
+                    gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+            cli_err("%s", msg);
+            ret = 0;
+            goto out;
+        }
+
+        dict = dict_new();
+
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_unserialize(rsp.friends.friends_val, rsp.friends.friends_len,
+                               &dict);
+
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             msg);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
+
+        ret = dict_get_int32_sizen(dict, "count", &count);
+        if (ret) {
+            goto out;
+        }
+
+        ret = friend_output_fn(dict, count);
+        if (ret) {
+            goto out;
+        }
+    } else {
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             NULL);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        } else {
+            ret = -1;
+        }
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    if (ret)
+        cli_err("%s: failed", cmd);
+
+    cli_cmd_broadcast_response(ret);
+
+    if (dict)
+        dict_unref(dict);
+
+    if (rsp.friends.friends_val) {
+        free(rsp.friends.friends_val);
+    }
+    return ret;
+}
+
+static int
+gf_cli_get_state_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *daemon_name = NULL;
+    char *ofilepath = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", myframe, out);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Failed to get daemon state: %s", rsp.op_errstr);
+        else
+            cli_err(
+                "Failed to get daemon state. Check glusterd"
+                " log file for more details");
+    } else {
+        ret = dict_get_str_sizen(dict, "daemon", &daemon_name);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get daemon name");
+
+        ret = dict_get_str_sizen(dict, "ofilepath", &ofilepath);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get filepath");
+
+        if (daemon_name && ofilepath)
+            cli_out("%s state dumped to %s", daemon_name, ofilepath);
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    return ret;
+}
+
+static void
+cli_out_options(char *substr, char *optstr, char *valstr)
+{
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
+
+    ptr1 = substr;
+    ptr2 = optstr;
+
+    while (ptr1) {
+        /* Avoiding segmentation fault. */
+        if (!ptr2)
+            return;
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+    }
+
+    if (*ptr2 == '\0')
+        return;
+    cli_out("%s: %s", ptr2, valstr);
+}
+
+static int
+_gf_cli_output_volinfo_opts(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    int ret = 0;
+    char *key = NULL;
+    char *ptr = NULL;
+    data_t *value = NULL;
+
+    key = tmp;
+
+    ptr = strstr(k, "option.");
+    if (ptr) {
+        value = v;
+        if (!value) {
+            ret = -1;
+            goto out;
+        }
+        cli_out_options(key, k, v->data);
+    }
+out:
+    return ret;
+}
+
+static int
+print_brick_details(dict_t *dict, int volcount, int start_index, int end_index,
+                    int replica_count)
+{
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int index = start_index;
+    int isArbiter = 0;
+    int ret = -1;
+    char *brick = NULL;
+
+    while (index <= end_index) {
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", volcount,
+                          index);
+        ret = dict_get_strn(dict, key, keylen, &brick);
+        if (ret)
+            goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter",
+                          volcount, index);
+        if (dict_getn(dict, key, keylen))
+            isArbiter = 1;
+        else
+            isArbiter = 0;
+
+        if (isArbiter)
+            cli_out("Brick%d: %s (arbiter)", index, brick);
+        else
+            cli_out("Brick%d: %s", index, brick);
+        index++;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+gf_cli_print_number_of_bricks(int type, int brick_count, int dist_count,
+                              int stripe_count, int replica_count,
+                              int disperse_count, int redundancy_count,
+                              int arbiter_count)
+{
+    if (type == GF_CLUSTER_TYPE_NONE) {
+        cli_out("Number of Bricks: %d", brick_count);
+    } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                (brick_count / dist_count), disperse_count - redundancy_count,
+                redundancy_count, brick_count);
+    } else {
+        /* For both replicate and stripe, dist_count is
+           good enough */
+        if (arbiter_count == 0) {
+            cli_out("Number of Bricks: %d x %d = %d",
+                    (brick_count / dist_count), dist_count, brick_count);
+        } else {
+            cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                    (brick_count / dist_count), dist_count - arbiter_count,
+                    arbiter_count, brick_count);
+        }
+    }
+}
+
+static int
+gf_cli_get_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    int ret = -1;
+    int opt_count = 0;
+    int32_t i = 0;
+    int32_t j = 1;
+    int32_t status = 0;
+    int32_t type = 0;
+    int32_t brick_count = 0;
+    int32_t dist_count = 0;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t disperse_count = 0;
+    int32_t redundancy_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t snap_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t vol_type = 0;
+    int32_t transport = 0;
+    char *volume_id_str = NULL;
+    char *volname = NULL;
+    char *ta_brick = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    char key[64] = {0};
+    int keylen;
+    char err_str[2048] = {0};
+    gf_cli_rsp rsp = {0};
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to get vol: %d", rsp.op_ret);
+
+    if (!rsp.dict.dict_len) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    if (!count) {
+        switch (local->get_vol.flags) {
+            case GF_CLI_GET_NEXT_VOLUME:
+                GF_FREE(local->get_vol.volname);
+                local->get_vol.volname = NULL;
+                ret = 0;
+                goto out;
+
+            case GF_CLI_GET_VOLUME:
+                snprintf(err_str, sizeof(err_str), "Volume %s does not exist",
+                         local->get_vol.volname);
+                ret = -1;
+                if (!(global_state->mode & GLUSTER_MODE_XML))
+                    goto out;
+        }
+    }
+
+    if (rsp.op_ret) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        ret = -1;
+        goto out;
+    }
+
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* For GET_NEXT_VOLUME output is already begun in
+         * and will also end in gf_cli_get_next_volume()
+         */
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_begin(local, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+        }
+
+        if (dict) {
+            ret = cli_xml_output_vol_info(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+        }
+
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_end(local);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        }
+        goto out;
+    }
+
+    while (i < count) {
+        cli_out(" ");
+        keylen = snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32n(dict, key, keylen, &type);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32n(dict, key, keylen, &status);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &brick_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &dist_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &stripe_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &replica_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &disperse_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &redundancy_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &arbiter_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32n(dict, key, keylen, &transport);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_strn(dict, key, keylen, &volume_id_str);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &snap_count);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &thin_arbiter_count);
+        if (ret)
+            goto out;
+
+        // Distributed (stripe/replicate/stripe-replica) setups
+        vol_type = get_vol_type(type, dist_count, brick_count);
+
+        cli_out("Volume Name: %s", volname);
+        cli_out("Type: %s", vol_type_str[vol_type]);
+        cli_out("Volume ID: %s", volume_id_str);
+        cli_out("Status: %s", cli_vol_status_str[status]);
+        cli_out("Snapshot Count: %d", snap_count);
+
+        gf_cli_print_number_of_bricks(
+            type, brick_count, dist_count, stripe_count, replica_count,
+            disperse_count, redundancy_count, arbiter_count);
+
+        cli_out("Transport-type: %s",
+                ((transport == 0) ? "tcp"
+                                  : (transport == 1) ? "rdma" : "tcp,rdma"));
+        j = 1;
+
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
+
+        cli_out("Bricks:");
+        ret = print_brick_details(dict, i, j, brick_count, replica_count);
+        if (ret)
+            goto out;
+
+        if (thin_arbiter_count) {
+            snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick", i);
+            ret = dict_get_str(dict, key, &ta_brick);
+            if (ret)
+                goto out;
+            cli_out("Thin-arbiter-path: %s", ta_brick);
+        }
+
+        snprintf(key, sizeof(key), "volume%d.opt_count", i);
+        ret = dict_get_int32(dict, key, &opt_count);
+        if (ret)
+            goto out;
+
+        if (!opt_count)
+            goto out;
+
+        cli_out("Options Reconfigured:");
+
+        snprintf(key, sizeof(key), "volume%d.option.", i);
+
+        ret = dict_foreach(dict, _gf_cli_output_volinfo_opts, key);
+        if (ret)
+            goto out;
+
+        i++;
+    }
+
+    ret = 0;
+out:
+    if (ret)
+        cli_err("%s", err_str);
+
+    cli_cmd_broadcast_response(ret);
+
+    if (dict)
+        dict_unref(dict);
+
+    gf_free_xdr_cli_rsp(rsp);
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli_create_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to create volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+                goto out;
+            }
+        }
+
+        ret = cli_xml_output_vol_create(rsp_dict, rsp.op_ret, rsp.op_errno,
+                                        rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume create: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume create: %s: failed", volname);
+    else
+        cli_out(
+            "volume create: %s: success: "
+            "please start the volume to access data",
+            volname);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
+}
+
+static int
+gf_cli_delete_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to delete volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+                goto out;
+            }
+        }
+
+        ret = cli_xml_output_generic_volume("volDelete", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume delete: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume delete: %s: failed", volname);
+    else
+        cli_out("volume delete: %s: success", volname);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli3_1_uuid_get_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    char *uuid_str = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    frame->local = NULL;
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid get");
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidGenerate", dict, rsp.op_ret,
+                                  rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "") == 0)
+            cli_err("Get uuid was unsuccessful");
+        else
+            cli_err("%s", rsp.op_errstr);
+
+    } else {
+        ret = dict_get_str_sizen(dict, "uuid", &uuid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get uuid from dictionary");
+            goto out;
+        }
+        cli_out("UUID: %s", uuid_str);
+    }
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (dict)
+        dict_unref(dict);
+
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli3_1_uuid_reset_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    frame->local = NULL;
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid reset");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidReset", NULL, rsp.op_ret, rsp.op_errno,
+                                  rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("%s", rsp.op_errstr);
+    else
+        cli_out("resetting the peer uuid has been %s",
+                (rsp.op_ret) ? "unsuccessful" : "successful");
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
+
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli_start_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to start volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+                goto out;
+            }
+        }
+
+        ret = cli_xml_output_generic_volume("volStart", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume start: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume start: %s: failed", volname);
+    else
+        cli_out("volume start: %s: success", volname);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
+}
+
+static int
+gf_cli_stop_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to stop volume");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+                goto out;
+            }
+        }
+
+        ret = cli_xml_output_generic_volume("volStop", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Unable to get volname from dict");
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume stop: %s: failed", volname);
+    else
+        cli_out("volume stop: %s: success", volname);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
+}
+
+static int
+gf_cli_print_rebalance_status(dict_t *dict, enum gf_task_types task_type)
+{
+    int ret = -1;
+    int count = 0;
+    int i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_defrag_status_t status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    char *node_name = NULL;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    double elapsed = 0;
+    char *status_str = NULL;
+    char *size_str = NULL;
+    int32_t hrs = 0;
+    uint32_t min = 0;
+    uint32_t sec = 0;
+    gf_boolean_t fix_layout = _gf_false;
+    uint64_t max_time = 0;
+    uint64_t max_elapsed = 0;
+    uint64_t time_left = 0;
+    gf_boolean_t show_estimates = _gf_false;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "count not set");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        /* If information from a node is missing we should skip
+         * the node and try to fetch information of other nodes.
+         * If information is not found for all nodes, we should
+         * error out.
+         */
+        if (!ret)
+            break;
+        if (ret && i == count) {
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            goto out;
+        }
+    }
+
+    /* Fix layout will be sent to all nodes for the volume
+       so every status should be of type
+       GF_DEFRAG_STATUS_LAYOUT_FIX*
+    */
+
+    if ((task_type == GF_TASK_TYPE_REBALANCE) &&
+        (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED)) {
+        fix_layout = _gf_true;
+    }
+
+    if (fix_layout) {
+        cli_out("%35s %41s %27s", "Node", "status", "run time in h:m:s");
+        cli_out("%35s %41s %27s", "---------", "-----------", "------------");
+    } else {
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                "status",
+                "run time in"
+                " h:m:s");
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                "-----------", "-----------", "-----------", "-----------",
+                "-----------", "------------", "--------------");
+    }
+
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        files = 0;
+        size = 0;
+        lookup = 0;
+        skipped = 0;
+        status_str = NULL;
+        elapsed = 0;
+        time_left = 0;
+
+        /* Check if status is NOT_STARTED, and continue early */
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        if (ret == -ENOENT) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            gf_log("cli", GF_LOG_ERROR, "node down and has failed to set dict");
+            continue;
+            /* skip this node if value not available*/
+        } else if (ret) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            continue;
+            /* skip this node if value not available*/
+        }
+
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
+
+        if (GF_DEFRAG_STATUS_STARTED == status_rcd)
+            show_estimates = _gf_true;
+
+        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_strn(dict, key, keylen, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
+
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get file count");
+
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get size of xfer");
+
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookup);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get lookedup file count");
+
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get failures count");
+
+        snprintf(key, sizeof(key), "skipped-%d", i);
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get skipped count");
+
+        /* For remove-brick include skipped count into failure count*/
+        if (task_type != GF_TASK_TYPE_REBALANCE) {
+            failures += skipped;
+            skipped = 0;
+        }
+
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get run-time");
+
+        snprintf(key, sizeof(key), "time-left-%d", i);
+        ret = dict_get_uint64(dict, key, &time_left);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get time left");
+
+        if (elapsed > max_elapsed)
+            max_elapsed = elapsed;
+
+        if (time_left > max_time)
+            max_time = time_left;
+
+        /* Check for array bound */
+        if (status_rcd >= GF_DEFRAG_STATUS_MAX)
+            status_rcd = GF_DEFRAG_STATUS_MAX;
+
+        status_str = cli_vol_task_status_str[status_rcd];
+        size_str = gf_uint64_2human_readable(size);
+        hrs = elapsed / 3600;
+        min = ((uint64_t)elapsed % 3600) / 60;
+        sec = ((uint64_t)elapsed % 3600) % 60;
+
+        if (fix_layout) {
+            cli_out("%35s %50s %8d:%d:%d", node_name, status_str, hrs, min,
+                    sec);
+        } else {
+            if (size_str) {
+                cli_out("%40s %16" PRIu64
+                        " %13s"
+                        " %13" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %20s "
+                        "%8d:%02d:%02d",
+                        node_name, files, size_str, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            } else {
+                cli_out("%40s %16" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %13" PRIu64 " %13" PRIu64
+                        " %20s"
+                        " %8d:%02d:%02d",
+                        node_name, files, size, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            }
+        }
+        GF_FREE(size_str);
+    }
+
+    /* Max time will be non-zero if rebalance is still running */
+    if (max_time) {
+        hrs = max_time / 3600;
+        min = (max_time % 3600) / 60;
+        sec = (max_time % 3600) % 60;
+
+        if (hrs < REBAL_ESTIMATE_SEC_UPPER_LIMIT) {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : %8d:%02d:%02d",
+                hrs, min, sec);
+        } else {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : > 2 months. Please try again "
+                "later.");
+        }
+    } else {
+        /* Rebalance will return 0 if it could not calculate the
+         * estimates or if it is complete.
+         */
+        if (!show_estimates) {
+            goto out;
+        }
+        if (max_elapsed <= REBAL_ESTIMATE_START_TIME) {
+            cli_out(
+                "The estimated time for rebalance to complete "
+                "will be unavailable for the first 10 "
+                "minutes.");
+        } else {
+            cli_out(
+                "Rebalance estimated time unavailable. Please "
+                "try again later.");
+        }
+    }
+out:
+    return ret;
+}
+
+static int
+gf_cli_defrag_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    int cmd = 0;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *task_id_str = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "Failed to get volname");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "rebalance-command",
+                               (int32_t *)&cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_log("glusterd", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = dict_get_str_sizen(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "failed to get %s from dict",
+                   GF_REBALANCE_TID_KEY);
+        }
+        if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        } else {
+            if (!rsp.op_ret) {
+                /* append errstr in the cli msg for successful
+                 * case since unlock failures can be highlighted
+                 * event though rebalance command was successful
+                 */
+                snprintf(msg, sizeof(msg),
+                         "Rebalance on %s has been "
+                         "started successfully. Use "
+                         "rebalance status command to"
+                         " check status of the "
+                         "rebalance process.\nID: %s",
+                         volname, task_id_str);
+            } else {
+                snprintf(msg, sizeof(msg),
+                         "Starting rebalance on volume %s has "
+                         "been unsuccessful.",
+                         volname);
+            }
+        }
+        goto done;
+    }
+
+    if (cmd == GF_DEFRAG_CMD_STOP) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg), "rebalance volume %s stop failed",
+                         volname);
+            goto done;
+        } else {
+            /* append errstr in the cli msg for successful case
+             * since unlock failures can be highlighted event though
+             * rebalance command was successful */
+            snprintf(msg, sizeof(msg),
+                     "rebalance process may be in the middle of a "
+                     "file migration.\nThe process will be fully "
+                     "stopped once the migration of the file is "
+                     "complete.\nPlease check rebalance process "
+                     "for completion before doing any further "
+                     "brick related tasks on the volume.\n%s",
+                     rsp.op_errstr);
+        }
+    }
+    if (cmd == GF_DEFRAG_CMD_STATUS) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg),
+                         "Failed to get the status of rebalance process");
+            goto done;
+        } else {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_rebalance(cmd, dict, rsp.op_ret, rsp.op_errno,
+                                           rsp.op_errstr);
+        goto out;
+    }
+
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REBALANCE);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Failed to print rebalance status");
+
+done:
+    if (global_state->mode & GLUSTER_MODE_XML)
+        cli_xml_output_str("volRebalance", msg, rsp.op_ret, rsp.op_errno,
+                           rsp.op_errstr);
+    else {
+        if (rsp.op_ret)
+            cli_err("volume rebalance: %s: failed%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+        else
+            cli_out("volume rebalance: %s: success%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+    }
+    ret = rsp.op_ret;
+
+out:
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
+}
+
+static int
+gf_cli_rename_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+    snprintf(msg, sizeof(msg), "Rename volume %s",
+             (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volRename", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume rename: failed");
+    else
+        cli_out("volume rename: success");
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_reset_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset");
+
+    if (strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "reset volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volReset", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume reset: failed: %s", msg);
+    else
+        cli_out("volume reset: success: %s", msg);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_ganesha_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to ganesha");
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("nfs-ganesha: failed: %s", rsp.op_errstr);
+        else
+            cli_err("nfs-ganesha: failed");
+    }
+
+    else {
+        cli_out("nfs-ganesha : success ");
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
+}
+
+static char *
+is_server_debug_xlator(void *myframe)
+{
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char **words = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *debug_xlator = NULL;
+
+    frame = myframe;
+    local = frame->local;
+    words = (char **)local->words;
+
+    while (*words != NULL) {
+        if (strstr(*words, "trace") == NULL &&
+            strstr(*words, "error-gen") == NULL) {
+            words++;
+            continue;
+        }
+
+        key = *words;
+        words++;
+        value = *words;
+        if (value == NULL)
+            break;
+        if (strstr(value, "client")) {
+            words++;
+            continue;
+        } else {
+            if (!(strstr(value, "posix") || strstr(value, "acl") ||
+                  strstr(value, "locks") || strstr(value, "io-threads") ||
+                  strstr(value, "marker") || strstr(value, "index"))) {
+                words++;
+                continue;
+            } else {
+                debug_xlator = gf_strdup(key);
+                break;
+            }
+        }
+    }
+
+    return debug_xlator;
+}
+
+static int
+gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *help_str = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *debug_xlator = NULL;
+    char tmp_str[512] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to set");
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    /* For brick processes graph change does not happen on the fly.
+     * The process has to be restarted. So this is a check from the
+     * volume set option such that if debug xlators such as trace/errorgen
+     * are provided in the set command, warn the user.
+     */
+    debug_xlator = is_server_debug_xlator(myframe);
+
+    if (dict_get_str_sizen(dict, "help-str", &help_str) && !msg[0])
+        snprintf(msg, sizeof(msg), "Set volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+    if (rsp.op_ret == 0 && debug_xlator) {
+        snprintf(tmp_str, sizeof(tmp_str),
+                 "\n%s translator has been "
+                 "added to the server volume file. Please restart the"
+                 " volume for enabling the translator",
+                 debug_xlator);
+    }
+
+    if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
+        ret = cli_xml_output_str("volSet", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("volume set: failed: %s", rsp.op_errstr);
+        else
+            cli_err("volume set: failed");
+    } else {
+        if (help_str == NULL) {
+            if (debug_xlator == NULL)
+                cli_out("volume set: success");
+            else
+                cli_out("volume set: success%s", tmp_str);
+        } else {
+            cli_out("%s", help_str);
+        }
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(debug_xlator);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+int
+gf_cli_add_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to add brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Add Brick %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volAddBrick", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume add-brick: failed: %s", rsp.op_errstr);
+    else
+        cli_out("volume add-brick: success");
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli3_remove_brick_status_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int32_t command = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *cmd_str;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "command", &command);
+    if (ret)
+        goto out;
+
+    cmd = command;
+
+    switch (cmd) {
+        case GF_OP_CMD_STOP:
+            cmd_str = "stop";
+            break;
+        case GF_OP_CMD_STATUS:
+            cmd_str = "status";
+            break;
+        default:
+            cmd_str = "unknown";
+            break;
+    }
+
+    ret = rsp.op_ret;
+    if (rsp.op_ret == -1) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed: %s",
+                     cmd_str, rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed",
+                     cmd_str);
+
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+
+        cli_err("%s", msg);
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            strncpy(msg, DICT_UNSERIALIZE_FAIL, sizeof(msg));
+
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                rsp.op_ret = -1;
+                goto xml_output;
+            }
+
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
+            goto out;
+        }
+    }
+
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (strcmp(rsp.op_errstr, "")) {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr,
+                                                  "volRemoveBrick");
+        } else {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, msg,
+                                                  "volRemoveBrick");
+        }
+        goto out;
+    }
+
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REMOVE_BRICK);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to print remove-brick rebalance status");
+        goto out;
+    }
+
+    if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+        cli_out(
+            "'remove-brick' process may be in the middle of a "
+            "file migration.\nThe process will be fully stopped "
+            "once the migration of the file is complete.\nPlease "
+            "check remove-brick process for completion before "
+            "doing any further brick related tasks on the "
+            "volume.");
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_remove_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *cmd_str = "unknown";
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    char *task_id_str = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "command", (int32_t *)&cmd);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        rsp_dict = dict_new();
+        if (!rsp_dict) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    switch (cmd) {
+        case GF_OP_CMD_DETACH_START:
+        case GF_OP_CMD_START:
+            cmd_str = "start";
+
+            ret = dict_get_str_sizen(rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                     &task_id_str);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "remove-brick-id is not present in dict");
+            }
+            break;
+        case GF_OP_CMD_COMMIT:
+            cmd_str = "commit";
+            break;
+        case GF_OP_CMD_COMMIT_FORCE:
+            cmd_str = "commit force";
+            break;
+        default:
+            cmd_str = "unknown";
+            break;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to remove brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Remove Brick %s %s", cmd_str,
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_remove_brick(_gf_false, rsp_dict, rsp.op_ret,
+                                              rsp.op_errno, msg,
+                                              "volRemoveBrick");
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        cli_err("volume remove-brick %s: failed: %s", cmd_str, msg);
+    } else {
+        cli_out("volume remove-brick %s: success", cmd_str);
+        if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+            cli_out("ID: %s", task_id_str);
+        if (GF_OP_CMD_COMMIT == cmd)
+            cli_out(
+                "Check the removed bricks to ensure all files "
+                "are migrated.\nIf files with data are "
+                "found on the brick path, copy them via a "
+                "gluster mount point before re-purposing the "
+                "removed brick. ");
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
+}
+
+static int
+gf_cli_reset_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *reset_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &reset_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (strcmp(reset_op, "GF_RESET_OP_START") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    } else {
+        if (!strcmp(reset_op, "GF_RESET_OP_START")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick start operation failed";
+            else
+                rb_operation_str = "reset-brick start operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit operation failed";
+            else
+                rb_operation_str = "reset-brick commit operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit force operation failed";
+            else
+                rb_operation_str =
+                    "reset-brick commit force operation successful";
+        }
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume reset-brick: failed: %s", msg);
+    else
+        cli_out("volume reset-brick: success: %s", msg);
+    ret = rsp.op_ret;
+
+out:
+    if (frame)
+        frame->local = NULL;
+
+    if (local)
+        cli_local_wipe(local);
+
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
+}
+static int
+gf_cli_replace_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *replace_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &replace_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        if (rsp.op_ret || ret)
+            rb_operation_str = "replace-brick commit force operation failed";
+        else
+            rb_operation_str =
+                "replace-brick commit force operation successful";
+    } else {
+        gf_log(frame->this->name, GF_LOG_DEBUG, "Unknown operation");
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to replace brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume replace-brick: failed: %s", msg);
+    else
+        cli_out("volume replace-brick: success: %s", msg);
+    ret = rsp.op_ret;
+
+out:
+    if (frame)
+        frame->local = NULL;
+
+    if (local)
+        cli_local_wipe(local);
+
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
+}
+
+static int
+gf_cli_log_rotate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to log rotate");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "log rotate %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volLogRotate", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume log-rotate: failed: %s", msg);
+    else
+        cli_out("volume log-rotate: success");
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    return ret;
+}
+
+static int
+gf_cli_sync_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to sync");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "volume sync: failed: %s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "volume sync: %s",
+                 (rsp.op_ret) ? "failed" : "success");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volSync", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("%s", msg);
+    else
+        cli_out("%s", msg);
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+print_quota_list_usage_output(cli_local_t *local, char *path, int64_t avail,
+                              char *sl_str, quota_limits_t *limits,
+                              quota_meta_t *used_space, gf_boolean_t sl,
+                              gf_boolean_t hl, double sl_num,
+                              gf_boolean_t limit_set)
+{
+    int32_t ret = -1;
+    char *used_str = NULL;
+    char *avail_str = NULL;
+    char *hl_str = NULL;
+    char *sl_val = NULL;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_xml_output(local, path, limits->hl, sl_str, sl_num,
+                                   used_space->size, avail, sl ? "Yes" : "No",
+                                   hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
+        }
+        goto out;
+    }
+
+    used_str = gf_uint64_2human_readable(used_space->size);
+
+    if (limit_set) {
+        hl_str = gf_uint64_2human_readable(limits->hl);
+        sl_val = gf_uint64_2human_readable(sl_num);
+
+        if (!used_str) {
+            cli_out("%-40s %7s %7s(%s) %8" PRIu64 "%9" PRIu64
+                    ""
+                    "%15s %18s",
+                    path, hl_str, sl_str, sl_val, used_space->size, avail,
+                    sl ? "Yes" : "No", hl ? "Yes" : "No");
+        } else {
+            avail_str = gf_uint64_2human_readable(avail);
+            cli_out("%-40s %7s %7s(%s) %8s %7s %15s %20s", path, hl_str, sl_str,
+                    sl_val, used_str, avail_str, sl ? "Yes" : "No",
+                    hl ? "Yes" : "No");
+        }
+    } else {
+        cli_out("%-36s %10s %10s %14s %9s %15s %18s", path, "N/A", "N/A",
+                used_str, "N/A", "N/A", "N/A");
+    }
+
+    ret = 0;
+out:
+    GF_FREE(hl_str);
+    GF_FREE(used_str);
+    GF_FREE(avail_str);
+    GF_FREE(sl_val);
+
+    return ret;
+}
+
+static int
+print_quota_list_object_output(cli_local_t *local, char *path, int64_t avail,
+                               char *sl_str, quota_limits_t *limits,
+                               quota_meta_t *used_space, gf_boolean_t sl,
+                               gf_boolean_t hl, double sl_num,
+                               gf_boolean_t limit_set)
+{
+    int32_t ret = -1;
+    int64_t sl_val = sl_num;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_object_xml_output(local, path, sl_str, sl_val, limits,
+                                          used_space, avail, sl ? "Yes" : "No",
+                                          hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
+        }
+        goto out;
+    }
+
+    if (limit_set) {
+        cli_out("%-40s %9" PRIu64 " %9s(%" PRId64 ") %10" PRIu64
+                ""
+                "%10" PRIu64 " %11" PRIu64 " %15s %20s",
+                path, limits->hl, sl_str, sl_val, used_space->file_count,
+                used_space->dir_count, avail, sl ? "Yes" : "No",
+                hl ? "Yes" : "No");
+    } else {
+        cli_out("%-40s %9s %9s %10" PRIu64 " %10" PRIu64 " %11s %15s %20s",
+                path, "N/A", "N/A", used_space->file_count,
+                used_space->dir_count, "N/A", "N/A", "N/A");
+    }
+    ret = 0;
+
+out:
+
+    return ret;
+}
+
+static int
+print_quota_list_output(cli_local_t *local, char *path, char *default_sl,
+                        quota_limits_t *limits, quota_meta_t *used_space,
+                        int type, gf_boolean_t limit_set)
+{
+    int64_t avail = 0;
+    char percent_str[20] = {0};
+    char *sl_final = NULL;
+    int ret = -1;
+    double sl_num = 0;
+    gf_boolean_t sl = _gf_false;
+    gf_boolean_t hl = _gf_false;
+    int64_t used_size = 0;
+
+    GF_ASSERT(local);
+    GF_ASSERT(path);
+
+    if (limit_set) {
+        if (limits->sl < 0) {
+            ret = gf_string2percent(default_sl, &sl_num);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "could not convert default soft limit to percent");
+                goto out;
+            }
+            sl_num = (sl_num * limits->hl) / 100;
+            sl_final = default_sl;
+        } else {
+            sl_num = (limits->sl * limits->hl) / 100;
+            ret = snprintf(percent_str, sizeof(percent_str), "%" PRIu64 "%%",
+                           limits->sl);
+            if (ret < 0)
+                goto out;
+            sl_final = percent_str;
+        }
+        if (type == GF_QUOTA_OPTION_TYPE_LIST)
+            used_size = used_space->size;
+        else
+            used_size = used_space->file_count + used_space->dir_count;
+
+        if (limits->hl > used_size) {
+            avail = limits->hl - used_size;
+            hl = _gf_false;
+            if (used_size > sl_num)
+                sl = _gf_true;
+            else
+                sl = _gf_false;
+        } else {
+            avail = 0;
+            hl = sl = _gf_true;
+        }
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = print_quota_list_usage_output(local, path, avail, sl_final,
+                                            limits, used_space, sl, hl, sl_num,
+                                            limit_set);
+    else
+        ret = print_quota_list_object_output(local, path, avail, sl_final,
+                                             limits, used_space, sl, hl, sl_num,
+                                             limit_set);
+out:
+    return ret;
+}
+
+static int
+print_quota_list_from_mountdir(cli_local_t *local, char *mountdir,
+                               char *default_sl, char *path, int type)
+{
+    int ret = -1;
+    ssize_t xattr_size = 0;
+    quota_limits_t limits = {
+        0,
+    };
+    quota_meta_t used_space = {
+        0,
+    };
+    char *key = NULL;
+    gf_boolean_t limit_set = _gf_true;
+
+    GF_ASSERT(local);
+    GF_ASSERT(mountdir);
+    GF_ASSERT(path);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        key = QUOTA_LIMIT_KEY;
+    else
+        key = QUOTA_LIMIT_OBJECTS_KEY;
+
+    ret = sys_lgetxattr(mountdir, key, (void *)&limits, sizeof(limits));
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get the xattr %s on %s. Reason : %s", key, mountdir,
+               strerror(errno));
+
+        switch (errno) {
+#if defined(ENODATA)
+            case ENODATA:
+#endif
+#if defined(ENOATTR) && (ENOATTR != ENODATA)
+            case ENOATTR:
+#endif
+                /* If it's an ENOATTR, quota/inode-quota is
+                 * configured(limit is set at least for one directory).
+                 * The user is trying to issue 'list/list-objects'
+                 * command for a directory on which quota limit is
+                 * not set and we are showing the used-space in case
+                 * of list-usage and showing (dir_count, file_count)
+                 * in case of list-objects. Other labels are
+                 * shown "N/A".
+                 */
+
+                limit_set = _gf_false;
+                goto enoattr;
+                break;
+
+            default:
+                cli_err("%-40s %s", path, strerror(errno));
+                break;
+        }
+
+        goto out;
+    }
+
+    limits.hl = ntoh64(limits.hl);
+    limits.sl = ntoh64(limits.sl);
+
+enoattr:
+    xattr_size = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, NULL, 0);
+    if (xattr_size < (sizeof(int64_t) * 2) &&
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        ret = -1;
+
+        /* This can happen when glusterfs is upgraded from 3.6 to 3.7
+         * and the xattr healing is not completed.
+         */
+    } else if (xattr_size > (sizeof(int64_t) * 2)) {
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &used_space,
+                            sizeof(used_space));
+    } else if (xattr_size > 0) {
+        /* This is for compatibility.
+         * Older version had only file usage
+         */
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &(used_space.size),
+                            sizeof(used_space.size));
+        used_space.file_count = 0;
+        used_space.dir_count = 0;
+    } else {
+        ret = -1;
+    }
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota size on path %s: %s",
+               mountdir, strerror(errno));
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    used_space.size = ntoh64(used_space.size);
+    used_space.file_count = ntoh64(used_space.file_count);
+    used_space.dir_count = ntoh64(used_space.dir_count);
+
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, limit_set);
+out:
+    return ret;
+}
+
+static int
+gluster_remove_auxiliary_mount(char *volname)
+{
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    ret = gf_umount_lazy(this->name, mountdir, 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "umount on %s failed, reason : %s",
+               mountdir, strerror(errno));
+    }
+
+    return ret;
+}
+
+static int
+gf_cli_print_limit_list_from_dict(cli_local_t *local, char *volname,
+                                  dict_t *dict, char *default_sl, int count,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+    int ret = -1;
+    int i = 0;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int type = -1;
+
+    if (!dict || count <= 0)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota type");
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_begin(local, op_ret, op_errno,
+                                                        op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
+    } else {
+        print_quota_list_header(type);
+    }
+
+    while (count--) {
+        keylen = snprintf(key, sizeof(key), "path%d", i++);
+
+        ret = dict_get_strn(dict, key, keylen, &path);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_DEBUG, "Path not present in limit list");
+            continue;
+        }
+
+        ret = gf_canonicalize_path(path);
+        if (ret)
+            goto out;
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, path);
+        ret = print_quota_list_from_mountdir(local, mountdir, default_sl, path,
+                                             type);
+    }
+
+out:
+    return ret;
+}
+
+static int
+print_quota_list_from_quotad(call_frame_t *frame, dict_t *rsp_dict)
+{
+    char *path = NULL;
+    char *default_sl = NULL;
+    int ret = -1;
+    cli_local_t *local = NULL;
+    dict_t *gd_rsp_dict = NULL;
+    quota_meta_t used_space = {
+        0,
+    };
+    quota_limits_t limits = {
+        0,
+    };
+    quota_limits_t *size_limits = NULL;
+    int32_t type = 0;
+    int32_t success_count = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    gd_rsp_dict = local->dict;
+
+    ret = dict_get_int32_sizen(rsp_dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "path key is not present in dict");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(gd_rsp_dict, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get default soft limit");
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_KEY, (void **)&size_limits);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "limit key not present in dict on %s",
+                   path);
+            goto out;
+        }
+    } else {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
+                           (void **)&size_limits);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING,
+                   "object limit key not present in dict on %s", path);
+            goto out;
+        }
+    }
+
+    limits.hl = ntoh64(size_limits->hl);
+    limits.sl = ntoh64(size_limits->sl);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = quota_dict_get_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &used_space);
+    else
+        ret = quota_dict_get_inode_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                        SLEN(QUOTA_SIZE_KEY), &used_space);
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_WARNING, "size key not present in dict");
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   &success_count);
+        if (ret)
+            success_count = 0;
+
+        ret = dict_set_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   success_count + 1);
+    }
+    UNLOCK(&local->lock);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to set quota-list-success-count in dict");
+        goto out;
+    }
+
+    if (success_count == 0) {
+        if (!(global_state->mode & GLUSTER_MODE_XML)) {
+            print_quota_list_header(type);
+        } else {
+            ret = cli_xml_output_vol_quota_limit_list_begin(local, 0, 0, NULL);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto out;
+            }
+        }
+    }
+
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, _gf_true);
+out:
+    return ret;
+}
+
+static void *
+cli_cmd_broadcast_response_detached(void *opaque)
+{
+    int32_t ret = 0;
+
+    ret = (intptr_t)opaque;
+    cli_cmd_broadcast_response(ret);
+
+    return NULL;
+}
+
+static int32_t
+cli_quota_compare_path(struct list_head *list1, struct list_head *list2)
+{
+    struct list_node *node1 = NULL;
+    struct list_node *node2 = NULL;
+    dict_t *dict1 = NULL;
+    dict_t *dict2 = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    int ret = 0;
+
+    node1 = list_entry(list1, struct list_node, list);
+    node2 = list_entry(list2, struct list_node, list);
+
+    dict1 = node1->ptr;
+    dict2 = node2->ptr;
+
+    ret = dict_get_str_sizen(dict1, GET_ANCESTRY_PATH_KEY, &path1);
+    if (ret < 0)
+        return 0;
+
+    ret = dict_get_str_sizen(dict2, GET_ANCESTRY_PATH_KEY, &path2);
+    if (ret < 0)
+        return 0;
+
+    return strcmp(path1, path2);
+}
+
+static int
+cli_quotad_getlimit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    /*TODO: we need to gather the path, hard-limit, soft-limit and used space*/
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    struct list_node *node = NULL;
+    struct list_node *tmpnode = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    int32_t list_count = 0;
+    pthread_t th_id = {
+        0,
+    };
+    int32_t max_count = 0;
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(local->dict, "quota-list-count",
+                                   &list_count);
+        if (ret)
+            list_count = 0;
+
+        list_count++;
+        ret = dict_set_int32_sizen(local->dict, "quota-list-count", list_count);
+    }
+    UNLOCK(&local->lock);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set quota-list-count in dict");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        if (list_count == 0)
+            cli_err(
+                "Connection failed. Please check if quota "
+                "daemon is operational.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("quota command failed : %s", rsp.op_errstr);
+        else
+            cli_err("quota command : failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+
+        node = list_node_add_order(dict, &local->dict_list,
+                                   cli_quota_compare_path);
+        if (node == NULL) {
+            gf_log("cli", GF_LOG_ERROR, "failed to add node to the list");
+            dict_unref(dict);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "max_count", &max_count);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "failed to get max_count");
+        goto out;
+    }
+
+    if (list_count == max_count) {
+        list_for_each_entry_safe(node, tmpnode, &local->dict_list, list)
+        {
+            dict = node->ptr;
+            print_quota_list_from_quotad(frame, dict);
+            list_node_del(node);
+            dict_unref(dict);
+        }
+    }
+
+out:
+    /* Bad Fix: CLI holds the lock to process a command.
+     * When processing quota list command, below sequence of steps executed
+     * in the same thread and causing deadlock
+     *
+     * 1) CLI holds the lock
+     * 2) Send rpc_clnt_submit request to quotad for quota usage
+     * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
+     * 4) cbk function cli_quotad_getlimit_cbk invokes
+     *    cli_cmd_broadcast_response which tries to hold lock to broadcast
+     *    the results and hangs, because same thread has already holding
+     *    the lock
+     *
+     * Broadcasting response in a separate thread which is not a
+     * good fix. This needs to be re-visted with better solution
+     */
+    if (ret == -1) {
+        ret = pthread_create(&th_id, NULL, cli_cmd_broadcast_response_detached,
+                             (void *)-1);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "pthread_create failed: %s",
+                   strerror(errno));
+    } else {
+        cli_cmd_broadcast_response(ret);
+    }
+    gf_free_xdr_cli_rsp(rsp);
+
+    return ret;
+}
+
+static int
+cli_quotad_getlimit(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    ret = add_cli_cmd_timeout_to_dict(dict);
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+
+        goto out;
+    }
+
+    ret = cli_cmd_submit(global_quotad_rpc, &req, frame, &cli_quotad_clnt,
+                         GF_AGGREGATOR_GETLIMIT, NULL, this,
+                         cli_quotad_getlimit_cbk, (xdrproc_t)xdr_gf_cli_req);
+
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static void
+gf_cli_quota_list(cli_local_t *local, char *volname, dict_t *dict,
+                  char *default_sl, int count, int op_ret, int op_errno,
+                  char *op_errstr)
+{
+    if (!cli_cmd_connected())
+        goto out;
+
+    if (count > 0) {
+        GF_VALIDATE_OR_GOTO("cli", volname, out);
+
+        gf_cli_print_limit_list_from_dict(local, volname, dict, default_sl,
+                                          count, op_ret, op_errno, op_errstr);
+    }
+out:
+    return;
+}
+
+static int
+gf_cli_quota_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    char *default_sl = NULL;
+    cli_local_t *local = NULL;
+    char *default_sl_dup = NULL;
+    int32_t entry_count = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("quota command failed : %s", rsp.op_errstr);
+            if (rsp.op_ret == -ENOENT)
+                cli_err("please enter the path relative to the volume");
+        } else {
+            cli_err("quota command : failed");
+        }
+
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to quota command");
+
+    ret = dict_get_str_sizen(dict, "default-soft-limit", &default_sl);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE,
+               "failed to get default soft limit");
+
+    // default-soft-limit is part of rsp_dict only iff we sent
+    // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
+    if (default_sl) {
+        default_sl_dup = gf_strdup(default_sl);
+        if (!default_sl_dup) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, "default-soft-limit",
+                                    default_sl_dup);
+        if (ret) {
+            gf_log(frame->this->name, GF_LOG_TRACE,
+                   "failed to set default soft limit");
+            GF_FREE(default_sl_dup);
+        }
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get type");
+
+    ret = dict_get_int32_sizen(dict, "count", &entry_count);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get count");
+
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gf_cli_quota_list(local, volname, dict, default_sl, entry_count,
+                          rsp.op_ret, rsp.op_errno, rsp.op_errstr);
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_vol_quota_limit_list_end(local);
+            if (ret < 0) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+            goto out;
+        }
+    }
+
+xml_output:
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volQuota", NULL, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST &&
+        type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
+        cli_out("volume quota : success");
+
+    ret = rsp.op_ret;
+out:
+
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gluster_remove_auxiliary_mount(volname);
+    }
+
+    cli_cmd_broadcast_response(ret);
+    if (dict)
+        dict_unref(dict);
+
+    gf_free_xdr_cli_rsp(rsp);
+
+    return ret;
+}
+
+static int
+gf_cli_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "getspec failed");
+        ret = -1;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to getspec");
+
+    spec = GF_MALLOC(rsp.op_ret + 1, cli_mt_char);
+    if (!spec) {
+        gf_log("", GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    memcpy(spec, rsp.spec, rsp.op_ret);
+    spec[rsp.op_ret] = '\0';
+    cli_out("%s", spec);
+    GF_FREE(spec);
+
+    ret = 0;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_getspec_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_pmap_b2p_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "pump_b2p failed");
+        ret = -1;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to pmap b2p");
+
+    cli_out("%d", rsp.port);
+    GF_FREE(spec);
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_probe(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_probe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_PROBE,
+                          this, cli_rpc_prog, NULL);
+
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_deprobe(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+    int flags = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "flags", &flags);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "flags", 0);
+        if (ret)
+            goto out;
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_deprobe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEPROBE,
+                          this, cli_rpc_prog, NULL);
+
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_list_friends(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf1_cli_peer_list_req req = {
+        0,
+    };
+    int ret = 0;
+    unsigned long flags = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_ASSERT(frame->local == NULL);
+
+    flags = (long)data;
+    req.flags = flags;
+    frame->local = (void *)flags;
+    ret = cli_cmd_submit(
+        NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_LIST_FRIENDS, NULL, this,
+        gf_cli_list_friends_cbk, (xdrproc_t)xdr_gf1_cli_peer_list_req);
+
+out:
+    if (ret && frame) {
+        /*
+         * If everything goes fine, gf_cli_list_friends_cbk()
+         * [invoked through cli_cmd_submit()]resets the
+         * frame->local to NULL. In case cli_cmd_submit()
+         * fails in between, RESET frame->local here.
+         */
+        frame->local = NULL;
+    }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_get_state(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_state_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GET_STATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_get_next_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    cli_local_t *local = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = data;
+    local = frame->local;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_begin(local, 0, 0, "");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
+    }
+
+    ret = gf_cli_get_volume(frame, this, data);
+
+    if (!local || !local->get_vol.volname) {
+        if ((global_state->mode & GLUSTER_MODE_XML))
+            goto end_xml;
+
+        cli_err("No volumes present");
+        goto out;
+    }
+
+    ctx->volname = local->get_vol.volname;
+
+    while (ctx->volname) {
+        ret = gf_cli_get_volume(frame, this, ctx);
+        if (ret)
+            goto out;
+        ctx->volname = local->get_vol.volname;
+    }
+
+end_xml:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_end(local);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = data;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    if (ctx->volname) {
+        ret = dict_set_str_sizen(dict, "volname", ctx->volname);
+        if (ret)
+            goto out;
+    }
+
+    flags = ctx->flags;
+    ret = dict_set_int32_sizen(dict, "flags", flags);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to set flags");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_GET_VOLUME, NULL, this,
+                         gf_cli_get_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
+
+out:
+    if (dict)
+        dict_unref(dict);
+
+    GF_FREE(req.dict.dict_val);
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int32_t
+gf_cli3_1_uuid_get(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_get_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_UUID_GET,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli3_1_uuid_reset(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_reset_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_create_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_create_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_delete_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_delete_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_start_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_start_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog, NULL);
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_stop_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = data;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_stop_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_defrag_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_defrag_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_rename_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_RENAME_VOLUME, NULL, this,
+                         gf_cli_rename_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
+
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_reset_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_ganesha(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_ganesha_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_GANESHA,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_set_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_set_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+int32_t
+gf_cli_add_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_add_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    ;
+    gf_cli_req status_req = {{
+        0,
+    }};
+    ;
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t command = 0;
+    int32_t cmd = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "command", &command);
+    if (ret)
+        goto out;
+
+    if ((command != GF_OP_CMD_STATUS) && (command != GF_OP_CMD_STOP)) {
+        ret = cli_to_glusterd(
+            &req, frame, gf_cli_remove_brick_cbk, (xdrproc_t)xdr_gf_cli_req,
+            dict, GLUSTER_CLI_REMOVE_BRICK, this, cli_rpc_prog, NULL);
+    } else {
+        /* Need rebalance status to be sent :-) */
+        if (command == GF_OP_CMD_STATUS)
+            cmd |= GF_DEFRAG_CMD_STATUS;
+        else
+            cmd |= GF_DEFRAG_CMD_STOP;
+
+        ret = dict_set_int32_sizen(dict, "rebalance-command", (int32_t)cmd);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Failed to set dict");
+            goto out;
+        }
+
+        ret = cli_to_glusterd(
+            &status_req, frame, gf_cli3_remove_brick_status_cbk,
+            (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEFRAG_VOLUME, this,
+            cli_rpc_prog, NULL);
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    GF_FREE(status_req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_reset_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    char *dst_brick = NULL;
+    char *op = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_str_sizen(dict, "dst-brick", &dst_brick);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "dict_get on dst-brick failed");
+            goto out;
+        }
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog, NULL);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_replace_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t op = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_replace_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog, NULL);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_log_rotate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_log_rotate_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_sync_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_sync_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_getspec(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    dict_t *op_dict = NULL;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "volid", &req.key);
+    if (ret)
+        goto out;
+
+    op_dict = dict_new();
+    if (!op_dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32_sizen(op_dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(op_dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(op_dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_handshake_prog,
+                         GF_HNDSK_GETSPEC, NULL, this, gf_cli_getspec_cbk,
+                         (xdrproc_t)xdr_gf_getspec_req);
+
+out:
+    if (op_dict) {
+        dict_unref(op_dict);
+    }
+    GF_FREE(req.xdata.xdata_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_quota(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_quota_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_QUOTA,
+                          this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_pmap_b2p(call_frame_t *frame, xlator_t *this, void *data)
+{
+    pmap_port_by_brick_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "brick", &req.brick);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_pmap_prog, GF_PMAP_PORTBYBRICK,
+                         NULL, this, gf_cli_pmap_b2p_cbk,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int
+gf_cli_fsm_log_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf1_cli_fsm_log_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    int tr_count = 0;
+    char key[64] = {0};
+    int keylen;
+    int i = 0;
+    char *old_state = NULL;
+    char *new_state = NULL;
+    char *event = NULL;
+    char *time = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("fsm log unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.fsm_log.fsm_log_val, rsp.fsm_log.fsm_log_len,
+                           &dict);
+
+    if (ret) {
+        cli_err(DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &tr_count);
+    if (!ret && tr_count)
+        cli_out("number of transitions: %d", tr_count);
+    else
+        cli_err("No transitions");
+    for (i = 0; i < tr_count; i++) {
+        keylen = snprintf(key, sizeof(key), "log%d-old-state", i);
+        ret = dict_get_strn(dict, key, keylen, &old_state);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "log%d-event", i);
+        ret = dict_get_strn(dict, key, keylen, &event);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "log%d-new-state", i);
+        ret = dict_get_strn(dict, key, keylen, &new_state);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "log%d-time", i);
+        ret = dict_get_strn(dict, key, keylen, &time);
+        if (ret)
+            goto out;
+        cli_out(
+            "Old State: [%s]\n"
+            "New State: [%s]\n"
+            "Event    : [%s]\n"
+            "timestamp: [%s]\n",
+            old_state, new_state, event, time);
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    if (dict) {
+        dict_unref(dict);
+    }
+    gf_free_xdr_fsm_log_rsp(rsp);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_fsm_log(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf1_cli_fsm_log_req req = {
+        0,
+    };
+
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
+
+    if (!frame || !this || !data)
+        goto out;
+    req.name = data;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_FSM_LOG,
+                         NULL, this, gf_cli_fsm_log_cbk,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static int
+gf_cli_gsync_config_command(dict_t *dict)
+{
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str_sizen(dict, "subop", &subop) != 0)
+        return -1;
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        cli_out(GEOREP " config updated successfully");
+        return 0;
+    }
+
+    if (dict_get_str_sizen(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str_sizen(dict, "slave", &slave) != 0)
+        return -1;
+
+    if (dict_get_str_sizen(dict, "master", &master) != 0)
+        master = NULL;
+    if (dict_get_str_sizen(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
+
+    ret = dict_get_str_sizen(dict, "conf_path", &confpath);
+    if (ret || !confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
+
+    return runner_run(&runner);
+}
+
+static int
+gf_cli_print_status(char **title_values, gf_gsync_status_t **sts_vals,
+                    int *spacing, int gsync_count, int number_of_fields,
+                    int is_detail)
+{
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int status_fields = 8; /* Indexed at 0 */
+    int total_spacing = 0;
+    char **output_values = NULL;
+    char *tmp = NULL;
+    char *hyphens = NULL;
+
+    /* calculating spacing for hyphens */
+    for (i = 0; i < number_of_fields; i++) {
+        /* Suppressing detail output for status */
+        if ((!is_detail) && (i > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            continue;
+        }
+        spacing[i] += 3; /* Adding extra space to
+                            distinguish between fields */
+        total_spacing += spacing[i];
+    }
+    total_spacing += 4; /* For the spacing between the fields */
+
+    /* char pointers for each field */
+    output_values = GF_MALLOC(number_of_fields * sizeof(char *),
+                              gf_common_mt_char);
+    if (!output_values) {
+        ret = -1;
+        goto out;
+    }
+    for (i = 0; i < number_of_fields; i++) {
+        output_values[i] = GF_CALLOC(spacing[i] + 1, sizeof(char),
+                                     gf_common_mt_char);
+        if (!output_values[i]) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cli_out(" ");
+
+    /* setting the title "NODE", "MASTER", etc. from title_values[]
+       and printing the same */
+    for (j = 0; j < number_of_fields; j++) {
+        if ((!is_detail) && (j > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            output_values[j][0] = '\0';
+            continue;
+        }
+        memset(output_values[j], ' ', spacing[j]);
+        memcpy(output_values[j], title_values[j], strlen(title_values[j]));
+        output_values[j][spacing[j]] = '\0';
+    }
+    cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", output_values[0],
+            output_values[1], output_values[2], output_values[3],
+            output_values[4], output_values[5], output_values[6],
+            output_values[7], output_values[8], output_values[9],
+            output_values[10], output_values[11], output_values[12],
+            output_values[13], output_values[14], output_values[15]);
+
+    hyphens = GF_MALLOC((total_spacing + 1) * sizeof(char), gf_common_mt_char);
+    if (!hyphens) {
+        ret = -1;
+        goto out;
+    }
+
+    /* setting and printing the hyphens */
+    memset(hyphens, '-', total_spacing);
+    hyphens[total_spacing] = '\0';
+    cli_out("%s", hyphens);
+
+    for (i = 0; i < gsync_count; i++) {
+        for (j = 0; j < number_of_fields; j++) {
+            if ((!is_detail) && (j > status_fields)) {
+                /* Suppressing detailed output for
+                 * status */
+                output_values[j][0] = '\0';
+                continue;
+            }
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+            memset(output_values[j], ' ', spacing[j]);
+            memcpy(output_values[j], tmp, strlen(tmp));
+            output_values[j][spacing[j]] = '\0';
+        }
+
+        cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
+                output_values[0], output_values[1], output_values[2],
+                output_values[3], output_values[4], output_values[5],
+                output_values[6], output_values[7], output_values[8],
+                output_values[9], output_values[10], output_values[11],
+                output_values[12], output_values[13], output_values[14],
+                output_values[15]);
+    }
+
+out:
+    if (output_values) {
+        for (i = 0; i < number_of_fields; i++) {
+            if (output_values[i])
+                GF_FREE(output_values[i]);
+        }
+        GF_FREE(output_values);
+    }
+
+    if (hyphens)
+        GF_FREE(hyphens);
+
+    return ret;
+}
+
+int
+gf_gsync_status_t_comparator(const void *p, const void *q)
+{
+    char *slavekey1 = NULL;
+    char *slavekey2 = NULL;
+
+    slavekey1 = get_struct_variable(20, (*(gf_gsync_status_t **)p));
+    slavekey2 = get_struct_variable(20, (*(gf_gsync_status_t **)q));
+    if (!slavekey1 || !slavekey2) {
+        gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+        return 0;
+    }
+
+    return strcmp(slavekey1, slavekey2);
+}
+
+static int
+gf_cli_read_status_data(dict_t *dict, gf_gsync_status_t **sts_vals,
+                        int *spacing, int gsync_count, int number_of_fields)
+{
+    char *tmp = NULL;
+    char sts_val_name[PATH_MAX] = "";
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+
+    /* Storing per node status info in each object */
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+
+        /* Fetching the values from dict, and calculating
+           the max length for each field */
+        ret = dict_get_bin(dict, sts_val_name, (void **)&(sts_vals[i]));
+        if (ret)
+            goto out;
+
+        for (j = 0; j < number_of_fields; j++) {
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+            if (strlen(tmp) > spacing[j])
+                spacing[j] = strlen(tmp);
+        }
+    }
+
+    /* Sort based on Session Slave */
+    qsort(sts_vals, gsync_count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
+
+out:
+    return ret;
+}
+
+static int
+gf_cli_gsync_status_output(dict_t *dict, gf_boolean_t is_detail)
+{
+    int gsync_count = 0;
+    int i = 0;
+    int ret = 0;
+    int spacing[16] = {0};
+    int num_of_fields = 16;
+    char errmsg[1024] = "";
+    char *master = NULL;
+    char *slave = NULL;
+    static char *title_values[] = {"MASTER NODE",
+                                   "MASTER VOL",
+                                   "MASTER BRICK",
+                                   "SLAVE USER",
+                                   "SLAVE",
+                                   "SLAVE NODE",
+                                   "STATUS",
+                                   "CRAWL STATUS",
+                                   "LAST_SYNCED",
+                                   "ENTRY",
+                                   "DATA",
+                                   "META",
+                                   "FAILURES",
+                                   "CHECKPOINT TIME",
+                                   "CHECKPOINT COMPLETED",
+                                   "CHECKPOINT COMPLETION TIME"};
+    gf_gsync_status_t **sts_vals = NULL;
+
+    /* Checks if any session is active or not */
+    ret = dict_get_int32_sizen(dict, "gsync-count", &gsync_count);
+    if (ret) {
+        ret = dict_get_str_sizen(dict, "master", &master);
+
+        ret = dict_get_str_sizen(dict, "slave", &slave);
+
+        if (master) {
+            if (slave)
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions between %s"
+                         " and %s",
+                         master, slave);
+            else
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions for %s", master);
+        } else
+            snprintf(errmsg, sizeof(errmsg),
+                     "No active geo-replication sessions");
+
+        gf_log("cli", GF_LOG_INFO, "%s", errmsg);
+        cli_out("%s", errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < num_of_fields; i++)
+        spacing[i] = strlen(title_values[i]);
+
+    /* gsync_count = number of nodes reporting output.
+       each sts_val object will store output of each
+       node */
+    sts_vals = GF_MALLOC(gsync_count * sizeof(gf_gsync_status_t *),
+                         gf_common_mt_char);
+    if (!sts_vals) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_cli_read_status_data(dict, sts_vals, spacing, gsync_count,
+                                  num_of_fields);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to read status data");
+        goto out;
+    }
+
+    ret = gf_cli_print_status(title_values, sts_vals, spacing, gsync_count,
+                              num_of_fields, is_detail);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to print status output");
+        goto out;
+    }
+
+out:
+    if (sts_vals)
+        GF_FREE(sts_vals);
+
+    return ret;
+}
+
+static int32_t
+write_contents_to_common_pem_file(dict_t *dict, int output_count)
+{
+    char *workdir = NULL;
+    char common_pem_file[PATH_MAX] = "";
+    char *output = NULL;
+    char output_name[32] = "";
+    int bytes_written = 0;
+    int fd = -1;
+    int ret = -1;
+    int i = -1;
+
+    ret = dict_get_str_sizen(dict, "glusterd_workdir", &workdir);
+    if (ret || !workdir) {
+        gf_log("", GF_LOG_ERROR, "Unable to fetch workdir");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(common_pem_file, sizeof(common_pem_file),
+             "%s/geo-replication/common_secret.pem.pub", workdir);
+
+    sys_unlink(common_pem_file);
+
+    fd = open(common_pem_file, O_WRONLY | O_CREAT, 0600);
+    if (fd == -1) {
+        gf_log("", GF_LOG_ERROR, "Failed to open %s Error : %s",
+               common_pem_file, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
+        }
+        if (output) {
+            bytes_written = sys_write(fd, output, strlen(output));
+            if (bytes_written != strlen(output)) {
+                gf_log("", GF_LOG_ERROR, "Failed to write to %s",
+                       common_pem_file);
+                ret = -1;
+                goto out;
+            }
+            /* Adding the new line character */
+            bytes_written = sys_write(fd, "\n", 1);
+            if (bytes_written != 1) {
+                gf_log("", GF_LOG_ERROR, "Failed to add new line char");
+                ret = -1;
+                goto out;
+            }
+            output = NULL;
+        }
+    }
+
+    cli_out("Common secret pub file present at %s", common_pem_file);
+    ret = 0;
+out:
+    if (fd >= 0)
+        sys_close(fd);
+
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli_sys_exec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    int output_count = -1;
+    int i = -1;
+    char *output = NULL;
+    char *command = NULL;
+    char output_name[32] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Command failed.");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "output_count", &output_count);
+    if (ret) {
+        cli_out("Command executed successfully.");
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "command", &command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to get command from dict");
+        goto out;
+    }
+
+    if (!strcmp(command, "gsec_create")) {
+        ret = write_contents_to_common_pem_file(dict, output_count);
+        if (!ret)
+            goto out;
+    }
+
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
+        }
+        if (output) {
+            cli_out("%s", output);
+            output = NULL;
+        }
+    }
+
+    ret = 0;
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_copy_file_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Copy unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    cli_out("Successfully copied file.");
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_gsync_set_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *gsync_status = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int32_t type = 0;
+    gf_boolean_t status_detail = _gf_false;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_gsync(dict, rsp.op_ret, rsp.op_errno,
+                                       rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        cli_err("%s",
+                rsp.op_errstr ? rsp.op_errstr : GEOREP " command unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "gsync-status", &gsync_status);
+    if (!ret)
+        cli_out("%s", gsync_status);
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "failed to get type");
+        goto out;
+    }
+
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out(
+                "%s " GEOREP " session between %s & %s has been successful",
+                type == GF_GSYNC_OPTION_TYPE_START ? "Starting" : "Stopping",
+                master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_PAUSE:
+        case GF_GSYNC_OPTION_TYPE_RESUME:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out("%s " GEOREP " session between %s & %s has been successful",
+                    type == GF_GSYNC_OPTION_TYPE_PAUSE ? "Pausing" : "Resuming",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            ret = gf_cli_gsync_config_command(dict);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+            status_detail = dict_get_str_boolean(dict, "status-detail",
+                                                 _gf_false);
+            ret = gf_cli_gsync_status_output(dict, status_detail);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Deleting " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Creating " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
+
+        default:
+            cli_out(GEOREP " command executed successfully");
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_sys_exec(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_sys_exec_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_SYS_EXEC,
+                          this, cli_rpc_prog, NULL);
+    if (ret)
+        if (!frame || !this || !data) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
+        }
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_copy_file(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_copy_file_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog, NULL);
+    if (ret)
+        if (!frame || !this || !data) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
+        }
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_gsync_set(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_gsync_set_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+int
+cli_profile_info_percentage_cmp(void *a, void *b)
+{
+    cli_profile_info_t *ia = NULL;
+    cli_profile_info_t *ib = NULL;
+
+    ia = a;
+    ib = b;
+    if (ia->percentage_avg_latency < ib->percentage_avg_latency)
+        return -1;
+    else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
+        return 1;
+
+    return 0;
+}
+
+static void
+cmd_profile_volume_brick_out(dict_t *dict, int count, int interval)
+{
+    char key[256] = {0};
+    int i = 0;
+    uint64_t sec = 0;
+    uint64_t r_count = 0;
+    uint64_t w_count = 0;
+    uint64_t rb_counts[32] = {0};
+    uint64_t wb_counts[32] = {0};
+    cli_profile_info_t profile_info[GF_FOP_MAXVALUE] = {{0}};
+    cli_profile_info_t upcall_info[GF_UPCALL_FLAGS_MAXVALUE] = {
+        {0},
+    };
+    char output[128] = {0};
+    int per_line = 0;
+    char read_blocks[128] = {0};
+    char write_blocks[128] = {0};
+    int index = 0;
+    int is_header_printed = 0;
+    int ret = 0;
+    double total_percentage_latency = 0;
+
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &rb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+    }
+
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &wb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &upcall_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+        upcall_info[i].fop_name = (char *)gf_upcall_list[i];
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &profile_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].avg_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].min_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].max_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+        }
+        profile_info[i].fop_name = (char *)gf_fop_list[i];
+
+        total_percentage_latency += (profile_info[i].fop_hits *
+                                     profile_info[i].avg_latency);
+    }
+    if (total_percentage_latency) {
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            profile_info[i]
+                .percentage_avg_latency = 100 * ((profile_info[i].avg_latency *
+                                                  profile_info[i].fop_hits) /
+                                                 total_percentage_latency);
+        }
+        gf_array_insertionsort(profile_info, 1, GF_FOP_MAXVALUE - 1,
+                               sizeof(cli_profile_info_t),
+                               cli_profile_info_percentage_cmp);
+    }
+    snprintf(key, sizeof(key), "%d-%d-duration", count, interval);
+    ret = dict_get_uint64(dict, key, &sec);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", count, interval);
+    ret = dict_get_uint64(dict, key, &r_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", count, interval);
+    ret = dict_get_uint64(dict, key, &w_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    if (interval == -1)
+        cli_out("Cumulative Stats:");
+    else
+        cli_out("Interval %d Stats:", interval);
+    snprintf(output, sizeof(output), "%14s", "Block Size:");
+    snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+    snprintf(write_blocks, sizeof(write_blocks), "%14s", "No. of Writes:");
+    index = 14;
+    for (i = 0; i < 32; i++) {
+        if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
+            continue;
+        per_line++;
+        snprintf(output + index, sizeof(output) - index, "%19" PRIu32 "b+ ",
+                 (1U << i));
+        if (rb_counts[i]) {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index,
+                     "%21" PRId64 " ", rb_counts[i]);
+        } else {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index, "%21s ",
+                     "0");
+        }
+        if (wb_counts[i]) {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21" PRId64 " ", wb_counts[i]);
+        } else {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21s ", "0");
+        }
+        index += 22;
+        if (per_line == 3) {
+            cli_out("%s", output);
+            cli_out("%s", read_blocks);
+            cli_out("%s", write_blocks);
+            cli_out(" ");
+            per_line = 0;
+            snprintf(output, sizeof(output), "%14s", "Block Size:");
+            snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+            snprintf(write_blocks, sizeof(write_blocks), "%14s",
+                     "No. of Writes:");
+            index = 14;
+        }
+    }
+
+    if (per_line != 0) {
+        cli_out("%s", output);
+        cli_out("%s", read_blocks);
+        cli_out("%s", write_blocks);
+    }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        if (profile_info[i].fop_hits == 0)
+            continue;
+        if (is_header_printed == 0) {
+            cli_out("%10s %13s %13s %13s %14s %11s", "%-latency", "Avg-latency",
+                    "Min-Latency", "Max-Latency", "No. of calls", "Fop");
+            cli_out("%10s %13s %13s %13s %14s %11s", "---------", "-----------",
+                    "-----------", "-----------", "------------", "----");
+            is_header_printed = 1;
+        }
+        if (profile_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                profile_info[i].percentage_avg_latency,
+                profile_info[i].avg_latency, profile_info[i].min_latency,
+                profile_info[i].max_latency, profile_info[i].fop_hits,
+                profile_info[i].fop_name);
+        }
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        if (upcall_info[i].fop_hits == 0)
+            continue;
+        if (upcall_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                upcall_info[i].percentage_avg_latency,
+                upcall_info[i].avg_latency, upcall_info[i].min_latency,
+                upcall_info[i].max_latency, upcall_info[i].fop_hits,
+                upcall_info[i].fop_name);
+        }
+    }
+
+    cli_out(" ");
+    cli_out("%12s: %" PRId64 " seconds", "Duration", sec);
+    cli_out("%12s: %" PRId64 " bytes", "Data Read", r_count);
+    cli_out("%12s: %" PRId64 " bytes", "Data Written", w_count);
+    cli_out(" ");
+}
+
+static int32_t
+gf_cli_profile_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[64] = {0};
+    int len;
+    int interval = 0;
+    int i = 1;
+    int32_t brick_count = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    char str[1024] = {
+        0,
+    };
+    int stats_cleared = 0;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to profile");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+        cli_err("%s", rsp.op_errstr);
+    } else {
+        switch (op) {
+            case GF_CLI_STATS_START:
+                cli_out("Starting volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_STOP:
+                cli_out("Stopping volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_INFO:
+                break;
+            default:
+                cli_out("volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+        }
+    }
+
+    if (rsp.op_ret) {
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    if (GF_CLI_STATS_INFO != op) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "info-op", (int32_t *)&info_op);
+    if (ret)
+        goto out;
+
+    while (i <= brick_count) {
+        len = snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_strn(dict, key, len, &brick);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get brick name");
+            goto out;
+        }
+
+        ret = dict_get_str_boolean(dict, "nfs", _gf_false);
+
+        if (ret)
+            len = snprintf(str, sizeof(str), "NFS Server : %s", brick);
+        else
+            len = snprintf(str, sizeof(str), "Brick: %s", brick);
+        cli_out("%s", str);
+        memset(str, '-', len);
+        cli_out("%s", str);
+
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            len = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, len, &stats_cleared);
+            if (ret)
+                goto out;
+            cli_out(stats_cleared ? "Cleared stats."
+                                  : "Failed to clear stats.");
+        } else {
+            len = snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
+
+            len = snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
+        }
+        i++;
+    }
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_profile_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_profile_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_top_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[256] = {0};
+    int keylen;
+    int i = 0;
+    int32_t brick_count = 0;
+    char brick[1024];
+    int32_t members = 0;
+    char *filename;
+    char *bricks;
+    uint64_t value = 0;
+    int32_t j = 0;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    uint64_t nr_open = 0;
+    uint64_t max_nr_open = 0;
+    double throughput = 0;
+    double time = 0;
+    int32_t time_sec = 0;
+    long int time_usec = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *openfd_str = NULL;
+    gf_boolean_t nfs = _gf_false;
+    gf_boolean_t clear_stats = _gf_false;
+    int stats_cleared = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to top");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("volume top unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
+
+    if (op != GF_CLI_STATS_TOP) {
+        ret = 0;
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_top(dict, rsp.op_ret, rsp.op_errno,
+                                     rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        }
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    keylen = snprintf(key, sizeof(key), "%d-top-op", 1);
+    ret = dict_get_int32n(dict, key, keylen, (int32_t *)&top_op);
+    if (ret)
+        goto out;
+
+    clear_stats = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+
+    while (i < brick_count) {
+        i++;
+        keylen = snprintf(brick, sizeof(brick), "%d-brick", i);
+        ret = dict_get_strn(dict, brick, keylen, &bricks);
+        if (ret)
+            goto out;
+
+        nfs = dict_get_str_boolean(dict, "nfs", _gf_false);
+
+        if (clear_stats) {
+            keylen = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, keylen, &stats_cleared);
+            if (ret)
+                goto out;
+            cli_out(stats_cleared ? "Cleared stats for %s %s"
+                                  : "Failed to clear stats for %s %s",
+                    nfs ? "NFS server on" : "brick", bricks);
+            continue;
+        }
+
+        if (nfs)
+            cli_out("NFS Server : %s", bricks);
+        else
+            cli_out("Brick: %s", bricks);
+
+        keylen = snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32n(dict, key, keylen, &members);
+
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &nr_open);
+                if (ret)
+                    break;
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_nr_open);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_strn(dict, key, keylen, &openfd_str);
+                if (ret)
+                    goto out;
+                cli_out("Current open fds: %" PRIu64 ", Max open fds: %" PRIu64
+                        ", Max openfd time: %s",
+                        nr_open, max_nr_open, openfd_str);
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
+                if (!members) {
+                    continue;
+                }
+                cli_out("Count\t\tfilename\n=======================");
+                break;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time);
+                }
+                if (!ret)
+                    cli_out("Throughput %.2f MBps time %.4f secs", throughput,
+                            time / 1e6);
+
+                if (!members) {
+                    continue;
+                }
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH, "MBps",
+                        VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
+                        VOL_TOP_PERF_TIME_WIDTH, "Time");
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                        "====", VOL_TOP_PERF_FILENAME_DEF_WIDTH,
+                        "========", VOL_TOP_PERF_TIME_WIDTH, "====");
+                break;
+            default:
+                goto out;
+        }
+
+        for (j = 1; j <= members; j++) {
+            keylen = snprintf(key, sizeof(key), "%d-filename-%d", i, j);
+            ret = dict_get_strn(dict, key, keylen, &filename);
+            if (ret)
+                break;
+            snprintf(key, sizeof(key), "%d-value-%d", i, j);
+            ret = dict_get_uint64(dict, key, &value);
+            if (ret)
+                goto out;
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                keylen = snprintf(key, sizeof(key), "%d-time-sec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_sec);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-time-usec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_usec);
+                if (ret)
+                    goto out;
+                gf_time_fmt(timestr, sizeof timestr, time_sec, gf_timefmt_FT);
+                snprintf(timestr + strlen(timestr),
+                         sizeof timestr - strlen(timestr), ".%ld", time_usec);
+                if (strlen(filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
+                    cli_out("%*" PRIu64 " %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                            value, VOL_TOP_PERF_FILENAME_DEF_WIDTH, filename,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
+                else
+                    cli_out("%*" PRIu64 " ...%-*s %-*s",
+                            VOL_TOP_PERF_SPEED_WIDTH, value,
+                            VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            filename + strlen(filename) -
+                                VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
+            } else {
+                cli_out("%" PRIu64 "\t\t%s", value, filename);
+            }
+        }
+    }
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+
+    if (dict)
+        dict_unref(dict);
+
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_top_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_top_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int
+gf_cli_getwd_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        cli_err("getwd failed");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to getwd");
+
+    cli_out("%s", rsp.wd);
+
+    ret = 0;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    if (rsp.wd) {
+        free(rsp.wd);
+    }
+
+    return ret;
+}
+
+static int32_t
+gf_cli_getwd(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf1_cli_getwd_req req = {
+        0,
+    };
+
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+
+    if (!frame || !this)
+        goto out;
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_GETWD,
+                         NULL, this, gf_cli_getwd_cbk,
+                         (xdrproc_t)xdr_gf1_cli_getwd_req);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
+}
+
+static void
+cli_print_volume_status_mempool(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    int32_t mempool_count = 0;
+    char *name = NULL;
+    int32_t hotcount = 0;
+    int32_t coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int32_t maxalloc = 0;
+    uint64_t pool_misses = 0;
+    int32_t maxstdalloc = 0;
+    char key[128] = {
+        /* prefix is really small 'brick%d' really */
+        0,
+    };
+    int keylen;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    keylen = snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32n(dict, key, keylen, &mempool_count);
+    if (ret)
+        goto out;
+
+    cli_out("Mempool Stats\n-------------");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+            "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc", "Misses",
+            "Max-StdAlloc");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+            "---------", "------------", "----------", "--------", "--------",
+            "------------");
+
+    for (i = 0; i < mempool_count; i++) {
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_strn(dict, key, keylen, &name);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &hotcount);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &coldcount);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &maxalloc);
+        if (ret)
+            goto out;
+
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix,
+                          i);
+        ret = dict_get_int32n(dict, key, keylen, &maxstdalloc);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &pool_misses);
+        if (ret)
+            goto out;
+
+        cli_out("%-30s %9d %9d %12" PRIu64 " %10" PRIu64 " %8d %8" PRIu64
+                " %12d",
+                name, hotcount, coldcount, paddedsizeof, alloccount, maxalloc,
+                pool_misses, maxstdalloc);
+    }
+
+out:
+    return;
+}
+
+static void
+cli_print_volume_status_mem(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    char key[64] = {
+        0,
+    };
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int val = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Memory status for volume : %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            continue;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            continue;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
+
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        cli_out("Mallinfo\n--------");
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.arena", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Arena", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Ordblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Smblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblkhd", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Usmblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Fsmblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Uordblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Fordblks", val);
+
+        snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Keepcost", val);
+
+        cli_out(" ");
+        snprintf(key, sizeof(key), "brick%d", i);
+        cli_print_volume_status_mempool(dict, key);
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
+
+static void
+cli_print_volume_status_client_list(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int client_count = 0;
+    int current_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int total = 0;
+    char *name = NULL;
+    gf_boolean_t is_fuse_done = _gf_false;
+    gf_boolean_t is_gfapi_done = _gf_false;
+    gf_boolean_t is_rebalance_done = _gf_false;
+    gf_boolean_t is_glustershd_done = _gf_false;
+    gf_boolean_t is_quotad_done = _gf_false;
+    gf_boolean_t is_snapd_done = _gf_false;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "client-count", &client_count);
+    if (ret)
+        goto out;
+
+    cli_out("%-48s %15s", "Name", "count");
+    cli_out("%-48s %15s", "-----", "------");
+    for (i = 0; i < client_count; i++) {
+        name = NULL;
+        snprintf(key, sizeof(key), "client%d.name", i);
+        ret = dict_get_str(dict, key, &name);
+
+        if (!strncmp(name, "fuse", 4)) {
+            if (!is_fuse_done) {
+                is_fuse_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "fuse-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strncmp(name, "gfapi", 5)) {
+            if (!is_gfapi_done) {
+                is_gfapi_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "gfapi-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "rebalance")) {
+            if (!is_rebalance_done) {
+                is_rebalance_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "rebalance-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "glustershd")) {
+            if (!is_glustershd_done) {
+                is_glustershd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "glustershd-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "quotad")) {
+            if (!is_quotad_done) {
+                is_quotad_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "quotad-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "snapd")) {
+            if (!is_snapd_done) {
+                is_snapd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "snapd-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        }
+
+    print:
+        cli_out("%-48s %15d", name, current_count);
+    }
+out:
+    cli_out("\ntotal clients for volume %s : %d ", volname, total);
+    cli_out(
+        "-----------------------------------------------------------------\n");
+    return;
+}
+
+static void
+cli_print_volume_status_clients(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int client_count = 0;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+    char key[128] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        hostname = NULL;
+        path = NULL;
+        online = -1;
+        client_count = 0;
+        clientname = NULL;
+        bytesread = 0;
+        byteswrite = 0;
+
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+
+        if (hostname && path) {
+            if (notbrick)
+                cli_out("%s : %s", hostname, path);
+            else
+                cli_out("Brick : %s:%s", hostname, path);
+        }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.clientcount", i);
+        ret = dict_get_int32(dict, key, &client_count);
+
+        if (hostname && path)
+            cli_out("Clients connected : %d", client_count);
+        if (client_count == 0)
+            continue;
+
+        cli_out("%-48s %15s %15s %15s", "Hostname", "BytesRead", "BytesWritten",
+                "OpVersion");
+        cli_out("%-48s %15s %15s %15s", "--------", "---------", "------------",
+                "---------");
+        for (j = 0; j < client_count; j++) {
+            snprintf(key, sizeof(key), "brick%d.client%d.hostname", i, j);
+            ret = dict_get_str(dict, key, &clientname);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.bytesread", i, j);
+            ret = dict_get_uint64(dict, key, &bytesread);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", i, j);
+            ret = dict_get_uint64(dict, key, &byteswrite);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.opversion", i, j);
+            ret = dict_get_uint32(dict, key, &opversion);
+
+            cli_out("%-48s %15" PRIu64 " %15" PRIu64 " %15" PRIu32, clientname,
+                    bytesread, byteswrite, opversion);
+        }
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
+
+#ifdef DEBUG /* this function is only used in debug */
+static void
+cli_print_volume_status_inode_entry(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char inode_type;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+
+    switch (ia_type) {
+        case IA_IFREG:
+            inode_type = 'R';
+            break;
+        case IA_IFDIR:
+            inode_type = 'D';
+            break;
+        case IA_IFLNK:
+            inode_type = 'L';
+            break;
+        case IA_IFBLK:
+            inode_type = 'B';
+            break;
+        case IA_IFCHR:
+            inode_type = 'C';
+            break;
+        case IA_IFIFO:
+            inode_type = 'F';
+            break;
+        case IA_IFSOCK:
+            inode_type = 'S';
+            break;
+        default:
+            inode_type = 'I';
+            break;
+    }
+
+    cli_out("%-40s %14" PRIu64 " %14" PRIu32 " %9c", gfid, nlookup, ref,
+            inode_type);
+
+out:
+    return;
+}
+#endif
+
+static void
+cli_print_volume_status_itables(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    uint32_t lru_limit = 0;
+#ifdef DEBUG
+    int i = 0;
+#endif
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.lru_limit", prefix);
+    ret = dict_get_uint32(dict, key, &lru_limit);
+    if (ret)
+        goto out;
+    cli_out("LRU limit     : %u", lru_limit);
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (active_size != 0) {
+        cli_out("Active inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < active_size; i++) {
+        snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("Active Inodes : %u", active_size);
+
+#endif
+
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (lru_size != 0) {
+        cli_out("LRU inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < lru_size; i++) {
+        snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("LRU Inodes    : %u", lru_size);
+#endif
+
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+#ifdef DEBUG
+    if (purge_size != 0) {
+        cli_out("Purged inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < purge_size; i++) {
+        snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+#else
+    cli_out("Purge Inodes  : %u", purge_size);
+#endif
+
+out:
+    return;
+}
+
+static void
+cli_print_volume_status_inode(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Inode tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            goto out;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
+
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
+
+        for (j = 0; j < conn_count; j++) {
+            if (conn_count > 1)
+                cli_out("Connection %d:", j + 1);
+
+            snprintf(key, sizeof(key), "brick%d.conn%d.itable", i, j);
+            cli_print_volume_status_itables(dict, key);
+            cli_out(" ");
+        }
+    }
+out:
+    cli_out("----------------------------------------------");
+    return;
+}
+
+void
+cli_print_volume_status_fdtable(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+
+    cli_out("RefCount = %d  MaxFDs = %d  FirstFree = %d", refcount, maxfds,
+            firstfree);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    if (0 == openfds) {
+        cli_err("No open fds");
+        goto out;
+    }
+
+    cli_out("%-19s %-19s %-19s %-19s", "FD Entry", "PID", "RefCount", "Flags");
+    cli_out("%-19s %-19s %-19s %-19s", "--------", "---", "--------", "-----");
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
+        if (ret)
+            continue;
+
+        cli_out("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount, fd_flags);
+    }
+
+out:
+    return;
+}
+
+static void
+cli_print_volume_status_fd(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("FD tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            goto out;
+
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
+
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
+
+        for (j = 0; j < conn_count; j++) {
+            cli_out("Connection %d:", j + 1);
+
+            snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", i, j);
+            cli_print_volume_status_fdtable(dict, key);
+            cli_out(" ");
+        }
+    }
+out:
+    cli_out("----------------------------------------------");
+    return;
+}
+
+static void
+cli_print_volume_status_call_frame(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int ref_count = 0;
+    char *translator = 0;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+
+    if (!dict || !prefix)
+        return;
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        return;
+
+    cli_out("  Ref Count   = %d", ref_count);
+    cli_out("  Translator  = %s", translator);
+    cli_out("  Completed   = %s", (complete ? "Yes" : "No"));
+
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret)
+        cli_out("  Parent      = %s", parent);
+
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret)
+        cli_out("  Wind From   = %s", wind_from);
+
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret)
+        cli_out("  Wind To     = %s", wind_to);
+
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret)
+        cli_out("  Unwind From = %s", unwind_from);
+
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret)
+        cli_out("  Unwind To   = %s", unwind_to);
+}
+
+static void
+cli_print_volume_status_call_stack(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    // char            *op = NULL;
+    int count = 0;
+    int i = 0;
+
+    if (!prefix)
+        return;
+
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        return;
+
+    /*
+    snprintf (key, sizeof (key), "%s.op", prefix);
+    ret = dict_get_str (dict, key, &op);
+    if (ret)
+            return;
+    */
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        return;
+
+    cli_out(" UID    : %d", uid);
+    cli_out(" GID    : %d", gid);
+    cli_out(" PID    : %d", pid);
+    cli_out(" Unique : %" PRIu64, unique);
+    // cli_out ("\tOp     : %s", op);
+    cli_out(" Frames : %d", count);
+
+    for (i = 0; i < count; i++) {
+        cli_out(" Frame %d", i + 1);
+
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        cli_print_volume_status_call_frame(dict, key);
+    }
+
+    cli_out(" ");
+}
+
+static void
+cli_print_volume_status_callpool(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int call_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Pending calls for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            goto out;
+
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
+
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.callpool.count", i);
+        ret = dict_get_int32(dict, key, &call_count);
+        if (ret)
+            goto out;
+        cli_out("Pending calls: %d", call_count);
+
+        if (0 == call_count)
+            continue;
+
+        for (j = 0; j < call_count; j++) {
+            cli_out("Call Stack%d", j + 1);
+
+            snprintf(key, sizeof(key), "brick%d.callpool.stack%d", i, j);
+            cli_print_volume_status_call_stack(dict, key);
+        }
+    }
+
+out:
+    cli_out("----------------------------------------------");
+    return;
+}
+
+static void
+cli_print_volume_status_tasks(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+    int count = 0;
+    int task_count = 0;
+    int status = 0;
+    char *op = NULL;
+    char *task_id_str = NULL;
+    char *volname = NULL;
+    char key[64] = {
+        0,
+    };
+    char task[32] = {
+        0,
+    };
+    char *brick = NULL;
+
+    ret = dict_get_int32_sizen(dict, "tasks", &task_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get tasks count");
+        return;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    cli_out("Task Status of Volume %s", volname);
+    cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+
+    if (task_count == 0) {
+        cli_out("There are no active volume tasks");
+        cli_out(" ");
+        return;
+    }
+
+    for (i = 0; i < task_count; i++) {
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &op);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "Task", op);
+
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "ID", task_id_str);
+
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            return;
+
+        snprintf(task, sizeof(task), "task%d", i);
+
+        if (!strcmp(op, "Remove brick")) {
+            snprintf(key, sizeof(key), "%s.count", task);
+            ret = dict_get_int32(dict, key, &count);
+            if (ret)
+                goto out;
+
+            cli_out("%-20s", "Removed bricks:");
+
+            for (j = 1; j <= count; j++) {
+                snprintf(key, sizeof(key), "%s.brick%d", task, j);
+                ret = dict_get_str(dict, key, &brick);
+                if (ret)
+                    goto out;
+
+                cli_out("%-20s", brick);
+            }
+        }
+        cli_out("%-20s : %-20s", "Status", cli_vol_task_status_str[status]);
+        cli_out(" ");
+    }
+
+out:
+    return;
+}
+
+static int
+gf_cli_status_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int i = 0;
+    int type = -1;
+    int hot_brick_count = -1;
+    int pid = -1;
+    uint32_t cmd = 0;
+    gf_boolean_t notbrick = _gf_false;
+    char key[64] = {
+        0,
+    };
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_volume_status_t status = {0};
+    cli_local_t *local = NULL;
+    gf_boolean_t wipe_local = _gf_false;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received response to status cmd");
+
+    local = ((call_frame_t *)myframe)->local;
+    if (!local) {
+        local = cli_local_get();
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get local");
+            goto out;
+        }
+        wipe_local = _gf_true;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg),
+                     "Unable to obtain volume status information.");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            if (!local->all)
+                cli_xml_output_str("volStatus", msg, rsp.op_ret, rsp.op_errno,
+                                   rsp.op_errstr);
+            ret = 0;
+            goto out;
+        }
+
+        cli_err("%s", msg);
+        if (local && local->all) {
+            ret = 0;
+            cli_out(" ");
+        } else
+            ret = -1;
+
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_ALL)) {
+        if (local && local->dict) {
+            dict_ref(dict);
+            ret = dict_set_static_ptr(local->dict, "rsp-dict", dict);
+            ret = 0;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "local not found");
+            ret = -1;
+        }
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_begin(local, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+        if (cmd & GF_CLI_STATUS_TASKS) {
+            ret = cli_xml_output_vol_status_tasks_detail(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Error outputting to xml");
+                goto xml_end;
+            }
+        } else {
+            ret = cli_xml_output_vol_status(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+
+    xml_end:
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_end(local);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        }
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
+        (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
+        (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
+        notbrick = _gf_true;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            cli_print_volume_status_mem(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENTS:
+            cli_print_volume_status_clients(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENT_LIST:
+            cli_print_volume_status_client_list(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_INODE:
+            cli_print_volume_status_inode(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_FD:
+            cli_print_volume_status_fd(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CALLPOOL:
+            cli_print_volume_status_callpool(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_TASKS:
+            cli_print_volume_status_tasks(dict);
+            goto cont;
+            break;
+        default:
+            break;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "hot_brick_count", &hot_brick_count);
+    if (ret)
+        goto out;
+
+    cli_out("Status of volume: %s", volname);
+
+    if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
+        cli_out("%-*s %s  %s  %s  %s", CLI_VOL_STATUS_BRICK_LEN,
+                "Gluster process", "TCP Port", "RDMA Port", "Online", "Pid");
+        cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+    }
+
+    status.brick = GF_MALLOC(PATH_MAX + 256, gf_common_mt_strdup);
+    if (!status.brick) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i <= index_max; i++) {
+        status.rdma_port = 0;
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+        if (ret)
+            continue;
+
+        /* Brick/not-brick is handled separately here as all
+         * types of nodes are contained in the default output
+         */
+        status.brick[0] = '\0';
+        if (!strcmp(hostname, "NFS Server") ||
+            !strcmp(hostname, "Self-heal Daemon") ||
+            !strcmp(hostname, "Quota Daemon") ||
+            !strcmp(hostname, "Snapshot Daemon") ||
+            !strcmp(hostname, "Scrubber Daemon") ||
+            !strcmp(hostname, "Bitrot Daemon"))
+            snprintf(status.brick, PATH_MAX + 255, "%s on %s", hostname, path);
+        else {
+            snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+            ret = dict_get_int32(dict, key, &(status.rdma_port));
+            if (ret)
+                continue;
+            snprintf(status.brick, PATH_MAX + 255, "Brick %s:%s", hostname,
+                     path);
+        }
+
+        snprintf(key, sizeof(key), "brick%d.port", i);
+        ret = dict_get_int32(dict, key, &(status.port));
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &(status.online));
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "brick%d.pid", i);
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret)
+            continue;
+        if (pid == -1)
+            ret = gf_asprintf(&(status.pid_str), "%s", "N/A");
+        else
+            ret = gf_asprintf(&(status.pid_str), "%d", pid);
+
+        if (ret == -1)
+            goto out;
+
+        if ((cmd & GF_CLI_STATUS_DETAIL)) {
+            ret = cli_get_detail_status(dict, i, &status);
+            if (ret)
+                goto out;
+            cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+            cli_print_detailed_status(&status);
+        } else {
+            cli_print_brick_status(&status);
+        }
+
+        /* Allocatated memory using gf_asprintf*/
+        GF_FREE(status.pid_str);
+    }
+    cli_out(" ");
+
+    if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
+        cli_print_volume_status_tasks(dict);
+cont:
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(status.brick);
+    if (local && wipe_local) {
+        cli_local_wipe(local);
+    }
+
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_status_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_status_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int
+gf_cli_status_volume_all(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int i = 0;
+    int ret = -1;
+    int vol_count = -1;
+    uint32_t cmd = 0;
+    char key[1024] = {0};
+    char *volname = NULL;
+    void *vol_dict = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (!frame)
+        goto out;
+
+    if (!frame->local)
+        goto out;
+
+    local = frame->local;
+
+    ret = dict_get_uint32(local->dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    local->all = _gf_true;
+
+    ret = gf_cli_status_volume(frame, this, data);
+
+    if (ret)
+        goto out;
+
+    ret = dict_get_ptr(local->dict, "rsp-dict", &vol_dict);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen((dict_t *)vol_dict, "vol_count", &vol_count);
+    if (ret) {
+        cli_err("Failed to get names of volumes");
+        goto out;
+    }
+
+    /* remove the "all" flag in cmd */
+    cmd &= ~GF_CLI_STATUS_ALL;
+    cmd |= GF_CLI_STATUS_VOL;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        // TODO: Pass proper op_* values
+        ret = cli_xml_output_vol_status_begin(local, 0, 0, NULL);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto xml_end;
+        }
+    }
+
+    if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; i < vol_count; i++) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+
+        ret = snprintf(key, sizeof(key), "vol%d", i);
+        ret = dict_get_strn(vol_dict, key, ret, &volname);
+        if (ret)
+            goto out;
+
+        ret = dict_set_str_sizen(dict, "volname", volname);
+        if (ret)
+            goto out;
+
+        ret = dict_set_uint32(dict, "cmd", cmd);
+        if (ret)
+            goto out;
+
+        ret = gf_cli_status_volume(frame, this, dict);
+        if (ret)
+            goto out;
+
+        dict_unref(dict);
+    }
+
+xml_end:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_status_end(local);
+    }
+
+out:
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "status all failed");
+
+    if (vol_dict)
+        dict_unref(vol_dict);
+
+    if (ret && dict)
+        dict_unref(dict);
+
+    if (local)
+        cli_local_wipe(local);
+
+    if (frame)
+        frame->local = NULL;
+
+    return ret;
+}
+
+static int
+gf_cli_mount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf1_cli_mount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
+
+    if (rsp.op_ret == 0) {
+        ret = 0;
+        cli_out("%s", rsp.path);
+    } else {
+        /* weird sounding but easy to parse... */
+        cli_err("%d : failed with this errno (%s)", rsp.op_errno,
+                strerror(rsp.op_errno));
+        ret = -1;
+    }
+
+out:
+    cli_cmd_broadcast_response(ret);
+    if (rsp.path) {
+        free(rsp.path);
+    }
+    return ret;
+}
+
+static int32_t
+gf_cli_mount(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf1_cli_mount_req req = {
+        0,
+    };
+    int ret = -1;
+    void **dataa = data;
+    char *label = NULL;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data)
+        goto out;
+
+    label = dataa[0];
+    dict = dataa[1];
+
+    req.label = label;
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_MOUNT,
+                         NULL, this, gf_cli_mount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_mount_req);
+
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int
+gf_cli_umount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    gf1_cli_umount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
+
+    if (rsp.op_ret == 0)
+        ret = 0;
+    else {
+        cli_err("umount failed");
+        ret = -1;
+    }
+
+out:
+    cli_cmd_broadcast_response(ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_umount(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf1_cli_umount_req req = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data)
+        goto out;
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "path", &req.path);
+    if (ret == 0)
+        ret = dict_get_int32_sizen(dict, "lazy", &req.lazy);
+
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_UMOUNT,
+                         NULL, this, gf_cli_umount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_umount_req);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static void
+cmd_heal_volume_statistics_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[256] = {0};
+    char *hostname = NULL;
+    uint64_t i = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = NULL;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    cli_out("------------------------------------------------");
+    cli_out("\nCrawl statistics for brick no %d", brick);
+    cli_out("Hostname of brick %s", hostname);
+
+    snprintf(key, sizeof key, "statistics-%d-count", brick);
+    ret = dict_get_uint64(dict, key, &num_entries);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < num_entries; i++) {
+        snprintf(key, sizeof key, "statistics_crawl_type-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_str(dict, key, &crawl_type);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof key, "statistics_healed_cnt-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_uint64(dict, key, &healed_count);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof key, "statistics_sb_cnt-%d-%" PRIu64, brick, i);
+        ret = dict_get_uint64(dict, key, &split_brain_count);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_heal_failed_cnt-%d-%" PRIu64,
+                 brick, i);
+        ret = dict_get_uint64(dict, key, &heal_failed_count);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_strt_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &start_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_end_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &end_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_inprogress-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_int32(dict, key, &progress);
+        if (ret)
+            goto out;
+
+        cli_out("\nStarting time of crawl: %s", start_time_str);
+        if (progress == 1)
+            cli_out("Crawl is in progress");
+        else
+            cli_out("Ending time of crawl: %s", end_time_str);
+
+        cli_out("Type of crawl: %s", crawl_type);
+        cli_out("No. of entries healed: %" PRIu64, healed_count);
+        cli_out("No. of entries in split-brain: %" PRIu64, split_brain_count);
+        cli_out("No. of heal failed entries: %" PRIu64, heal_failed_count);
+    }
+
+out:
+    return;
+}
+
+static void
+cmd_heal_volume_brick_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    uint64_t i = 0;
+    uint32_t time = 0;
+    char timestr[GF_TIMESTR_SIZE] = {0};
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && status[0] != '\0')
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-count", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
+        cli_out("Number of entries: %" PRIu64, num_entries);
+
+        for (i = 0; i < num_entries; i++) {
+            snprintf(key, sizeof key, "%d-%" PRIu64, brick, i);
+            ret = dict_get_str(dict, key, &path);
+            if (ret)
+                continue;
+            time = 0;
+            snprintf(key, sizeof key, "%d-%" PRIu64 "-time", brick, i);
+            ret = dict_get_uint32(dict, key, &time);
+            if (ret || !time) {
+                cli_out("%s", path);
+            } else {
+                gf_time_fmt(timestr, sizeof timestr, time, gf_timefmt_FT);
+                if (i == 0) {
+                    cli_out("at                    path on brick");
+                    cli_out("-----------------------------------");
+                }
+                cli_out("%s %s", timestr, path);
+            }
+        }
+    }
+
+out:
+    return;
+}
+
+static void
+cmd_heal_volume_statistics_heal_count_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && strlen(status))
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-hardlinks", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
+        if (ret)
+            cli_out("No gathered input for this brick");
+        else
+            cli_out("Number of entries: %" PRIu64, num_entries);
+    }
+
+out:
+    return;
+}
+
+static int
+gf_is_cli_heal_get_command(gf_xl_afr_op_t heal_op)
+{
+    /* If the command is get command value is 1 otherwise 0, for
+       invalid commands -1 */
+    static int get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = -1,
+        [GF_SHD_OP_HEAL_INDEX] = 0,
+        [GF_SHD_OP_HEAL_FULL] = 0,
+        [GF_SHD_OP_INDEX_SUMMARY] = 1,
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
+        [GF_SHD_OP_STATISTICS] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
+        [GF_SHD_OP_HEAL_ENABLE] = 0,
+        [GF_SHD_OP_HEAL_DISABLE] = 0,
+    };
+
+    if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
+        return get_cmds[heal_op] == 1;
+    return _gf_false;
+}
+
+int
+gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int brick_count = 0;
+    int i = 0;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    const char *operation = NULL;
+    const char *substr = NULL;
+    const char *heal_op_str = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "heal-op", (int32_t *)&heal_op);
+    // TODO: Proper XML output
+    //#if (HAVE_LIB_XML)
+    //        if (global_state->mode & GLUSTER_MODE_XML) {
+    //                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
+    //                                           rsp.op_errno, rsp.op_errstr);
+    //                if (ret)
+    //                        gf_log ("cli", GF_LOG_ERROR, XML_ERROR);
+    //                goto out;
+    //        }
+    //#endif
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to heal volume");
+
+    operation = "Gathering ";
+    substr = "";
+    switch (heal_op) {
+        case GF_SHD_OP_HEAL_INDEX:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform index self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_HEAL_FULL:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform full self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            heal_op_str = "list of entries to be healed";
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            heal_op_str = "list of split brain entries";
+            break;
+        case GF_SHD_OP_STATISTICS:
+            heal_op_str = "crawl statistics";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+            heal_op_str = "count of entries to be healed";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            heal_op_str = "count of entries to be healed per replica";
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+        case GF_SHD_OP_HEAL_SUMMARY:
+        case GF_SHD_OP_HEALED_FILES:
+        case GF_SHD_OP_HEAL_FAILED_FILES:
+            /* These cases are never hit; they're coded just to silence the
+             * compiler warnings.*/
+            break;
+
+        case GF_SHD_OP_INVALID:
+            heal_op_str = "invalid heal op";
+            break;
+        case GF_SHD_OP_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable heal";
+            break;
+        case GF_SHD_OP_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable granular entry heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable granular entry heal";
+            break;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("%s%s on volume %s has been unsuccessful:", operation,
+                    heal_op_str, volname);
+            cli_err("%s", rsp.op_errstr);
+        }
+        ret = rsp.op_ret;
+        goto out;
+    } else {
+        cli_out("%s%s on volume %s has been successful %s", operation,
+                heal_op_str, volname, substr);
+    }
+
+    ret = rsp.op_ret;
+    if (!gf_is_cli_heal_get_command(heal_op))
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        ret = -1;
+        goto out;
+    }
+
+    switch (heal_op) {
+        case GF_SHD_OP_STATISTICS:
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_out(dict, i);
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_heal_count_out(dict, i);
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_brick_out(dict, i);
+            break;
+        default:
+            break;
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+static int32_t
+gf_cli_heal_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+
+    dict = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_heal_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog, NULL);
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_statedump_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "Volume statedump successful";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to statedump");
+    if (rsp.op_ret)
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volStatedump", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("volume statedump: failed: %s", msg);
+    else
+        cli_out("volume statedump: success");
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_statedump_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_statedump_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_list_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int vol_count = 0;
+    ;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_list(dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("%s", rsp.op_errstr);
+    else {
+        ret = dict_get_int32_sizen(dict, "count", &vol_count);
+        if (ret)
+            goto out;
+
+        if (vol_count == 0) {
+            cli_err("No volumes present in cluster");
+            goto out;
+        }
+        for (i = 0; i < vol_count; i++) {
+            ret = snprintf(key, sizeof(key), "volume%d", i);
+            ret = dict_get_strn(dict, key, ret, &volname);
+            if (ret)
+                goto out;
+            cli_out("%s", volname);
+        }
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+static int32_t
+gf_cli_list_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_LIST_VOLUME, NULL, this,
+                         gf_cli_list_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_clearlocks_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *lk_summary = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to clear-locks");
+
+    if (rsp.op_ret) {
+        cli_err("Volume clear-locks unsuccessful");
+        cli_err("%s", rsp.op_errstr);
+
+    } else {
+        if (!rsp.dict.dict_len) {
+            cli_err("Possibly no locks cleared");
+            ret = 0;
+            goto out;
+        }
+
+        dict = dict_new();
+
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+
+        ret = dict_get_str(dict, "lk-summary", &lk_summary);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to get lock summary from dictionary");
+            goto out;
+        }
+        cli_out("Volume clear-locks successful");
+        cli_out("%s", lk_summary);
+    }
+
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int32_t
+gf_cli_clearlocks_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_clearlocks_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+cli_snapshot_remove_reply(gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
+{
+    int32_t ret = -1;
+    char *snap_name = NULL;
+    int32_t delete_cmd = -1;
+    cli_local_t *local = NULL;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(rsp);
+    GF_ASSERT(dict);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
+        goto end;
+    }
+
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_begin(local, rsp->op_ret,
+                                               rsp->op_errno, rsp->op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for delete");
+            goto end;
+        }
+    }
+
+    if (rsp->op_ret && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("snapshot delete: failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
+        delete_cmd == GF_SNAP_DELETE_TYPE_VOL) {
+        local = ((call_frame_t *)frame)->local;
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+            goto out;
+        }
+
+        /* During first call back of snapshot delete of type
+         * ALL and VOL, We will get the snapcount and snapnames.
+         * Hence to make the subsequent rpc calls for individual
+         * snapshot delete, We need to save it in local dictionary.
+         */
+        dict_copy(dict, local->dict);
+        ret = 0;
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_delete(local, dict, rsp);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot delete command");
+            goto out;
+        }
+        /* Error out in case of the op already failed */
+        if (rsp->op_ret) {
+            ret = rsp->op_ret;
+            goto out;
+        }
+    } else {
+        ret = dict_get_str_sizen(dict, "snapname", &snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+            goto out;
+        }
+
+        cli_out("snapshot delete: %s: snap removed successfully", snap_name);
+    }
+    ret = 0;
+
+out:
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_end(local);
+    }
+end:
+    return ret;
+}
+
+static int
+cli_snapshot_config_display(dict_t *dict, gf_cli_rsp *rsp)
+{
+    char buf[PATH_MAX] = "";
+    char *volname = NULL;
+    int ret = -1;
+    int config_command = 0;
+    uint64_t value = 0;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    uint64_t i = 0;
+    uint64_t voldisplaycount = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+
+    if (rsp->op_ret) {
+        cli_err("Snapshot Config : failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* Ignore the error, as the key specified is optional */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+
+    ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+
+    ret = dict_get_str_sizen(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit &&
+        config_command != GF_SNAP_CONFIG_DISPLAY && !auto_delete &&
+        !snap_activate) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "Could not fetch config-key");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    /* Ignore the error, as volname is optional */
+
+    if (!volname) {
+        volname = "System";
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+            if (hard_limit && soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "& snap-max-soft-limit for system set successfully");
+            } else if (hard_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-soft-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (auto_delete) {
+                cli_out("snapshot config: auto-delete successfully set");
+            } else if (snap_activate) {
+                cli_out("snapshot config: activate-on-create successfully set");
+            }
+            break;
+
+        case GF_SNAP_CONFIG_DISPLAY:
+            cli_out("\nSnapshot System Configuration:");
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap_max_hard_limit for %s", volname);
+                ret = -1;
+                goto out;
+            }
+            cli_out("snap-max-hard-limit : %" PRIu64, value);
+
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap-max-soft-limit for %s", volname);
+                ret = -1;
+                goto out;
+            }
+            cli_out("snap-max-soft-limit : %" PRIu64 "%%", soft_limit);
+
+            cli_out("auto-delete : %s", auto_delete);
+
+            cli_out("activate-on-create : %s\n", snap_activate);
+
+            cli_out("Snapshot Volume Configuration:");
+
+            ret = dict_get_uint64(dict, "voldisplaycount", &voldisplaycount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch voldisplaycount");
+                ret = -1;
+                goto out;
+            }
+
+            for (i = 0; i < voldisplaycount; i++) {
+                ret = snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname",
+                               i);
+                ret = dict_get_strn(dict, buf, ret, &volname);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("\nVolume : %s", volname);
+
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("snap-max-hard-limit : %" PRIu64, value);
+
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-active-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Could not fetch"
+                           " effective snap_max_hard_limit for %s",
+                           volname);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-hard-limit : %" PRIu64, value);
+
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-soft-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-soft-limit : %" PRIu64
+                        " "
+                        "(%" PRIu64 "%%)",
+                        value, soft_limit);
+            }
+            break;
+        default:
+            break;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function is used to print the volume related information
+ * of a snap.
+ *
+ * arg - 0, dict       : Response Dictionary.
+ * arg - 1, prefix str : snaplist.snap{0..}.vol{0..}.*
+ */
+static int
+cli_get_each_volinfo_in_snap(dict_t *dict, char *keyprefix,
+                             gf_boolean_t snap_driven)
+{
+    char key[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int value = 0;
+    int ret = -1;
+    char indent[5] = "\t";
+    char *volname = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &get_buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
+        }
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap Volume Name", ":",
+                get_buffer);
+
+        ret = snprintf(key, sizeof(key), "%s.origin-volname", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+            cli_out("%-12s", "Origin:");
+        }
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Origin Volume name", ":",
+                volname);
+
+        ret = snprintf(key, sizeof(key), "%s.snapcount", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_int32(dict, key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
+        }
+        cli_out("%s%s %s      %s %d", indent, "Snaps taken for", volname, ":",
+                value);
+
+        ret = snprintf(key, sizeof(key), "%s.snaps-available", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_int32(dict, key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
+        }
+        cli_out("%s%s %s  %s %d", indent, "Snaps available for", volname, ":",
+                value);
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Status", ":", get_buffer);
+out:
+    return ret;
+}
+
+/* This function is used to print snap related information
+ * arg - 0, dict       : Response dictionary.
+ * arg - 1, prefix_str : snaplist.snap{0..}.*
+ */
+static int
+cli_get_volinfo_in_snap(dict_t *dict, char *keyprefix)
+{
+    char key[PATH_MAX] = "";
+    int i = 0;
+    int volcount = 0;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.vol-count", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, key, &volcount);
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list details of volume in a snap");
+            goto out;
+        }
+        cli_out(" ");
+    }
+
+out:
+    return ret;
+}
+
+static int
+cli_get_each_snap_info(dict_t *dict, char *prefix_str, gf_boolean_t snap_driven)
+{
+    char key_buffer[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int ret = -1;
+    char indent[5] = "";
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix_str);
+
+    if (!snap_driven)
+        strcat(indent, "\t");
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (!ret) {
+        /* Ignore error for description */
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Description", ":",
+                get_buffer);
+    }
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
+               prefix_str);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Created", ":", get_buffer);
+
+    if (snap_driven) {
+        cli_out("%-12s", "Snap Volumes:\n");
+        ret = cli_get_volinfo_in_snap(dict, prefix_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to list details of the snaps");
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+/* This is a generic function to print snap related information.
+ * arg - 0, dict : Response Dictionary
+ */
+static int
+cli_call_snapshot_info(dict_t *dict, gf_boolean_t bool_snap_driven)
+{
+    int snap_count = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_int32_sizen(dict, "snapcount", &snap_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapcount");
+        goto out;
+    }
+
+    if (snap_count == 0) {
+        cli_out("No snapshots present");
+    }
+
+    for (i = 1; i <= snap_count; i++) {
+        ret = snprintf(key, sizeof(key), "snap%d", i);
+        if (ret < 0) {
+            goto out;
+        }
+        ret = cli_get_each_snap_info(dict, key, bool_snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+static int
+cli_get_snaps_in_volume(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int count = 0;
+    int avail = 0;
+    char key[32] = "";
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "origin-volname", &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+
+    ret = dict_get_int32_sizen(dict, "snapcount", &avail);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snapcount");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
+
+    ret = dict_get_int32_sizen(dict, "snaps-available", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+
+    for (i = 1; i <= avail; i++) {
+        snprintf(key, sizeof(key), "snap%d", i);
+        ret = cli_get_each_snap_info(dict, key, _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
+        }
+
+        ret = snprintf(key, sizeof(key), "snap%d.vol1", i);
+        if (ret < 0) {
+            goto out;
+        }
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not get volume related information");
+            goto out;
+        }
+
+        cli_out(" ");
+    }
+out:
+    return ret;
+}
+
+static int
+cli_snapshot_list(dict_t *dict)
+{
+    int snapcount = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_int32_sizen(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snap count");
+        goto out;
+    }
+
+    if (snapcount == 0) {
+        cli_out("No snapshots present");
+    }
+
+    for (i = 1; i <= snapcount; i++) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_strn(dict, key, ret, &get_buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            cli_out("%s", get_buffer);
+        }
+    }
+out:
+    return ret;
+}
+
+static int
+cli_get_snap_volume_status(dict_t *dict, char *key_prefix)
+{
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(key_prefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", key_prefix);
+    if (ret < 0) {
+        goto out;
+    }
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    for (i = 0; i < brickcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Path");
+            continue;
+        }
+        cli_out("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Volume Group");
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", buffer);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", buffer);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            cli_out("\t%-17s %s   %s", "Brick PID", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %d", "Brick PID", ":", pid);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Data Percent");
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", buffer);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get LV Size");
+            cli_out("\t%-17s %s   %s", "LV Size", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "LV Size", ":", buffer);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+cli_get_single_snap_status(dict_t *dict, char *keyprefix)
+{
+    int ret = -1;
+    char key[64] = ""; /* keyprefix is ""status.snap0" */
+    int i = 0;
+    int volcount = 0;
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+    cli_out("\nSnap Name : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+    cli_out("Snap UUID : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    for (i = 0; i < volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = cli_get_snap_volume_status(dict, key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+static int32_t
+cli_populate_req_dict_for_delete(dict_t *snap_dict, dict_t *dict, size_t index)
+{
+    int32_t ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
+
+    ret = dict_set_int32_sizen(snap_dict, "sub-cmd", GF_SNAP_DELETE_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save command type in snap dictionary");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "snapname%zu", index);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save snapname");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_DELETE);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save command type");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot delete");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as delete");
+        goto out;
+    }
+out:
+    return ret;
+}
+
+static int
+cli_populate_req_dict_for_status(dict_t *snap_dict, dict_t *dict, int index)
+{
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
+
+    ret = dict_set_uint32(snap_dict, "sub-cmd", GF_SNAP_STATUS_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type in snap dict");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "status.snap%d.snapname", index);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, key, ret, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapname");
+        goto out;
+    }
+
+    ret = dict_set_str_sizen(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save snapname in snap dict");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_STATUS);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot status");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as status");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "hold_vol_locks", _gf_false);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Setting volume lock flag failed");
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+static int
+cli_snapshot_status(dict_t *dict, gf_cli_rsp *rsp, call_frame_t *frame)
+{
+    int ret = -1;
+    int status_cmd = -1;
+    cli_local_t *local = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+    GF_ASSERT(frame);
+
+    local = ((call_frame_t *)frame)->local;
+    if (!local) {
+        gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+        goto out;
+    }
+
+    if (rsp->op_ret) {
+        if (rsp->op_errstr) {
+            ret = dict_set_dynstr_with_alloc(local->dict, "op_err_str",
+                                             rsp->op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set op_errstr in local dictionary");
+                goto out;
+            }
+        }
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((status_cmd != GF_SNAP_STATUS_TYPE_SNAP) &&
+        (status_cmd != GF_SNAP_STATUS_TYPE_ITER)) {
+        dict_copy(dict, local->dict);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_status_single_snap(local, dict, "status.snap0");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot status");
+        }
+    } else {
+        ret = cli_get_single_snap_status(dict, "status.snap0");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch status of snap");
+        }
+    }
+
+out:
+    return ret;
+}
+
+static int
+gf_cli_generate_snapshot_event(gf_cli_rsp *rsp, dict_t *dict, int32_t type,
+                               char *snap_name, char *volname, char *snap_uuid,
+                               char *clone_name)
+{
+    int ret = -1;
+    int config_command = 0;
+    int32_t delete_cmd = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+    char msg[PATH_MAX] = {
+        0,
+    };
+    char option[512] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+    GF_VALIDATE_OR_GOTO("cli", rsp, out);
+
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (!volname) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CREATE_FAILED,
+                         "snapshot_name=%s;volume_name=%s;error=%s", snap_name,
+                         volname,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_CREATED,
+                     "snapshot_name=%s;volume_name=%s;snapshot_uuid=%s",
+                     snap_name, volname, snap_uuid);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_ACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_ACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DEACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_DEACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_RESTORE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volname");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_RESTORED,
+                     "snapshot_name=%s;snapshot_uuid=%s;volume_name=%s",
+                     snap_name, snap_uuid, volname);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
+                goto out;
+            }
+
+            /*
+             * Need not generate any event (success or failure) for delete *
+             * all, as it will trigger individual delete for all snapshots *
+             */
+            if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL) {
+                ret = 0;
+                break;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DELETE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_DELETED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+                goto out;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
+                goto out;
+            }
+
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CLONE_FAILED,
+                         "snapshot_name=%s;clone_name=%s;error=%s", snap_name,
+                         clone_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+                goto out;
+            }
+
+            gf_event(EVENT_SNAPSHOT_CLONED,
+                     "snapshot_name=%s;clone_name=%s;clone_uuid=%s", snap_name,
+                     clone_name, snap_uuid);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CONFIG_UPDATE_FAILED, "error=%s",
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+                goto out;
+            }
+
+            if (config_command == GF_SNAP_CONFIG_DISPLAY) {
+                ret = 0;
+                break;
+            }
+
+            /* These are optional parameters therefore ignore the error */
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+            ret = dict_get_str_sizen(dict, "snap-activate-on-create",
+                                     &snap_activate);
+
+            if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR,
+                       "At least one option from "
+                       "snap-max-hard-limit, snap-max-soft-limit, "
+                       "auto-delete and snap-activate-on-create "
+                       "should be set");
+                goto out;
+            }
+
+            if (hard_limit || soft_limit) {
+                snprintf(option, sizeof(option), "%s=%" PRIu64,
+                         hard_limit ? "hard_limit" : "soft_limit",
+                         hard_limit ? hard_limit : soft_limit);
+            } else if (auto_delete || snap_activate) {
+                snprintf(option, sizeof(option), "%s=%s",
+                         auto_delete ? "auto-delete" : "snap-activate",
+                         auto_delete ? auto_delete : snap_activate);
+            }
+
+            volname = NULL;
+            ret = dict_get_str_sizen(dict, "volname", &volname);
+
+            snprintf(msg, sizeof(msg), "config_type=%s;%s",
+                     volname ? "volume_config" : "system_config", option);
+
+            gf_event(EVENT_SNAPSHOT_CONFIG_UPDATED, "%s", msg);
+
+            ret = 0;
+            break;
+
+        default:
+            gf_log("cli", GF_LOG_WARNING,
+                   "Cannot generate event for unknown type.");
+            ret = 0;
+            goto out;
+    }
+
+out:
+    return ret;
+}
+
+/*
+ * Fetch necessary data from dict at one place instead of *
+ * repeating the same code again and again.               *
+ */
+static int
+gf_cli_snapshot_get_data_from_dict(dict_t *dict, char **snap_name,
+                                   char **volname, char **snap_uuid,
+                                   int8_t *soft_limit_flag, char **clone_name)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+
+    if (snap_name) {
+        ret = dict_get_str_sizen(dict, "snapname", snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapname from dict");
+        }
+    }
+
+    if (volname) {
+        ret = dict_get_str_sizen(dict, "volname1", volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get volname1 from dict");
+        }
+    }
+
+    if (snap_uuid) {
+        ret = dict_get_str_sizen(dict, "snapuuid", snap_uuid);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapuuid from dict");
+        }
+    }
+
+    if (soft_limit_flag) {
+        ret = dict_get_int8(dict, "soft-limit-reach", soft_limit_flag);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG,
+                   "failed to get soft-limit-reach from dict");
+        }
+    }
+
+    if (clone_name) {
+        ret = dict_get_str_sizen(dict, "clonename", clone_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get clonename from dict");
+        }
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+gf_cli_snapshot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *snap_name = NULL;
+    char *clone_name = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    gf_boolean_t snap_driven = _gf_false;
+    int8_t soft_limit_flag = -1;
+    char *volname = NULL;
+    char *snap_uuid = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get type");
+        goto out;
+    }
+
+    ret = gf_cli_snapshot_get_data_from_dict(
+        dict, &snap_name, &volname, &snap_uuid, &soft_limit_flag, &clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch data from dict.");
+        goto out;
+    }
+
+#if (USE_EVENTS)
+    ret = gf_cli_generate_snapshot_event(&rsp, dict, type, snap_name, volname,
+                                         snap_uuid, clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate snapshot event");
+        goto out;
+    }
+#endif
+
+    /* Snapshot status and delete command is handled separately */
+    if (global_state->mode & GLUSTER_MODE_XML &&
+        GF_SNAP_OPTION_TYPE_STATUS != type &&
+        GF_SNAP_OPTION_TYPE_DELETE != type) {
+        ret = cli_xml_output_snapshot(type, dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        }
+
+        goto out;
+    }
+
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            if (rsp.op_ret) {
+                cli_err("snapshot create: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            if (!snap_name) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+            }
+
+            cli_out("snapshot create: success: Snap %s created successfully",
+                    snap_name);
+
+            if (soft_limit_flag == 1) {
+                cli_out(
+                    "Warning: Soft-limit of volume (%s) is "
+                    "reached. Snapshot creation is not possible "
+                    "once hard-limit is reached.",
+                    volname);
+            }
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            if (rsp.op_ret) {
+                cli_err("snapshot clone: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+                goto out;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
+                goto out;
+            }
+
+            cli_out("snapshot clone: success: Clone %s created successfully",
+                    clone_name);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            if (rsp.op_ret) {
+                cli_err("snapshot restore: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            cli_out("Snapshot restore: %s: Snap restored successfully",
+                    snap_name);
+
+            ret = 0;
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            if (rsp.op_ret) {
+                cli_err("snapshot activate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            cli_out("Snapshot activate: %s: Snap activated successfully",
+                    snap_name);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            if (rsp.op_ret) {
+                cli_err("snapshot deactivate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
+
+            cli_out("Snapshot deactivate: %s: Snap deactivated successfully",
+                    snap_name);
+
+            ret = 0;
+            break;
+        case GF_SNAP_OPTION_TYPE_INFO:
+            if (rsp.op_ret) {
+                cli_err("Snapshot info : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+            if (snap_driven == _gf_true) {
+                ret = cli_call_snapshot_info(dict, snap_driven);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
+                }
+            } else if (snap_driven == _gf_false) {
+                ret = cli_get_snaps_in_volume(dict);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
+                }
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = cli_snapshot_config_display(dict, &rsp);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot config output.");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_LIST:
+            if (rsp.op_ret) {
+                cli_err("Snapshot list : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
+
+            ret = cli_snapshot_list(dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to display snapshot list");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = cli_snapshot_remove_reply(&rsp, dict, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to delete snap");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+            ret = cli_snapshot_status(dict, &rsp, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot status output.");
+                goto out;
+            }
+            break;
+
+        default:
+            cli_err("Unknown command executed");
+            ret = -1;
+            goto out;
+    }
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+
+    free(rsp.dict.dict_val);
+    free(rsp.op_errstr);
+
+    return ret;
+}
+
+int32_t
+gf_cli_snapshot_for_delete(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int32_t ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int32_t snapcount = 0;
+    int i = 0;
+    char question[PATH_MAX] = "";
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* No need multiple RPCs for individual snapshot delete*/
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+#ifdef HAVE_LIB_XML
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapCount", "%d", snapcount);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to write xml element \"snapCount\"");
+            goto out;
+        }
+#endif /* HAVE_LIB_XML */
+    } else if (snapcount == 0) {
+        cli_out("No snapshots present");
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_DELETE_TYPE_ALL) {
+        snprintf(question, sizeof(question),
+                 "System contains %d snapshot(s).\nDo you still "
+                 "want to continue and delete them? ",
+                 snapcount);
+    } else {
+        ret = dict_get_str_sizen(local->dict, "volname", &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to fetch volname from local dictionary");
+            goto out;
+        }
+
+        snprintf(question, sizeof(question),
+                 "Volume (%s) contains %d snapshot(s).\nDo you still want to "
+                 "continue and delete them? ",
+                 volname, snapcount);
+    }
+
+    answer = cli_cmd_get_confirmation(global_state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled snapshot delete operation for snap delete");
+        goto out;
+    }
+
+    for (i = 1; i <= snapcount; i++) {
+        ret = -1;
+
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
+
+        ret = cli_populate_req_dict_for_delete(snap_dict, local->dict, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
+        }
+
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
+        if (ret) {
+            /* Fail the operation if deleting one of the
+             * snapshots is failed
+             */
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot delete failed");
+            goto out;
+        }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
+
+out:
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_snapshot_for_status(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int snapcount = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* Snapshot status of single snap (i.e. GF_SNAP_STATUS_TYPE_SNAP)
+     * is already handled. Therefore we can return from here.
+     * If want to get status of all snaps in the system or volume then
+     * we should get them one by one.*/
+    if ((cmd == GF_SNAP_STATUS_TYPE_SNAP) ||
+        (cmd == GF_SNAP_STATUS_TYPE_ITER)) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "status.snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
+
+    if (snapcount == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_out("No snapshots present");
+    }
+
+    for (i = 0; i < snapcount; i++) {
+        ret = -1;
+
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
+
+        ret = cli_populate_req_dict_for_status(snap_dict, local->dict, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
+        }
+
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
+
+        /* Ignore the return value and error for snapshot
+         * status of type "ALL" or "VOL"
+         *
+         * Scenario : There might be case where status command
+         * and delete command might be issued at the same time.
+         * In that case when status tried to fetch detail of
+         * snap which has been deleted by concurrent command,
+         * then it will show snapshot not present. Which will
+         * not be appropriate.
+         */
+        if (ret && (cmd != GF_SNAP_STATUS_TYPE_ALL &&
+                    cmd != GF_SNAP_STATUS_TYPE_VOL)) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot status failed");
+            goto out;
+        }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
+
+    ret = 0;
+out:
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+    int tmp_ret = -1;
+    cli_local_t *local = NULL;
+    char *err_str = NULL;
+    int type = -1;
+
+    if (!frame || !frame->local || !this || !data)
+        goto out;
+
+    local = frame->local;
+
+    options = data;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_begin_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to begin snapshot xml composite op");
+            goto out;
+        }
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options, GLUSTER_CLI_SNAP,
+                          this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for snapshot failed");
+        goto xmlend;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "type", &type);
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type) {
+        ret = gf_cli_snapshot_for_status(frame, this, data);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot status command failed");
+        }
+
+        goto xmlend;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_DELETE == type) {
+        ret = gf_cli_snapshot_for_delete(frame, this, data);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot delete command failed");
+        }
+
+        goto xmlend;
+    }
+
+    ret = 0;
+
+xmlend:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_end_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to end snapshot xml composite op");
+            goto out;
+        }
+    }
+out:
+    if (ret && local && GF_SNAP_OPTION_TYPE_STATUS == type) {
+        tmp_ret = dict_get_str_sizen(local->dict, "op_err_str", &err_str);
+        if (tmp_ret || !err_str) {
+            cli_err("Snapshot Status : failed: %s",
+                    "Please check log file for details");
+        } else {
+            cli_err("Snapshot Status : failed: %s", err_str);
+            dict_del_sizen(local->dict, "op_err_str");
+        }
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* XML mode handles its own error */
+        ret = 0;
+    }
+    return ret;
+}
+
+static int32_t
+gf_cli_barrier_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to barrier");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr && (strlen(rsp.op_errstr) > 1)) {
+            cli_err("volume barrier: command unsuccessful : %s", rsp.op_errstr);
+        } else {
+            cli_err("volume barrier: command unsuccessful");
+        }
+    } else {
+        cli_out("volume barrier: command successful");
+    }
+    ret = rsp.op_ret;
+
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_barrier_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_barrier_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BARRIER_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int32_t
+gf_cli_get_vol_opt_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int i = 0;
+    char dict_key[50] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to get volume option");
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume get option: failed: %s",
+                     rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume get option: failed");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_str("volGetopts", msg, rsp.op_ret,
+                                     rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        } else {
+            cli_err("%s", msg);
+        }
+        ret = rsp.op_ret;
+        goto out_nolog;
+    }
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_getopts(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            ret = 0;
+        }
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "warning", &value);
+    if (!ret) {
+        cli_out("%s", value);
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count from the dictionary");
+        goto out;
+    }
+
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR, "Value of count :%d is invalid", count);
+        ret = -1;
+        goto out;
+    }
+
+    cli_out("%-40s%-40s", "Option", "Value");
+    cli_out("%-40s%-40s", "------", "-----");
+    for (i = 1; i <= count; i++) {
+        ret = snprintf(dict_key, sizeof dict_key, "key%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve %s from the dictionary", dict_key);
+            goto out;
+        }
+        ret = snprintf(dict_key, sizeof dict_key, "value%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve key value for %s from the dictionary",
+                   dict_key);
+            goto out;
+        }
+        cli_out("%-40s%-40s", key, value);
+    }
+
+out:
+    if (ret) {
+        cli_out(
+            "volume get option failed. Check the cli/glusterd log "
+            "file for more details");
+    }
+
+out_nolog:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
+
+static int
+gf_cli_get_vol_opt(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_vol_opt_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_GET_VOL_OPT, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
+
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict)
+{
+    int ret = 0;
+
+    if (cli_default_conn_timeout > 120) {
+        ret = dict_set_uint32(dict, "timeout", cli_default_conn_timeout);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Failed to save timeout to dict");
+        }
+    }
+    return ret;
+}
+
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref)
+{
+    int ret = 0;
+    size_t len = 0;
+    char *cmd = NULL;
+    int i = 0;
+    const char **words = NULL;
+    cli_local_t *local = NULL;
+
+    if (!this || !frame || !frame->local || !dict) {
+        ret = -1;
+        goto out;
+    }
+
+    local = frame->local;
+
+    if (!local->words) {
+        ret = -1;
+        goto out;
+    }
+
+    words = local->words;
+
+    while (words[i])
+        len += strlen(words[i++]) + 1;
+
+    cmd = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!cmd) {
+        ret = -1;
+        goto out;
+    }
+    cmd[0] = '\0';
+
+    for (i = 0; words[i]; i++) {
+        strncat(cmd, words[i], len - 1);
+        if (words[i + 1] != NULL)
+            strncat(cmd, " ", len - 1);
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "cmd-str", cmd);
+    if (ret)
+        goto out;
+
+    ret = add_cli_cmd_timeout_to_dict(dict);
+
+    ret = dict_allocate_and_serialize(dict, &(req->dict).dict_val,
+                                      &(req->dict).dict_len);
+
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "failed to get serialized length of dict");
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, req, frame, prog, procnum, iobref, this, cbkfn,
+                         (xdrproc_t)xdrproc);
+out:
+    return ret;
+}
+
+static int
+gf_cli_print_bitrot_scrub_status(dict_t *dict)
+{
+    int i = 1;
+    int j = 0;
+    int ret = -1;
+    int count = 0;
+    char key[64] = {
+        0,
+    };
+    char *volname = NULL;
+    char *node_name = NULL;
+    char *scrub_freq = NULL;
+    char *state_scrub = NULL;
+    char *scrub_impact = NULL;
+    char *bad_file_str = NULL;
+    char *scrub_log_file = NULL;
+    char *bitrot_log_file = NULL;
+    uint64_t scrub_files = 0;
+    uint64_t unsigned_files = 0;
+    uint64_t scrub_time = 0;
+    uint64_t days = 0;
+    uint64_t hours = 0;
+    uint64_t minutes = 0;
+    uint64_t seconds = 0;
+    char *last_scrub = NULL;
+    uint64_t error_count = 0;
+    int8_t scrub_running = 0;
+    char *scrub_state_op = NULL;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "failed to get count value from dictionary");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "features.scrub", &state_scrub);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub state value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-throttle", &scrub_impact);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub impact value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-freq", &scrub_freq);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub -freq value");
+
+    ret = dict_get_str_sizen(dict, "bitrot_log_file", &bitrot_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get bitrot log file location");
+
+    ret = dict_get_str_sizen(dict, "scrub_log_file", &scrub_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrubber log file location");
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "scrub-running-%d", i);
+        ret = dict_get_int8(dict, key, &scrub_running);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
+        if (scrub_running)
+            break;
+    }
+
+    if (scrub_running)
+        gf_asprintf(&scrub_state_op, "%s (In Progress)", state_scrub);
+    else
+        gf_asprintf(&scrub_state_op, "%s (Idle)", state_scrub);
+
+    cli_out("\n%s: %s\n", "Volume name ", volname);
+
+    cli_out("%s: %s\n", "State of scrub", scrub_state_op);
+
+    cli_out("%s: %s\n", "Scrub impact", scrub_impact);
+
+    cli_out("%s: %s\n", "Scrub frequency", scrub_freq);
+
+    cli_out("%s: %s\n", "Bitrot error log location", bitrot_log_file);
+
+    cli_out("%s: %s\n", "Scrubber error log location", scrub_log_file);
+
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        last_scrub = NULL;
+        scrub_time = 0;
+        error_count = 0;
+        scrub_files = 0;
+        unsigned_files = 0;
+
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
+
+        snprintf(key, sizeof(key), "scrubbed-files-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
+
+        snprintf(key, sizeof(key), "unsigned-files-%d", i);
+        ret = dict_get_uint64(dict, key, &unsigned_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get unsigned files");
+
+        snprintf(key, sizeof(key), "scrub-duration-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_time);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub duration");
+
+        snprintf(key, sizeof(key), "last-scrub-time-%d", i);
+        ret = dict_get_str(dict, key, &last_scrub);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub time");
+        snprintf(key, sizeof(key), "error-count-%d", i);
+        ret = dict_get_uint64(dict, key, &error_count);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get error count");
+
+        cli_out("\n%s\n",
+                "=========================================================");
+
+        cli_out("%s: %s\n", "Node", node_name);
+
+        cli_out("%s: %" PRIu64 "\n", "Number of Scrubbed files", scrub_files);
+
+        cli_out("%s: %" PRIu64 "\n", "Number of Skipped files", unsigned_files);
+
+        if ((!last_scrub) || !strcmp(last_scrub, ""))
+            cli_out("%s: %s\n", "Last completed scrub time",
+                    "Scrubber pending to complete.");
+        else
+            cli_out("%s: %s\n", "Last completed scrub time", last_scrub);
+
+        /* Printing last scrub duration time in human readable form*/
+        seconds = scrub_time % 60;
+        minutes = (scrub_time / 60) % 60;
+        hours = (scrub_time / 3600) % 24;
+        days = scrub_time / 86400;
+        cli_out("%s: %" PRIu64 ":%" PRIu64 ":%" PRIu64 ":%" PRIu64 "\n",
+                "Duration of last scrub (D:M:H:M:S)", days, hours, minutes,
+                seconds);
+
+        cli_out("%s: %" PRIu64 "\n", "Error count", error_count);
+
+        if (error_count) {
+            cli_out("%s:\n", "Corrupted object's [GFID]");
+            /* Printing list of bad file's (Corrupted object's)*/
+            for (j = 0; j < error_count; j++) {
+                snprintf(key, sizeof(key), "quarantine-%d-%d", j, i);
+                ret = dict_get_str(dict, key, &bad_file_str);
+                if (!ret) {
+                    cli_out("%s\n", bad_file_str);
+                }
+            }
+        }
+    }
+    cli_out("%s\n",
+            "=========================================================");
+
+out:
+    GF_FREE(scrub_state_op);
+    return 0;
+}
+
+static int
+gf_cli_bitrot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int type = 0;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *scrub_cmd = NULL;
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    char *cmd_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Bitrot command failed : %s", rsp.op_errstr);
+        else
+            cli_err("Bitrot command : failed");
+
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to bit rot command");
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command type");
+        goto out;
+    }
+
+    if ((type == GF_BITROT_CMD_SCRUB_STATUS) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = gf_cli_print_bitrot_scrub_status(dict);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to print bitrot scrub status");
+        }
+        goto out;
+    }
+
+    /* Ignoring the error, as using dict val for cli output only */
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "scrub-value", &scrub_cmd);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "Failed to get scrub command");
+
+    ret = dict_get_str_sizen(dict, "cmd-str", &cmd_str);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get command string");
+
+    if (cmd_str)
+        cmd_op = strrchr(cmd_str, ' ') + 1;
+
+    switch (type) {
+        case GF_BITROT_OPTION_TYPE_ENABLE:
+            cli_out("volume bitrot: success bitrot enabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_DISABLE:
+            cli_out("volume bitrot: success bitrot disabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_CMD_SCRUB_ONDEMAND:
+            cli_out("volume bitrot: scrubber started ondemand for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB:
+            if (!strncmp("pause", scrub_cmd, sizeof("pause")))
+                cli_out("volume bitrot: scrubber paused for volume %s",
+                        volname);
+            if (!strncmp("resume", scrub_cmd, sizeof("resume")))
+                cli_out("volume bitrot: scrubber resumed for volume %s",
+                        volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+            cli_out(
+                "volume bitrot: scrub-frequency is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+            cli_out(
+                "volume bitrot: scrub-throttle is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+    }
+
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (!rsp.op_ret)
+        cli_out("volume bitrot: success");
+
+    ret = rsp.op_ret;
+
+out:
+    if (dict)
+        dict_unref(dict);
+
+    gf_free_xdr_cli_rsp(rsp);
+    cli_cmd_broadcast_response(ret);
+    return ret;
+}
+
+static int32_t
+gf_cli_bitrot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_bitrot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BITROT, this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for bitrot failed");
+        goto out;
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_NULL] = {"NULL", NULL},
+    [GLUSTER_CLI_PROBE] = {"PROBE_QUERY", gf_cli_probe},
+    [GLUSTER_CLI_DEPROBE] = {"DEPROBE_QUERY", gf_cli_deprobe},
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS", gf_cli_list_friends},
+    [GLUSTER_CLI_UUID_RESET] = {"UUID_RESET", gf_cli3_1_uuid_reset},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", gf_cli3_1_uuid_get},
+    [GLUSTER_CLI_CREATE_VOLUME] = {"CREATE_VOLUME", gf_cli_create_volume},
+    [GLUSTER_CLI_DELETE_VOLUME] = {"DELETE_VOLUME", gf_cli_delete_volume},
+    [GLUSTER_CLI_START_VOLUME] = {"START_VOLUME", gf_cli_start_volume},
+    [GLUSTER_CLI_STOP_VOLUME] = {"STOP_VOLUME", gf_cli_stop_volume},
+    [GLUSTER_CLI_RENAME_VOLUME] = {"RENAME_VOLUME", gf_cli_rename_volume},
+    [GLUSTER_CLI_DEFRAG_VOLUME] = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", gf_cli_get_volume},
+    [GLUSTER_CLI_GET_NEXT_VOLUME] = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
+    [GLUSTER_CLI_SET_VOLUME] = {"SET_VOLUME", gf_cli_set_volume},
+    [GLUSTER_CLI_ADD_BRICK] = {"ADD_BRICK", gf_cli_add_brick},
+    [GLUSTER_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", gf_cli_remove_brick},
+    [GLUSTER_CLI_REPLACE_BRICK] = {"REPLACE_BRICK", gf_cli_replace_brick},
+    [GLUSTER_CLI_LOG_ROTATE] = {"LOG ROTATE", gf_cli_log_rotate},
+    [GLUSTER_CLI_GETSPEC] = {"GETSPEC", gf_cli_getspec},
+    [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
+    [GLUSTER_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", gf_cli_sync_volume},
+    [GLUSTER_CLI_RESET_VOLUME] = {"RESET_VOLUME", gf_cli_reset_volume},
+    [GLUSTER_CLI_FSM_LOG] = {"FSM_LOG", gf_cli_fsm_log},
+    [GLUSTER_CLI_GSYNC_SET] = {"GSYNC_SET", gf_cli_gsync_set},
+    [GLUSTER_CLI_PROFILE_VOLUME] = {"PROFILE_VOLUME", gf_cli_profile_volume},
+    [GLUSTER_CLI_QUOTA] = {"QUOTA", gf_cli_quota},
+    [GLUSTER_CLI_TOP_VOLUME] = {"TOP_VOLUME", gf_cli_top_volume},
+    [GLUSTER_CLI_GETWD] = {"GETWD", gf_cli_getwd},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME", gf_cli_status_volume},
+    [GLUSTER_CLI_STATUS_ALL] = {"STATUS_ALL", gf_cli_status_volume_all},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", gf_cli_mount},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", gf_cli_umount},
+    [GLUSTER_CLI_HEAL_VOLUME] = {"HEAL_VOLUME", gf_cli_heal_volume},
+    [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME",
+                                      gf_cli_statedump_volume},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", gf_cli_list_volume},
+    [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME",
+                                     gf_cli_clearlocks_volume},
+    [GLUSTER_CLI_COPY_FILE] = {"COPY_FILE", gf_cli_copy_file},
+    [GLUSTER_CLI_SYS_EXEC] = {"SYS_EXEC", gf_cli_sys_exec},
+    [GLUSTER_CLI_SNAP] = {"SNAP", gf_cli_snapshot},
+    [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER VOLUME", gf_cli_barrier_volume},
+    [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", gf_cli_get_vol_opt},
+    [GLUSTER_CLI_BITROT] = {"BITROT", gf_cli_bitrot},
+    [GLUSTER_CLI_GET_STATE] = {"GET_STATE", gf_cli_get_state},
+    [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", gf_cli_reset_brick},
+    [GLUSTER_CLI_GANESHA] = {"GANESHA", gf_cli_ganesha},
+};
+
+struct rpc_clnt_program cli_prog = {
+    .progname = "Gluster CLI",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numproc = GLUSTER_CLI_MAXVALUE,
+    .proctable = gluster_cli_actors,
+};
+
+static struct rpc_clnt_procedure cli_quotad_procs[GF_AGGREGATOR_MAXVALUE] = {
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL},
+    [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", cli_quotad_getlimit},
+};
+
+struct rpc_clnt_program cli_quotad_clnt = {
+    .progname = "CLI Quotad client",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numproc = GF_AGGREGATOR_MAXVALUE,
+    .proctable = cli_quotad_procs,
+};
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
new file mode 100644
index 00000000000..069de75801c
--- /dev/null
+++ b/cli/src/cli-xml-output.c
@@ -0,0 +1,5839 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdlib.h>
+#include "cli.h"
+#include "cli1-xdr.h"
+#include <glusterfs/run.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/upcall-utils.h>
+
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
+
+/*
+ * IMPORTANT NOTE:
+ * All exported functions in this file which use libxml need use a
+ * #if (HAVE_LIB_XML), #else, #endif
+ * For eg,
+ *      int exported_func () {
+ *              #if (HAVE_LIB_XML)
+ *                      <Stuff using libxml>
+ *              #else
+ *                      return 0;
+ *              #endif
+ *      }
+ *
+ *  All other functions, which are called internally within this file need to be
+ *  within #if (HAVE_LIB_XML), #endif statements
+ *  For eg,
+ *      #if (HAVE_LIB_XML)
+ *      int internal_func ()
+ *      {
+ *      }
+ *      #endif
+ *
+ *  Following the above format ensures that all xml related code is compiled
+ *  only when libxml2 is present, and also keeps the rest of the codebase free
+ *  of #if (HAVE_LIB_XML)
+ */
+
+#if (HAVE_LIB_XML)
+
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+
+#define XML_RET_CHECK_AND_GOTO(ret, label)                                     \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        } else                                                                 \
+            ret = 0;                                                           \
+    } while (0)
+
+int
+cli_begin_xml_output(xmlTextWriterPtr *writer, xmlDocPtr *doc)
+{
+    int ret = -1;
+
+    *writer = xmlNewTextWriterDoc(doc, 0);
+    if (*writer == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <cliOutput> */
+    ret = xmlTextWriterStartElement(*writer, (xmlChar *)"cliOutput");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_end_xml_output(xmlTextWriterPtr writer, xmlDocPtr doc)
+{
+    int ret = -1;
+
+    /* </cliOutput> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndDocument(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Dump xml document to stdout and pretty format it */
+    xmlSaveFormatFileEnc("-", doc, "UTF-8", 1);
+
+    xmlFreeTextWriter(writer);
+    xmlFreeDoc(doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_common(xmlTextWriterPtr writer, int op_ret, int op_errno,
+                      char *op_errstr)
+{
+    int ret = -1;
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opRet", "%d",
+                                          op_ret);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrno", "%d",
+                                          op_errno);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (op_errstr)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", op_errstr);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", "");
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    if (op) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"cliOp", "%s",
+                                              op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (str) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"output", "%s",
+                                              str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_data_pair(dict_t *this, char *key, data_t *value, void *data)
+{
+    int ret = -1;
+    xmlTextWriterPtr *writer = NULL;
+
+    writer = (xmlTextWriterPtr *)data;
+
+    ret = xmlTextWriterWriteFormatElement(*writer, (xmlChar *)key, "%s",
+                                          value->data);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <"op"> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (dict)
+        dict_foreach(dict, cli_xml_output_data_pair, &writer);
+
+    /* </"op"> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_status_common(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int *online,
+                                 gf_boolean_t *node_present)
+{
+    int ret = -1;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *uuid = NULL;
+    int port = 0;
+    int rdma_port = 0;
+    int status = 0;
+    int pid = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.hostname", brick_index);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret) {
+        *node_present = _gf_false;
+        goto out;
+    }
+    *node_present = _gf_true;
+
+    /* <node>
+     * will be closed in the calling function cli_xml_output_vol_status()*/
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname", "%s",
+                                          hostname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.path", brick_index);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                          path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.peerid", brick_index);
+    ret = dict_get_str(dict, key, &uuid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"peerid", "%s",
+                                          uuid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.status", brick_index);
+    ret = dict_get_int32(dict, key, &status);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    *online = status;
+
+    snprintf(key, sizeof(key), "brick%d.port", brick_index);
+    ret = dict_get_int32(dict, key, &port);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "brick%d.rdma_port", brick_index);
+    ret = dict_get_int32(dict, key, &rdma_port);
+
+    /* If the process is either offline or doesn't provide a port (shd)
+     * port = "N/A"
+     * else print the port number of the process.
+     */
+
+    /*
+     * Tag 'port' can be removed once console management is started
+     * to support new tag ports.
+     */
+
+    if (*online == 1 && port != 0)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%d",
+                                              port);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%s",
+                                              "N/A");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"ports");
+    if (*online == 1 && (port != 0 || rdma_port != 0)) {
+        if (port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%d", port);
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%s", "N/A");
+        }
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        if (rdma_port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%d", rdma_port);
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%s", "N/A");
+        }
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.pid", brick_index);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_detail(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index)
+{
+    int ret = -1;
+    uint64_t size_total = 0;
+    uint64_t size_free = 0;
+    char *device = NULL;
+    uint64_t block_size = 0;
+    char *mnt_options = NULL;
+    char *fs_name = NULL;
+    char *inode_size = NULL;
+    uint64_t inodes_total = 0;
+    uint64_t inodes_free = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.total", brick_index);
+    ret = dict_get_uint64(dict, key, &size_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeTotal",
+                                              "%" PRIu64, size_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free", brick_index);
+    ret = dict_get_uint64(dict, key, &size_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeFree",
+                                              "%" PRIu64, size_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.device", brick_index);
+    ret = dict_get_str(dict, key, &device);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"device", "%s",
+                                              device);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.block_size", brick_index);
+    ret = dict_get_uint64(dict, key, &block_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"blockSize",
+                                              "%" PRIu64, block_size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", brick_index);
+    ret = dict_get_str(dict, key, &mnt_options);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"mntOptions",
+                                              "%s", mnt_options);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.fs_name", brick_index);
+    ret = dict_get_str(dict, key, &fs_name);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsName", "%s",
+                                              fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.inode_size", brick_index);
+    ret = dict_get_str(dict, key, &inode_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodeSize",
+                                              "%s", fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.total_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesTotal",
+                                              "%" PRIu64, inodes_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesFree",
+                                              "%" PRIu64, inodes_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = 0;
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_mempool(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
+{
+    int ret = -1;
+    int mempool_count = 0;
+    char *name = NULL;
+    int hotcount = 0;
+    int coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int maxalloc = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <mempool> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mempool");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32(dict, key, &mempool_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          mempool_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < mempool_count; i++) {
+        /* <pool> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pool");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_str(dict, key, &name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32(dict, key, &hotcount);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hotCount",
+                                              "%d", hotcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32(dict, key, &coldcount);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"coldCount",
+                                              "%d", coldcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"padddedSizeOf", "%" PRIu64, paddedsizeof);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"allocCount",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"poolMisses",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxStdAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </pool> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </mempool> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_mem(xmlTextWriterPtr writer, dict_t *dict,
+                              int brick_index)
+{
+    int ret = -1;
+    int arena = 0;
+    int ordblks = 0;
+    int smblks = 0;
+    int hblks = 0;
+    int hblkhd = 0;
+    int usmblks = 0;
+    int fsmblks = 0;
+    int uordblks = 0;
+    int fordblks = 0;
+    int keepcost = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <memStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"memStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <mallinfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mallinfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.arena", brick_index);
+    ret = dict_get_int32(dict, key, &arena);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"arena", "%d",
+                                          arena);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", brick_index);
+    ret = dict_get_int32(dict, key, &ordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ordblks", "%d",
+                                          ordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", brick_index);
+    ret = dict_get_int32(dict, key, &smblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"smblks", "%d",
+                                          smblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", brick_index);
+    ret = dict_get_int32(dict, key, &hblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblks", "%d",
+                                          hblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", brick_index);
+    ret = dict_get_int32(dict, key, &hblkhd);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblkhd", "%d",
+                                          hblkhd);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", brick_index);
+    ret = dict_get_int32(dict, key, &usmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"usmblks", "%d",
+                                          usmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", brick_index);
+    ret = dict_get_int32(dict, key, &fsmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsmblks", "%d",
+                                          fsmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", brick_index);
+    ret = dict_get_int32(dict, key, &uordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uordblks", "%d",
+                                          uordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", brick_index);
+    ret = dict_get_int32(dict, key, &fordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fordblks", "%d",
+                                          fordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", brick_index);
+    ret = dict_get_int32(dict, key, &keepcost);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"keepcost", "%d",
+                                          keepcost);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </mallinfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d", brick_index);
+    ret = cli_xml_output_vol_status_mempool(writer, dict, key);
+    if (ret)
+        goto out;
+
+    /* </memStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_clients(xmlTextWriterPtr writer, dict_t *dict,
+                                  int brick_index)
+{
+    int ret = -1;
+    int client_count = 0;
+    char *hostname = NULL;
+    uint64_t bytes_read = 0;
+    uint64_t bytes_write = 0;
+    uint32_t opversion = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <clientsStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"clientsStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.clientcount", brick_index);
+    ret = dict_get_int32(dict, key, &client_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"clientCount",
+                                          "%d", client_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < client_count; i++) {
+        /* <client> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"client");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.hostname", brick_index, i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.bytesread", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_read);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesRead",
+                                              "%" PRIu64, bytes_read);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_write);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesWrite",
+                                              "%" PRIu64, bytes_write);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.opversion", brick_index,
+                 i);
+        ret = dict_get_uint32(dict, key, &opversion);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opVersion",
+                                              "%" PRIu32, opversion);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </client> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </clientsStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_inode_entry(xmlTextWriterPtr writer, dict_t *dict,
+                                      char *prefix)
+{
+    int ret = -1;
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <inode> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inode");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gfid", "%s",
+                                          gfid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nLookup",
+                                          "%" PRIu64, nlookup);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ref", "%" PRIu32,
+                                          ref);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"iaType", "%d",
+                                          ia_type);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </inode> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_itable(xmlTextWriterPtr writer, dict_t *dict,
+                                 char *prefix)
+{
+    int ret = -1;
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activeSize",
+                                          "%" PRIu32, active_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (active_size != 0) {
+        /* <active> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"active");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (i = 0; i < active_size; i++) {
+            snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
+                goto out;
+        }
+        /* </active> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lruSize",
+                                          "%" PRIu32, lru_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (lru_size != 0) {
+        /* <lru> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"lru");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (i = 0; i < lru_size; i++) {
+            snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
+                goto out;
+        }
+        /* </lru> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"purgeSize",
+                                          "%" PRIu32, purge_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (purge_size != 0) {
+        /* <purge> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"purge");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (i = 0; i < purge_size; i++) {
+            snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
+                goto out;
+        }
+        /* </purge> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_inode(xmlTextWriterPtr writer, dict_t *dict,
+                                int brick_index)
+{
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <inodeStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inodeStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.conn%d.itable", brick_index, i);
+        ret = cli_xml_output_vol_status_itable(writer, dict, key);
+        if (ret)
+            goto out;
+
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </inodeStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_fdtable(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
+{
+    int ret = -1;
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdTable> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdTable");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          refcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxFds",
+                                          "%" PRIu32, maxfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"firstFree", "%d",
+                                          firstfree);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"openFds", "%d",
+                                          openfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
+        if (ret)
+            continue;
+
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
+        if (ret)
+            continue;
+
+        /* <fd> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fd");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"entry", "%d",
+                                              i + 1);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d",
+                                              fd_pid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount",
+                                              "%d", fd_refcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"flags", "%d",
+                                              fd_flags);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fd> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fdTable> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_fd(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index)
+{
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", brick_index, i);
+        ret = cli_xml_output_vol_status_fdtable(writer, dict, key);
+        if (ret)
+            goto out;
+
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fdStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_callframe(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
+{
+    int ret = -1;
+    int ref_count = 0;
+    char *translator = NULL;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <callFrame> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callFrame");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          ref_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"translator", "%s",
+                                          translator);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"complete", "%d",
+                                          complete);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"parent", "%s",
+                                              parent);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windFrom",
+                                              "%s", wind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windTo", "%s",
+                                              wind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindFrom",
+                                              "%s", unwind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindTo",
+                                              "%s", unwind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </callFrame> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_callstack(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
+{
+    int ret = -1;
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    int frame_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callStack> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callStack");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uid", "%d", uid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gid", "%d", gid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unique",
+                                          "%" PRIu64, unique);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &frame_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"frameCount", "%d",
+                                          frame_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < frame_count; i++) {
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        ret = cli_xml_output_vol_status_callframe(writer, dict, key);
+        if (ret)
+            goto out;
+    }
+
+    /* </callStack> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_callpool(xmlTextWriterPtr writer, dict_t *dict,
+                                   int brick_index)
+{
+    int ret = -1;
+    int call_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callpoolStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callpoolStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.callpool.count", brick_index);
+    ret = dict_get_int32(dict, key, &call_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          call_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < call_count; i++) {
+        snprintf(key, sizeof(key), "brick%d.callpool.stack%d", brick_index, i);
+        ret = cli_xml_output_vol_status_callstack(writer, dict, key);
+        if (ret)
+            goto out;
+    }
+
+    /* </callpoolStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_status_end(cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(local->writer, local->doc);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_remove_brick_task_params(xmlTextWriterPtr writer, dict_t *dict,
+                                        char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int count = 0;
+    int i = 0;
+    char *brick = NULL;
+
+    /* <params> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"params");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        goto out;
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "%s.brick%d", prefix, i);
+        ret = dict_get_str(dict, key, &brick);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brick", "%s",
+                                              brick);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        brick = NULL;
+    }
+
+    /* </param> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_tasks(cli_local_t *local, dict_t *dict)
+{
+    int ret = -1;
+    char *task_type = NULL;
+    char *task_id_str = NULL;
+    int status = 0;
+    int tasks = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <tasks> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"tasks");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "tasks", &tasks);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < tasks; i++) {
+        /* <task> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"task");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &task_type);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%s", task_type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_task_status_str[status]);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d", i);
+        if (!strcmp(task_type, "Remove brick")) {
+            ret = cli_xml_output_remove_brick_task_params(local->writer, dict,
+                                                          key);
+            if (ret)
+                goto out;
+        }
+
+        /* </task> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </tasks> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+#endif
+
+int
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    char *volname = NULL;
+
+    /*<volume>*/
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_xml_output_vol_status_tasks(local, dict);
+    if (ret)
+        goto out;
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    char *volname = NULL;
+    int brick_count = 0;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    int online = 0;
+    gf_boolean_t node_present = _gf_true;
+    int i;
+    int type = -1;
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"nodeCount",
+                                          "%d", brick_count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret)
+        goto out;
+
+    for (i = 0; i <= index_max; i++) {
+        ret = cli_xml_output_vol_status_common(local->writer, dict, i, &online,
+                                               &node_present);
+        if (ret) {
+            if (node_present)
+                goto out;
+            else
+                continue;
+        }
+
+        switch (cmd & GF_CLI_STATUS_MASK) {
+            case GF_CLI_STATUS_DETAIL:
+                ret = cli_xml_output_vol_status_detail(local->writer, dict, i);
+                if (ret)
+                    goto out;
+                break;
+
+            case GF_CLI_STATUS_MEM:
+                if (online) {
+                    ret = cli_xml_output_vol_status_mem(local->writer, dict, i);
+                    if (ret)
+                        goto out;
+                }
+                break;
+
+            case GF_CLI_STATUS_CLIENTS:
+                if (online) {
+                    ret = cli_xml_output_vol_status_clients(local->writer, dict,
+                                                            i);
+                    if (ret)
+                        goto out;
+                }
+                break;
+
+            case GF_CLI_STATUS_INODE:
+                if (online) {
+                    ret = cli_xml_output_vol_status_inode(local->writer, dict,
+                                                          i);
+                    if (ret)
+                        goto out;
+                }
+                break;
+
+            case GF_CLI_STATUS_FD:
+                if (online) {
+                    ret = cli_xml_output_vol_status_fd(local->writer, dict, i);
+                    if (ret)
+                        goto out;
+                }
+                break;
+
+            case GF_CLI_STATUS_CALLPOOL:
+                if (online) {
+                    ret = cli_xml_output_vol_status_callpool(local->writer,
+                                                             dict, i);
+                    if (ret)
+                        goto out;
+                }
+                break;
+            default:
+                break;
+        }
+
+        /* </node>  was opened in cli_xml_output_vol_status_common()*/
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* Tasks are only present when a normal volume status call is done on a
+     * single volume or on all volumes
+     */
+    if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+        (cmd & (GF_CLI_STATUS_VOL | GF_CLI_STATUS_ALL))) {
+        ret = cli_xml_output_vol_status_tasks(local, dict);
+        if (ret)
+            goto out;
+    }
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_top_rw_perf(xmlTextWriterPtr writer, dict_t *dict,
+                               int brick_index, int member_index)
+{
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t throughput = 0;
+    struct timeval tv = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &throughput);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, throughput);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-time-sec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_sec);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%d-time-usec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_usec);
+    if (ret)
+        goto out;
+
+    gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"time", "%s",
+                                          timestr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_top_other(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index, int member_index)
+{
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t count = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int brick_count = 0;
+    int top_op = GF_CLI_TOP_NONE;
+    char *brick_name = NULL;
+    int members = 0;
+    uint64_t current_open = 0;
+    uint64_t max_open = 0;
+    char *max_open_time = NULL;
+    double throughput = 0.0;
+    double time_taken = 0.0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volTop> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volTop");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "1-top-op", &top_op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"topOp", "%d",
+                                          top_op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32(dict, key, &members);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"members",
+                                              "%d", members);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &current_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"currentOpen", "%" PRIu64, current_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpen", "%" PRIu64, max_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+                snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_str(dict, key, &max_open_time);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpenTime", "%s", max_open_time);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
+
+                break;
+
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time_taken);
+                }
+
+                if (!ret) {
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"throughput", "%f", throughput);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
+
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"timeTaken", "%f", time_taken);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
+                }
+
+                break;
+
+            default:
+                ret = -1;
+                goto out;
+        }
+
+        for (j = 1; j <= members; j++) {
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                ret = cli_xml_output_vol_top_rw_perf(writer, dict, i, j);
+            } else {
+                ret = cli_xml_output_vol_top_other(writer, dict, i, j);
+            }
+            if (ret)
+                goto out;
+        }
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volTop> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_profile_stats(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int interval)
+{
+    int ret = -1;
+    uint64_t read_count = 0;
+    uint64_t write_count = 0;
+    uint64_t hits = 0;
+    double avg_latency = 0.0;
+    double max_latency = 0.0;
+    double min_latency = 0.0;
+    uint64_t duration = 0;
+    uint64_t total_read = 0;
+    uint64_t total_write = 0;
+    char key[1024] = {0};
+    int i = 0;
+
+    /* <cumulativeStats> || <intervalStats> */
+    if (interval == -1)
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"cumulativeStats");
+    else
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"intervalStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <blockStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"blockStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < 32; i++) {
+        /* <block> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"block");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu32, (1U << i));
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, brick_index, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &read_count);
+        if (ret)
+            read_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"reads",
+                                              "%" PRIu64, read_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, brick_index,
+                 interval, (1U << i));
+        ret = dict_get_uint64(dict, key, &write_count);
+        if (ret)
+            write_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"writes",
+                                              "%" PRIu64, write_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </block> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </blockStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <fopStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fopStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", brick_index, interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
+        if (ret)
+            goto cont;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &avg_latency);
+        if (ret)
+            goto cont;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &min_latency);
+        if (ret)
+            goto cont;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &max_latency);
+        if (ret)
+            goto cont;
+
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+    cont:
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", brick_index,
+                 interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
+        if (ret)
+            continue;
+
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fopStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-duration", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &duration);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"duration",
+                                          "%" PRIu64, duration);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_read);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalRead",
+                                          "%" PRIu64, total_read);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_write);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalWrite",
+                                          "%" PRIu64, total_write);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </cumulativeStats> || </intervalStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    int op = GF_CLI_STATS_NONE;
+    int info_op = GF_CLI_INFO_NONE;
+    int brick_count = 0;
+    char *brick_name = NULL;
+    int interval = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int stats_cleared = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volProfile> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volProfile");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volname", "%s",
+                                          volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "op", &op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"profileOp", "%d",
+                                          op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (GF_CLI_STATS_INFO != op)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "info-op", &info_op);
+    if (ret)
+        goto out;
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickName",
+                                              "%s", brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32(dict, key, &stats_cleared);
+            if (ret)
+                goto out;
+
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"clearStats", "%s",
+                stats_cleared ? "Cleared stats." : "Failed to clear stats.");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        } else {
+            snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
+                if (ret)
+                    goto out;
+            }
+
+            snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
+                if (ret)
+                    goto out;
+            }
+        }
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+cont:
+    /* </volProfile> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volume", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_info_option(xmlTextWriterPtr writer, char *substr,
+                               char *optstr, char *valstr)
+{
+    int ret = -1;
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
+
+    ptr1 = substr;
+    ptr2 = optstr;
+
+    while (ptr1) {
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+        if (!*ptr1)
+            break;
+        if (!*ptr2)
+            break;
+    }
+    if (*ptr2 == '\0')
+        goto out;
+
+    /* <option> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"option");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          ptr2);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"value", "%s",
+                                          valstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </option> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+struct tmp_xml_option_logger {
+    char *key;
+    xmlTextWriterPtr writer;
+};
+
+static int
+_output_vol_info_option(dict_t *d, char *k, data_t *v, void *data)
+{
+    int ret = 0;
+    char *ptr = NULL;
+    struct tmp_xml_option_logger *tmp = NULL;
+
+    tmp = data;
+
+    ptr = strstr(k, "option.");
+    if (!ptr)
+        goto out;
+
+    if (!v) {
+        ret = -1;
+        goto out;
+    }
+    ret = cli_xml_output_vol_info_option(tmp->writer, tmp->key, k, v->data);
+
+out:
+    return ret;
+}
+
+int
+cli_xml_output_vol_info_options(xmlTextWriterPtr writer, dict_t *dict,
+                                char *prefix)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char key[1024] = {
+        0,
+    };
+    struct tmp_xml_option_logger tmp = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "%s.opt_count", prefix);
+    ret = dict_get_int32(dict, key, &opt_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"optCount", "%d",
+                                          opt_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <options> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"options");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    snprintf(key, sizeof(key), "%s.option.", prefix);
+
+    tmp.key = key;
+    tmp.writer = writer;
+    ret = dict_foreach(dict, _output_vol_info_option, &tmp);
+    if (ret)
+        goto out;
+
+    /* </options> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+    int ret = 0;
+    int count = 0;
+    char *volname = NULL;
+    char *volume_id = NULL;
+    char *uuid = NULL;
+    int type = 0;
+    int status = 0;
+    int brick_count = 0;
+    int dist_count = 0;
+    int stripe_count = 0;
+    int replica_count = 0;
+    int arbiter_count = 0;
+    int snap_count = 0;
+    int isArbiter = 0;
+    int disperse_count = 0;
+    int redundancy_count = 0;
+    int transport = 0;
+    char *brick = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 1;
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"name",
+                                              "%s", volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_str(dict, key, &volume_id);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", volume_id);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_status_str[status]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32(dict, key, &snap_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapshotCount", "%d", snap_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32(dict, key, &brick_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"brickCount", "%d", brick_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32(dict, key, &dist_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"distCount", "%d",
+                                              (brick_count / dist_count));
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32(dict, key, &stripe_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"stripeCount", "%d", stripe_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32(dict, key, &replica_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"replicaCount", "%d", replica_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32(dict, key, &arbiter_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"arbiterCount", "%d", arbiter_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32(dict, key, &disperse_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"disperseCount", "%d", disperse_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32(dict, key, &redundancy_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"redundancyCount",
+                                              "%d", redundancy_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32(dict, key, &type);
+        if (ret)
+            goto out;
+        /* For Distributed-(stripe,replicate,stipe-replicate,disperse)
+           types
+         */
+        type = get_vol_type(type, dist_count, brick_count);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%d", type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"typeStr", "%s", vol_type_str[type]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32(dict, key, &transport);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"transport", "%d", transport);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        j = 1;
+
+        /* <bricks> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"bricks");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        while (j <= brick_count) {
+            ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"brick");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d.uuid", i, j);
+            ret = dict_get_str(dict, key, &uuid);
+            if (ret)
+                goto out;
+            ret = xmlTextWriterWriteFormatAttribute(
+                local->writer, (xmlChar *)"uuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d", i, j);
+            ret = dict_get_str(dict, key, &brick);
+            if (ret)
+                goto out;
+            ret = xmlTextWriterWriteFormatString(local->writer, "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"name", "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"hostUuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter", i, j);
+            if (dict_get(dict, key))
+                isArbiter = 1;
+            else
+                isArbiter = 0;
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"isArbiter", "%d", isArbiter);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            /* </brick> */
+            ret = xmlTextWriterEndElement(local->writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            j++;
+        }
+        /* </bricks> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = cli_xml_output_vol_info_options(local->writer, dict, key);
+        if (ret)
+            goto out;
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (volname) {
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
+        local->vol_count += count;
+    }
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_ASSERT(local);
+
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volInfo> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Init vol count */
+    local->vol_count = 0;
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_info_end(cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_ASSERT(local);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"count",
+                                          "%d", local->vol_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volInfo> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(local->writer, local->doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(local->writer, local->doc);
+
+out:
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volQuota> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volQuota");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+static int
+cli_xml_output_peer_hostnames(xmlTextWriterPtr writer, dict_t *dict,
+                              const char *prefix, int count)
+{
+    int ret = -1;
+    int i = 0;
+    char *hostname = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <hostnames> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"hostnames");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        if (ret < 0)
+            goto out;
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        hostname = NULL;
+    }
+
+    /* </hostnames> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *uuid = NULL;
+    char *hostname = NULL;
+    int connected = 0;
+    int state_id = 0;
+    char *state_str = NULL;
+    int hostname_count = 0;
+    int i = 1;
+    char key[1024] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <peerStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"peerStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (!dict)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    while (i <= count) {
+        /* <peer> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"peer");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_str(dict, key, &uuid);
+        if (ret)
+            goto out;
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                              uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32(dict, key, &hostname_count);
+        if ((ret == 0) && (hostname_count > 0)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = cli_xml_output_peer_hostnames(writer, dict, key,
+                                                hostname_count);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+
+        snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32(dict, key, &connected);
+        if (ret)
+            goto out;
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connected",
+                                              "%d", connected);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "friend%d.stateId", i);
+        ret = dict_get_int32(dict, key, &state_id);
+        if (!ret) {
+            /* ignore */
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"state",
+                                                  "%d", state_id);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+
+        snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_str(dict, key, &state_str);
+        if (!ret) {
+            /* ignore */
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"stateStr",
+                                                  "%s", state_str);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+
+        /* </peer> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        i++;
+    }
+
+cont:
+    /* </peerStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+/* Used for rebalance stop/status, remove-brick status */
+int
+cli_xml_output_vol_rebalance_status(xmlTextWriterPtr writer, dict_t *dict,
+                                    enum gf_task_types task_type)
+{
+    int ret = -1;
+    int count = 0;
+    char *node_name = NULL;
+    char *node_uuid = NULL;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookups = 0;
+    int status_rcd = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    uint64_t total_files = 0;
+    uint64_t total_size = 0;
+    uint64_t total_lookups = 0;
+    uint64_t total_failures = 0;
+    uint64_t total_skipped = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int overall_status = -1;
+    double elapsed = 0;
+    double overall_elapsed = 0;
+
+    if (!dict) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeCount", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < count) {
+        i++;
+        /* Getting status early, to skip nodes that don't have the
+         * rebalance process started
+         */
+        snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32(dict, key, &status_rcd);
+
+        /* If glusterd is down it fails to get the status, try
+         getting status from other nodes */
+        if (ret)
+            continue;
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
+
+        /* <node> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeName",
+                                              "%s", node_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "node-uuid-%d", i);
+        ret = dict_get_str(dict, key, &node_uuid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              node_uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            goto out;
+        total_files += files;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                              "%" PRIu64, files);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            goto out;
+        total_size += size;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu64, size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookups);
+        if (ret)
+            goto out;
+        total_lookups += lookups;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                              "%" PRIu64, lookups);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            goto out;
+
+        snprintf(key, sizeof(key), "skipped-%d", i);
+
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            goto out;
+
+        if (task_type == GF_TASK_TYPE_REMOVE_BRICK) {
+            failures += skipped;
+            skipped = 0;
+        }
+
+        total_failures += failures;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                              "%" PRIu64, failures);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        total_skipped += skipped;
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                              "%" PRIu64, skipped);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                              status_rcd);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"statusStr", "%s",
+            cli_vol_task_status_str[status_rcd]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime",
+                                              "%.2f", elapsed);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        if (elapsed > overall_elapsed) {
+            overall_elapsed = elapsed;
+        }
+
+        /* Rebalance has 5 states,
+         * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
+         * The precedence used to determine the aggregate status is as
+         * below,
+         * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
+         */
+        /* TODO: Move this to a common place utilities that both CLI and
+         * glusterd need.
+         * Till then if the below algorithm is changed, change it in
+         * glusterd_volume_status_aggregate_tasks_status in
+         * glusterd-utils.c
+         */
+
+        if (-1 == overall_status)
+            overall_status = status_rcd;
+        int rank[] = {[GF_DEFRAG_STATUS_STARTED] = 1,
+                      [GF_DEFRAG_STATUS_FAILED] = 2,
+                      [GF_DEFRAG_STATUS_STOPPED] = 3,
+                      [GF_DEFRAG_STATUS_COMPLETE] = 4,
+                      [GF_DEFRAG_STATUS_NOT_STARTED] = 5};
+        if (rank[status_rcd] <= rank[overall_status])
+            overall_status = status_rcd;
+
+        /* </node> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* Aggregate status */
+    /* <aggregate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"aggregate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                          "%" PRIu64, total_files);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size", "%" PRIu64,
+                                          total_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                          "%" PRIu64, total_lookups);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                          "%" PRIu64, total_failures);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                          "%" PRIu64, total_skipped);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (overall_status == -1) {
+        overall_status = status_rcd;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          overall_status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        writer, (xmlChar *)"statusStr", "%s",
+        cli_vol_task_status_str[overall_status]);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime", "%.2f",
+                                          overall_elapsed);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </aggregate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volRebalance> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volRebalance");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"op", "%d", op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REBALANCE);
+        if (ret)
+            goto out;
+    }
+
+    /* </volRebalance> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (status_op) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REMOVE_BRICK);
+        if (ret)
+            goto out;
+    }
+
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    if (dict) {
+        /* <volCreate> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volCreate");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = dict_get_str(dict, "volname", &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = dict_get_str(dict, "volume-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </volCreate> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
+
+    GF_ASSERT(op);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    if (dict) {
+        /* <"op"> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = dict_get_str(dict, "volname", &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = dict_get_str(dict, "vol-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </"op"> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+_output_gsync_config(FILE *fp, xmlTextWriterPtr writer, char *op_name)
+{
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    char *ptr = NULL;
+    char *v = NULL;
+    int blen = sizeof(resbuf);
+    int ret = 0;
+
+    for (;;) {
+        ptr = fgets(resbuf, blen, fp);
+        if (!ptr)
+            break;
+
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v)) {
+            /* strip trailing space */
+            *v-- = '\0';
+        }
+        if (v == resbuf) {
+            /* skip empty line */
+            continue;
+        }
+
+        if (op_name != NULL) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)op_name,
+                                                  "%s", resbuf);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            goto out;
+        }
+
+        v = strchr(resbuf, ':');
+        if (!v) {
+            ret = -1;
+            goto out;
+        }
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v) {
+            ret = -1;
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)resbuf, "%s",
+                                              v);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+#if (HAVE_LIB_XML)
+int
+get_gsync_config(runner_t *runner,
+                 int (*op_conf)(FILE *fp, xmlTextWriterPtr writer,
+                                char *op_name),
+                 xmlTextWriterPtr writer, char *op_name)
+{
+    int ret = 0;
+
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_log("cli", GF_LOG_ERROR, "spawning child failed");
+        return -1;
+    }
+
+    ret = op_conf(runner_chio(runner, STDOUT_FILENO), writer, op_name);
+
+    ret |= runner_end(runner);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "reading data from child failed");
+
+    return ret ? -1 : 0;
+}
+#endif
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_generate_gsync_config(dict_t *dict, xmlTextWriterPtr writer)
+{
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str(dict, "subop", &subop) != 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"message", "%s",
+            GEOREP " config updated successfully");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = 0;
+        goto out;
+    }
+
+    if (dict_get_str(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str(dict, "slave", &slave) != 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_str(dict, "master", &master) != 0)
+        master = NULL;
+
+    if (dict_get_str(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
+
+    ret = dict_get_str(dict, "conf_path", &confpath);
+    if (!confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
+
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+
+    if (op_name)
+        runner_add_arg(&runner, op_name);
+
+    ret = get_gsync_config(&runner, _output_gsync_config, writer, op_name);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_gsync_status(dict_t *dict, xmlTextWriterPtr writer)
+{
+    int ret = -1;
+    int i = 1;
+    int j = 0;
+    int count = 0;
+    const int number_of_fields = 20;
+    int closed = 1;
+    int session_closed = 1;
+    gf_gsync_status_t **status_values = NULL;
+    char status_value_name[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *volume = NULL;
+    char *volume_next = NULL;
+    char *slave = NULL;
+    char *slave_next = NULL;
+    static const char *title_values[] = {
+        "master_node", "", "master_brick", "slave_user", "slave", "slave_node",
+        "status", "crawl_status",
+        /* last_synced */
+        "", "entry", "data", "meta", "failures",
+        /* checkpoint_time */
+        "", "checkpoint_completed",
+        /* checkpoint_completion_time */
+        "", "master_node_uuid",
+        /* last_synced_utc */
+        "last_synced",
+        /* checkpoint_time_utc */
+        "checkpoint_time",
+        /* checkpoint_completion_time_utc */
+        "checkpoint_completion_time"};
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_int32(dict, "gsync-count", &count);
+    if (ret)
+        goto out;
+
+    status_values = GF_MALLOC(count * sizeof(gf_gsync_status_t *),
+                              gf_common_mt_char);
+    if (!status_values) {
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < count; i++) {
+        status_values[i] = GF_CALLOC(1, sizeof(gf_gsync_status_t),
+                                     gf_common_mt_char);
+        if (!status_values[i]) {
+            ret = -1;
+            goto out;
+        }
+
+        snprintf(status_value_name, sizeof(status_value_name), "status_value%d",
+                 i);
+
+        ret = dict_get_bin(dict, status_value_name,
+                           (void **)&(status_values[i]));
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+            goto out;
+        }
+    }
+
+    qsort(status_values, count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
+
+    for (i = 0; i < count; i++) {
+        if (closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            tmp = get_struct_variable(1, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name",
+                                                  "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"sessions");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            closed = 0;
+        }
+
+        if (session_closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"session");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            session_closed = 0;
+
+            tmp = get_struct_variable(21, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"session_slave", "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pair");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (j = 0; j < number_of_fields; j++) {
+            /* XML ignore fields */
+            if (strcmp(title_values[j], "") == 0)
+                continue;
+
+            tmp = get_struct_variable(j, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)title_values[j], "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        if (i + 1 < count) {
+            slave = get_struct_variable(20, status_values[i]);
+            slave_next = get_struct_variable(20, status_values[i + 1]);
+            volume = get_struct_variable(1, status_values[i]);
+            volume_next = get_struct_variable(1, status_values[i + 1]);
+            if (!slave || !slave_next || !volume || !volume_next) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+
+            if (strcmp(volume, volume_next) != 0) {
+                closed = 1;
+                session_closed = 1;
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            } else if (strcmp(slave, slave_next) != 0) {
+                session_closed = 1;
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
+        } else {
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+    }
+out:
+    if (status_values)
+        GF_FREE(status_values);
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int type = 0;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <geoRep> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"geoRep");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+        case GF_GSYNC_OPTION_TYPE_PAUSE:
+        case GF_GSYNC_OPTION_TYPE_RESUME:
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+            if (dict_get_str(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"master",
+                                                  "%s", master);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"slave",
+                                                  "%s", slave);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            if (op_ret == 0) {
+                ret = xmlTextWriterStartElement(writer, (xmlChar *)"config");
+                XML_RET_CHECK_AND_GOTO(ret, out);
+
+                ret = cli_xml_generate_gsync_config(dict, writer);
+                if (ret)
+                    goto out;
+
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
+
+            break;
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+            ret = cli_xml_output_vol_gsync_status(dict, writer);
+            if (ret) {
+                gf_log("cli", GF_LOG_DEBUG, "Failed to get gsync status");
+                goto out;
+            }
+            break;
+        default:
+            ret = 0;
+            break;
+    }
+
+    /* </geoRep> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(writer, doc);
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+/* This function will generate snapshot create output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing create output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_create(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot clone output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing create output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_clone(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", writer, out);
+    GF_VALIDATE_OR_GOTO("cli", doc, out);
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+
+    /* <CloneCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"CloneCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "clonename", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </CloneCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot restore output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing restore output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_restore(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapRestore> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapRestore");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get vol name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume id");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapRestore> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+
+/* This function will generate snapshot list output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing list output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_list(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char *str_value = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 1; i <= snapcount; ++i) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = dict_get_str(dict, key, &str_value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapshot",
+                                                  "%s", str_value);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
+    }
+
+    /* </snapList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+
+/* This function will generate xml output for origin volume
+ * of the given snapshot.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing info output
+ * @param keyprefix     prefix for dictionary key
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_info_orig_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix)
+{
+    int ret = -1;
+    int value = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <originVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"originVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%sorigin-volname", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%ssnapcount", keyprefix);
+
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapCount", "%d",
+                                          value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%ssnaps-available", keyprefix);
+
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapRemaining",
+                                          "%d", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </originVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate xml output of snapshot volume info.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing info output
+ * @param keyprefix     key prefix for dictionary
+ * @param snap_driven   boolean to check if output is based of volume
+ *                      or snapshot
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_info_snap_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
+{
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* If the command is snap_driven then we need to show origin volume
+     * info. Else this is shown in the start of info display.*/
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.", keyprefix);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
+        }
+    }
+
+    /* </snapVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot info of individual snapshot
+ * in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing info output
+ * @param keyprefix     key prefix for dictionary
+ * @param snap_driven   boolean to check if output is based of volume
+ *                      or snapshot
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_info_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
+{
+    char key_buffer[PATH_MAX] = "";
+    char *buffer = NULL;
+    int volcount = 0;
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", buffer);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", "");
+    }
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ", keyprefix);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"createTime", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol-count", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Fail to get snap vol count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    /* Display info of each snapshot volume */
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol%d", keyprefix,
+                       i);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_snap_vol(writer, doc, dict, key_buffer,
+                                             snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list "
+                   "details of volume in a snap");
+            goto out;
+        }
+    }
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    return ret;
+}
+
+/* This function will generate snapshot info output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing info output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_info(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char key[PATH_MAX] = "";
+    gf_boolean_t snap_driven = _gf_false;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapInfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+
+    /* If the approach is volume based then we should display origin volume
+     * information first followed by per snap info*/
+    if (!snap_driven) {
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, "");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
+        }
+    }
+
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get snapshot info of individual snapshots */
+    for (i = 1; i <= snapcount; ++i) {
+        snprintf(key, sizeof(key), "snap%d", i);
+
+        ret = cli_xml_snapshot_info_per_snap(writer, doc, dict, key,
+                                             snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        }
+    }
+
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapInfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+
+/* This function will generate snapshot status of individual
+ * snapshot volume in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing status output
+ * @param keyprefix     key prefix for dictionary
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_volume_status(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, const char *keyprefix)
+{
+    int ret = -1;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brickcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get status of every brick belonging to the snapshot volume */
+    for (i = 0; i < brickcount; i++) {
+        /* <snapInfo> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Brick Path");
+            /*
+             * If path itself is not present, then end *
+             * this brick's status and continue to the *
+             * brick                                   *
+             */
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            continue;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                              buffer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Volume Group");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "%s", buffer);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "%s", buffer);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "%d", pid);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Data Percent");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "%s", buffer);
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", keyprefix, i);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get LV Size");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "N/A");
+        } else {
+            /* Truncate any newline character */
+            buffer = strtok(buffer, "\n");
+
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "%s", buffer);
+        }
+
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+out:
+    return ret;
+}
+
+/* This function will generate snapshot status of individual
+ * snapshot in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing status output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_status_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                 dict_t *dict, const char *keyprefix)
+{
+    int ret = -1;
+    int volcount = 0;
+    int i = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get snapshot status of individual snapshot volume */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
+
+        ret = cli_xml_snapshot_volume_status(writer, doc, dict, key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
+        }
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot status output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing status output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_status(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    int snapcount = 0;
+    int i = 0;
+    int status_cmd = 0;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((GF_SNAP_STATUS_TYPE_SNAP == status_cmd) ||
+        (GF_SNAP_STATUS_TYPE_ITER == status_cmd)) {
+        snapcount = 1;
+    } else {
+        ret = dict_get_int32(dict, "status.snapcount", &snapcount);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                              snapcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    for (i = 0; i < snapcount; i++) {
+        snprintf(key, sizeof(key), "status.snap%d", i);
+
+        ret = cli_xml_snapshot_status_per_snap(writer, doc, dict, key);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "failed to create xml "
+                   "output for snapshot status");
+            goto out;
+        }
+    }
+
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+
+/* This function will generate snapshot config show output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing status output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_config_show(xmlTextWriterPtr writer, xmlDocPtr doc,
+                             dict_t *dict)
+{
+    int ret = -1;
+    uint64_t i = 0;
+    uint64_t value = 0;
+    uint64_t volcount = 0;
+    char buf[PATH_MAX] = "";
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <systemConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-hard-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                          "%" PRIu64, value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-soft-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                          "%" PRIu64 "%%", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "auto-delete", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch auto-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snap-activate-on-create", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch snap-activate-on-create-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activateOnCreate",
+                                          "%s", str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volumeConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "voldisplaycount", &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch volcount");
+        goto out;
+    }
+
+    /* Get config of all the volumes */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname", i);
+        ret = dict_get_str(dict, buf, &str_value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              str_value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-active-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not fetch"
+                   " effective snap_max_hard_limit for "
+                   "%s",
+                   str_value);
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"effectiveHardLimit", "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-soft-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot config set output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing status output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_config_set(xmlTextWriterPtr writer, xmlDocPtr doc,
+                            dict_t *dict)
+{
+    int ret = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *volname = NULL;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+    ret = dict_get_str(dict, "auto-delete", &auto_delete);
+    ret = dict_get_str(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "At least one option from "
+               "snap-max-hard-limit, snap-max-soft-limit, auto-delete"
+               " and snap-activate-on-create should be set");
+        goto out;
+    }
+
+    /* Ignore the error, as volname is optional */
+    ret = dict_get_str(dict, "volname", &volname);
+
+    if (NULL == volname) {
+        /* <systemConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    } else {
+        /* <volumeConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+    }
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (hard_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newHardLimit",
+                                              "%" PRIu64, hard_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (soft_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newSoftLimit",
+                                              "%" PRIu64, soft_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (auto_delete) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete",
+                                              "%s", auto_delete);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (snap_activate) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"activateOnCreate", "%s", snap_activate);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volumeConfig> or </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/* This function will generate snapshot config output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing config output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_config(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    int config_command = 0;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+            ret = cli_xml_snapshot_config_set(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config set command");
+                goto out;
+            }
+
+            break;
+        case GF_SNAP_CONFIG_DISPLAY:
+            ret = cli_xml_snapshot_config_show(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config show command");
+                goto out;
+            }
+            break;
+        default:
+            gf_log("cli", GF_LOG_ERROR, "Unknown config command :%d",
+                   config_command);
+            ret = -1;
+            goto out;
+    }
+
+    /* </snapConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+
+/* This function will generate snapshot activate or
+ * deactivate output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing activate or deactivate output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_activate_deactivate(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                     dict_t *dict, int cmd)
+{
+    int ret = -1;
+    char *buffer = NULL;
+    char *tag = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    if (GF_SNAP_OPTION_TYPE_ACTIVATE == cmd) {
+        tag = "snapActivate";
+    } else if (GF_SNAP_OPTION_TYPE_DEACTIVATE == cmd) {
+        tag = "snapDeactivate";
+    } else {
+        gf_log("cli", GF_LOG_ERROR, "invalid command %d", cmd);
+        goto out;
+    }
+
+    /* <snapActivate> or <snapDeactivate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)tag);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapActivate> or </snapDeactivate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+
+    return ret;
+}
+#endif /* HAVE_LIB_XML */
+
+/* This function will generate snapshot delete output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing delete output
+ * @param rsp           cli response
+ *
+ * @return 0 on success and -1 on failure
+ */
+int
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp)
+{
+    int ret = -1;
+#ifdef HAVE_LIB_XML
+    char *str_value = NULL;
+    xmlTextWriterPtr writer = local->writer;
+    xmlDocPtr doc = local->doc;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto xmlend;
+    }
+
+    if (!rsp->op_ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Success");
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Failure");
+        XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+        ret = cli_xml_output_common(writer, rsp->op_ret, rsp->op_errno,
+                                    rsp->op_errstr);
+    }
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto xmlend;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+xmlend:
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+#endif /* HAVE_LIB_XML */
+    ret = 0;
+out:
+
+    return ret;
+}
+
+int
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_ASSERT(local);
+
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_snap_status_end(cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_ASSERT(local);
+
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(local->writer, local->doc);
+out:
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    int delete_cmd = -1;
+
+    GF_ASSERT(local);
+
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapDelete");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_xml_output_snap_delete_end(cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_ASSERT(local);
+
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapDelete> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = cli_end_xml_output(local->writer, local->doc);
+out:
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif
+}
+/* This function will generate xml output for all the snapshot commands
+ *
+ * @param cmd_type      command type
+ * @param dict          dict containing snapshot command output
+ * @param op_ret        return value of the snapshot command
+ * @param op_errno      errno for the snapshot command
+ * @param op_errstr     error string for the snapshot command
+ *
+ * @return 0 on success and -1 on failure
+ */
+int
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml begin block");
+        goto out;
+    }
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml common block");
+        goto out;
+    }
+
+    /* In case of command failure just printing the error message is good
+     * enough */
+    if (0 != op_ret) {
+        goto end;
+    }
+
+    switch (cmd_type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            ret = cli_xml_snapshot_create(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot create command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            ret = cli_xml_snapshot_clone(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot clone command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            ret = cli_xml_snapshot_restore(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot restore command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_LIST:
+            ret = cli_xml_snapshot_list(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot list command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_STATUS:
+            ret = cli_xml_snapshot_status(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create"
+                       "xml output for snapshot status command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_INFO:
+            ret = cli_xml_snapshot_info(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot info command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            ret = cli_xml_snapshot_activate_deactivate(writer, doc, dict,
+                                                       cmd_type);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = cli_xml_snapshot_config(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
+        default:
+            gf_log("cli", GF_LOG_ERROR, "Unexpected snapshot command: %d",
+                   cmd_type);
+            goto out;
+    }
+
+end:
+    ret = cli_end_xml_output(writer, doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml end block");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
+#else
+    return 0;
+#endif /* HAVE_LIB_XML */
+}
+
+int
+cli_xml_snapshot_begin_composite_op(cli_local_t *local)
+{
+    int ret = -1;
+#ifdef HAVE_LIB_XML
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_begin(local, 0, 0, NULL);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_begin(local, 0, 0, NULL);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Error creating xml output");
+        goto out;
+    }
+
+#endif /* HAVE_LIB_XML */
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+cli_xml_snapshot_end_composite_op(cli_local_t *local)
+{
+    int ret = -1;
+#ifdef HAVE_LIB_XML
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_end(local);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_end(local);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Error creating xml "
+               "output");
+        goto out;
+    }
+#endif /* HAVE_LIB_XML */
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict, char *key)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("cli", (local != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (key != NULL), out);
+
+    ret = cli_xml_snapshot_status_per_snap(local->writer, local->doc, dict,
+                                           key);
+out:
+    return ret;
+#else
+    return 0;
+#endif /* HAVE_LIB_XML */
+}
+
+int
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+    int i = 0;
+    int ret = -1;
+    int count = 0;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char dict_key[50] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volGetopts");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count "
+               "from the dictionary");
+        goto out;
+    }
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Value of count :%d is "
+               "invalid",
+               count);
+        ret = -1;
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 1; i <= count; i++) {
+        sprintf(dict_key, "key%d", i);
+        ret = dict_get_str(dict, dict_key, &key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to"
+                   " retrieve %s from the "
+                   "dictionary",
+                   dict_key);
+            goto out;
+        }
+        sprintf(dict_key, "value%d", i);
+        ret = dict_get_str(dict, dict_key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to "
+                   "retrieve key value for %s from"
+                   "the dictionary",
+                   dict_key);
+            goto out;
+        }
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"Opt");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Option", "%s",
+                                              key);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Value", "%s",
+                                              value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    ret = cli_end_xml_output(writer, doc);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+#else
+    return 0;
+#endif /* HAVE_LIB_XML */
+}
+
+int
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"errstr",
+                                          "%s", errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+int
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        hl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_final);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRId64, sl_num);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"used_space", "%" PRId64, used);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"avail_space",
+        !limit_set ? "N/A" : "%" PRId64, avail);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    return ret;
+#else
+    return 0;
+#endif /* HAVE_LIB_XML */
+}
+
+int
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set)
+{
+#if (HAVE_LIB_XML)
+    int ret = -1;
+
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        limits->hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRIu64, sl_val);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"file_count", "%" PRId64,
+                                          used_space->file_count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"dir_count",
+                                          "%" PRIu64, used_space->dir_count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"available",
+                                          !limit_set ? "N/A" : "%" PRId64,
+                                          avail);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    return ret;
+#else
+    return 0;
+#endif /* HAVE_LIB_XML */
+}
diff --git a/cli/src/cli.c b/cli/src/cli.c
index 075b7f43d21..a52b39c5fb8 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -39,599 +29,884 @@
 #include <semaphore.h>
 #include <errno.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifdef HAVE_MALLOC_H
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
 #include "cli.h"
+#include "cli-quotad-client.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "list.h"
-#include "timer.h"
-#include "stack.h"
-#include "revision.h"
-#include "common-utils.h"
-#include "event.h"
-#include "globals.h"
-#include "syscall.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/revision.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/call-stub.h>
 #include <fnmatch.h>
 
-extern int connected;
-/* using argp for command line parsing */
-static char gf_doc[] = "";
-
-static char argp_doc[] = "";
+#include "xdr-generic.h"
 
-const char *argp_program_version = ""                                 \
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__ \
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"  \
-        "Copyright (c) 2006-2010 Gluster Inc. "                       \
-        "<http://www.gluster.com>\n"                                  \
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"              \
-        "You may redistribute copies of GlusterFS under the terms of "\
-        "the GNU Affero General Public License.";
+/* using argp for command line parsing */
 
+const char *argp_program_version =
+    "" PACKAGE_NAME " " PACKAGE_VERSION
+    "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION
+    "\n"
+    "Copyright (c) 2006-2016 Red Hat, Inc. "
+    "<https://www.gluster.org/>\n"
+    "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+    "It is licensed to you under your choice of the GNU Lesser\n"
+    "General Public License, version 3 or any later version (LGPLv3\n"
+    "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+    "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
-static struct argp_option gf_options[] = {
-        {0, 0, 0, 0, "Basic options:"},
-        {0, }
-};
+struct rpc_clnt *global_quotad_rpc;
 
 struct rpc_clnt *global_rpc;
 
 rpc_clnt_prog_t *cli_rpc_prog;
 
+extern struct rpc_clnt_program cli_prog;
 
-extern struct rpc_clnt_program cli3_1_prog;
+int cli_default_conn_timeout = 120;
+int cli_ten_minutes_timeout = 600;
 
-static error_t
-parse_opts (int key, char *arg, struct argp_state *argp_state)
+static int
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        struct cli_state  *state = NULL;
-        char             **argv = NULL;
-
-        state = argp_state->input;
-
-        switch (key) {
-        case ARGP_KEY_ARG:
-                if (!state->argc) {
-                        argv = calloc (state->argc + 2,
-                                       sizeof (*state->argv));
-                } else {
-                        argv = realloc (state->argv, (state->argc + 2) *
-                                        sizeof (*state->argv));
-                }
-                if (!argv)
-                        return -1;
-
-                state->argv = argv;
-
-                argv[state->argc] = strdup (arg);
-                if (!argv[state->argc])
-                        return -1;
-                state->argc++;
-                argv[state->argc] = NULL;
-
-                break;
-        }
-
-        return 0;
-}
-
-
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    if (!ctx)
+        return ret;
 
-static char *
-generate_uuid ()
-{
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
-
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
+    ret = xlator_mem_acct_init(THIS, cli_mt_end);
+    if (ret != 0) {
+        gf_log("cli", GF_LOG_ERROR, "Memory accounting init failed.");
+        return ret;
+    }
+
+    /* Resetting ret to -1 to so in case of failure
+     * we can relese allocated resource.
+     */
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate uuid.");
+        goto out;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create iobuf pool.");
+        goto out;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create event pool.");
+        goto out;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), cli_mt_call_pool_t);
+    if (!pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create call pool.");
+        goto out;
+    }
+
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create frame mem pool.");
+        goto out;
+    }
+
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
+
+    if (!pool->stack_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create stack mem pool.");
+        goto out;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to stub mem pool.");
+        goto out;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pool.");
+        goto out;
+    }
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pair pool.");
+        goto out;
+    }
+
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict data pool.");
+        goto out;
+    }
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create logbuf pool.");
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    cmd_args = &ctx->cmd_args;
+
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
+
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
+
+    ret = 0;
 
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
+out:
+    if (ret != 0) {
+        if (pool) {
+            mem_pool_destroy(pool->frame_mem_pool);
+            mem_pool_destroy(pool->stack_mem_pool);
         }
-
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (tmp_str, 1024, "%s-%d-%s:%"
-#ifdef GF_DARWIN_HOST_OS
-                  PRId32,
-#else
-                  "ld",
-#endif
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
+        GF_FREE(pool);
+        pool = NULL;
+        GF_FREE(ctx->process_uuid);
+        mem_pool_destroy(ctx->stub_mem_pool);
+        mem_pool_destroy(ctx->dict_pool);
+        mem_pool_destroy(ctx->dict_pair_pool);
+        mem_pool_destroy(ctx->dict_data_pool);
+        mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+logging_init(glusterfs_ctx_t *ctx, struct cli_state *state)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-
-        xlator_mem_acct_init (THIS, cli_mt_end);
-
-        ctx->process_uuid = generate_uuid ();
-        if (!ctx->process_uuid)
-                return -1;
-
-        ctx->page_size  = 128 * GF_UNIT_KB;
-
-        ctx->iobuf_pool = iobuf_pool_new (8 * GF_UNIT_MB, ctx->page_size);
-        if (!ctx->iobuf_pool)
-                return -1;
+    char *log_file = state->log_file ? state->log_file
+                                     : DEFAULT_CLI_LOG_FILE_DIRECTORY
+                         "/cli.log";
+
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(ctx, log_file, NULL) == -1) {
+        fprintf(stderr, "ERROR: failed to open logfile %s\n", log_file);
+    }
+
+    /* CLI should not have something to DEBUG after the release,
+       hence defaulting to INFO loglevel */
+    gf_log_set_loglevel(ctx, (state->log_level == GF_LOG_NONE)
+                                 ? GF_LOG_INFO
+                                 : state->log_level);
+
+    return 0;
+}
 
-        ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
-        if (!ctx->event_pool)
-                return -1;
+int
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          cli_mt_call_pool_t);
-        if (!pool)
-                return -1;
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
 
-        /* frame_mem_pool size 112 * 16k */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384);
+            new_iobref = 1;
+        }
 
-        if (!pool->frame_mem_pool)
-                return -1;
+        iobref_add(iobref, iobuf);
 
-        /* stack_mem_pool size 256 * 8k */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192); 
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
 
-        if (!pool->stack_mem_pool)
-                return -1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-        if (!ctx->stub_mem_pool)
-                return -1;
+    if (!rpc)
+        rpc = global_rpc;
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+out:
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
+}
 
-        pthread_mutex_init (&(ctx->lock), NULL);
+int
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data)
+{
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        cmd_args = &ctx->cmd_args;
+    this = mydata;
 
-        /* parsing command line arguments */
-        cmd_args->log_file  = "/dev/null";
-        cmd_args->log_level = GF_LOG_NONE;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            cli_cmd_broadcast_connected(_gf_true);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
+        }
 
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+        case RPC_CLNT_DISCONNECT: {
+            cli_cmd_broadcast_connected(_gf_false);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            if (!global_state->prompt && global_state->await_connected) {
+                ret = 1;
+                cli_out(
+                    "Connection failed. Please check if gluster "
+                    "daemon is operational.");
+                exit(ret);
+            }
+            break;
+        }
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+        default:
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return 0;
+    return ret;
 }
 
-
-static int
-logging_init (glusterfs_ctx_t *ctx)
+static gf_boolean_t
+is_valid_int(char *str)
 {
-        cmd_args_t *cmd_args = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        if (gf_log_init (cmd_args->log_file) == -1) {
-                fprintf (stderr,
-                         "failed to open logfile %s.  exiting\n",
-                         cmd_args->log_file);
-                return -1;
-        }
-
-        gf_log_set_loglevel (cmd_args->log_level);
-
-        return 0;
+    if (*str == '-')
+        ++str;
+
+    /* Handle empty string or just "-".*/
+    if (!*str)
+        return _gf_false;
+
+    /* Check for non-digit chars in the rest of the string */
+    while (*str) {
+        if (!isdigit(*str))
+            return _gf_false;
+        else
+            ++str;
+    }
+    return _gf_true;
 }
 
+/*
+ * ret: 0: option successfully processed
+ *      1: signalling end of option list
+ *     -1: unknown option
+ *     -2: parsing issue (avoid unknown option error)
+ */
 int
-cli_submit_request (void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    cli_serialize_t sfunc, xlator_t *this,
-                    fop_cbk_fn_t cbkfn)
+cli_opt_parse(char *opt, struct cli_state *state)
 {
-        int                     ret         = -1;
-        int                     count      = 0;
-        char                    start_ping = 0;
-        struct iovec            iov         = {0, };
-        struct iobuf            *iobuf = NULL;
-        char                    new_iobref = 0;
+    char *oarg = NULL;
+    gf_boolean_t secure_mgmt_tmp = 0;
+
+    if (strcmp(opt, "") == 0)
+        return 1;
+    if (strcmp(opt, "help") == 0) {
+        cli_out(
+            " peer help                - display help for peer commands\n"
+            " volume help              - display help for volume commands\n"
+            " volume bitrot help       - display help for volume"
+            " bitrot commands\n"
+            " volume quota help        - display help for volume"
+            " quota commands\n"
+            " snapshot help            - display help for snapshot commands\n"
+            " global help              - list global commands\n");
+        exit(0);
+    }
+
+    if (strcmp(opt, "version") == 0) {
+        cli_out("%s", argp_program_version);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-logdir") == 0) {
+        cli_out("%s", DEFAULT_LOG_FILE_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-statedumpdir") == 0) {
+        cli_out("%s", DEFAULT_VAR_RUN_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "xml") == 0) {
+#if (HAVE_LIB_XML)
+        state->mode |= GLUSTER_MODE_XML;
+#else
+        cli_err("XML output not supported. Ignoring '--xml' option");
+#endif
+        return 0;
+    }
 
-        GF_ASSERT (this);
+    if (strcmp(opt, "nolog") == 0) {
+        state->mode |= GLUSTER_MODE_GLFSHEAL_NOLOG;
+        return 0;
+    }
 
-        iobuf = iobuf_get (this->ctx->iobuf_pool);
-        if (!iobuf) {
-                goto out;
-        };
+    if (strcmp(opt, "wignore-partition") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE_PARTITION;
+        return 0;
+    }
 
-        if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        goto out;
-                }
+    if (strcmp(opt, "wignore") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE;
+        return 0;
+    }
 
-                new_iobref = 1;
+    oarg = strtail(opt, "mode=");
+    if (oarg) {
+        if (strcmp(oarg, "script") == 0) {
+            state->mode |= GLUSTER_MODE_SCRIPT;
+            return 0;
         }
 
-        iobref_add (iobref, iobuf);
+        if (strcmp(oarg, "interactive") == 0)
+            return 0;
 
-        iov.iov_base = iobuf->ptr;
-        iov.iov_len  = 128 * GF_UNIT_KB;
+        return -1;
+    }
 
+    oarg = strtail(opt, "remote-host=");
+    if (oarg) {
+        state->remote_host = oarg;
+        return 0;
+    }
 
-        /* Create the xdr payload */
-        if (req && sfunc) {
-                ret = sfunc (iov, req);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
-        }
+    oarg = strtail(opt, "inet6");
+    if (oarg) {
+        state->address_family = "inet6";
+        return 0;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (global_rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-
-        if (ret == 0) {
-                pthread_mutex_lock (&global_rpc->conn.lock);
-                {
-                        if (!global_rpc->conn.ping_started) {
-                                start_ping = 1;
-                        }
-                }
-                pthread_mutex_unlock (&global_rpc->conn.lock);
+    oarg = strtail(opt, "log-file=");
+    if (oarg) {
+        state->log_file = oarg;
+        return 0;
+    }
+    oarg = strtail(opt, "timeout=");
+    if (oarg) {
+        if (!is_valid_int(oarg) || atoi(oarg) <= 0) {
+            cli_err("timeout value should be a positive integer");
+            return -2; /* -2 instead of -1 to avoid unknown option
+                          error */
         }
+        cli_default_conn_timeout = atoi(oarg);
+        return 0;
+    }
+
+    oarg = strtail(opt, "log-level=");
+    if (oarg) {
+        int log_level = glusterd_check_log_level(oarg);
+        if (log_level == -1)
+            return -1;
+        state->log_level = (gf_loglevel_t)log_level;
+        return 0;
+    }
 
-        ret = 0;
+    oarg = strtail(opt, "glusterd-sock=");
+    if (oarg) {
+        state->glusterd_sock = oarg;
+        return 0;
+    }
+
+    oarg = strtail(opt, "secure-mgmt=");
+    if (oarg) {
+        if (gf_string2boolean(oarg, &secure_mgmt_tmp) == 0) {
+            if (secure_mgmt_tmp) {
+                /* See declaration for why this is an int. */
+                state->ctx->secure_mgmt = 1;
+            }
+        } else {
+            cli_err("invalid secure-mgmt value (ignored)");
+        }
+        return 0;
+    }
 
-out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        return ret;
+    return -1;
 }
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data)
+parse_cmdline(int argc, char *argv[], struct cli_state *state)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
-
-        this = mydata;
-
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-
-                cli_cmd_broadcast_connected ();
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-               break;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    char *opt = NULL;
+
+    state->argc = argc - 1;
+    state->argv = &argv[1];
+
+    /* Do this first so that an option can override. */
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        state->ctx->secure_mgmt = 1;
+        state->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    if (state->argc > GEO_REP_CMD_CONFIG_INDEX &&
+        strtail(state->argv[GEO_REP_CMD_INDEX], "geo") &&
+        strtail(state->argv[GEO_REP_CMD_CONFIG_INDEX], "co"))
+        goto done;
+
+    for (i = 0; i < state->argc; i++) {
+        opt = strtail(state->argv[i], "--");
+        if (!opt)
+            continue;
+        ret = cli_opt_parse(opt, state);
+        if (ret == -1) {
+            cli_out("unrecognized option --%s\n", opt);
+            usage();
+            return ret;
+        } else if (ret == -2) {
+            return ret;
         }
-
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                connected = 0;
-                break;
+        for (j = i; j < state->argc - 1; j++)
+            state->argv[j] = state->argv[j + 1];
+        state->argc--;
+        /* argv shifted, next check should be at i again */
+        i--;
+        if (ret == 1) {
+            /* end of cli options */
+            ret = 0;
+            break;
         }
+    }
 
-        default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+done:
+    state->argv[state->argc] = NULL;
 
-        return ret;
+    return ret;
 }
 
 int
-cli_opt_parse (char *opt, struct cli_state *state)
+cli_cmd_tree_init(struct cli_cmd_tree *tree)
 {
-        char *oarg;
-
-        oarg = strtail (opt, "mode=");
-        if (oarg) {
-                if (strcmp (oarg, "script") == 0) {
-                        state->mode |= GLUSTER_MODE_SCRIPT;
-                        return 0;
-                }
-                if (strcmp (oarg, "interactive") == 0)
-                        return 0;
-                return -1;
-        }
+    struct cli_cmd_word *root = NULL;
+    int ret = 0;
 
-        oarg = strtail (opt, "remote-host=");
-        if (oarg) {
-                state->remote_host = oarg;
-                return 0;
-        }
+    root = &tree->root;
+    root->tree = tree;
 
-        return -1;
+    return ret;
 }
 
 int
-parse_cmdline (int argc, char *argv[], struct cli_state *state)
+cli_state_init(struct cli_state *state)
 {
-        int         ret = 0;
-        int         i = 0;
-        int         j = 0;
-        char        *opt = NULL;
-        struct argp argp = { 0,};
-
-        argp.options    = gf_options;
-        argp.parser     = parse_opts;
-        argp.args_doc   = argp_doc;
-        argp.doc        = gf_doc;
-
-        for (i = 0; i < argc; i++) {
-                opt = strtail (argv[i], "--");
-                if (opt) {
-                        ret = cli_opt_parse (opt, state);
-                        if (ret == -1) {
-                                break;
-                        }
-                        for (j = i; j < argc - 1; j++)
-                                argv[j] = argv[j + 1];
-                        argc--;
-                }
-        }
+    struct cli_cmd_tree *tree = NULL;
+    int ret = 0;
 
-        ret = argp_parse (&argp, argc, argv,
-                          ARGP_IN_ORDER, NULL, state);
+    state->log_level = GF_LOG_NONE;
 
-        return ret;
-}
+    tree = &state->tree;
+    tree->state = state;
+
+    ret = cli_cmd_tree_init(tree);
 
+    return ret;
+}
 
 int
-cli_cmd_tree_init (struct cli_cmd_tree *tree)
+cli_usage_out(const char *usage)
 {
-        struct cli_cmd_word  *root = NULL;
-        int                   ret = 0;
+    GF_ASSERT(usage);
 
-        root = &tree->root;
-        root->tree = tree;
+    if (!usage || usage[0] == '\0')
+        return -1;
 
-        return ret;
+    cli_err("\nUsage:\n%s\n", usage);
+    return 0;
 }
 
-
 int
-cli_state_init (struct cli_state *state)
+_cli_err(const char *fmt, ...)
 {
-        struct cli_cmd_tree  *tree = NULL;
-        int                   ret = 0;
-
-
-        state->remote_host = "localhost";
-
-        tree = &state->tree;
-        tree->state = state;
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
-        ret = cli_cmd_tree_init (tree);
+    va_start(ap, fmt);
 
+#ifdef HAVE_READLINE
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_err(state, fmt, ap);
+        va_end(ap);
         return ret;
-}
-
-int
-cli_usage_out (const char *usage)
-{
-        GF_ASSERT (usage);
-        GF_ASSERT (usage[0] != '\0');
+    }
+#endif
 
-        if (!usage || usage[0] == '\0')
-                return -1;
+    ret = vfprintf(stderr, fmt, ap);
+    fprintf(stderr, "\n");
+    va_end(ap);
 
-        cli_out ("Usage: %s", usage);
-        return 0;
+    return ret;
 }
 
 int
-cli_out (const char *fmt, ...)
+_cli_out(const char *fmt, ...)
 {
-        struct cli_state *state = NULL;
-        va_list           ap;
-        int               ret = 0;
-
-        state = global_state;
-
-        va_start (ap, fmt);
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
+    va_start(ap, fmt);
 #ifdef HAVE_READLINE
-        if (state->rl_enabled && !state->rl_processing)
-                return cli_rl_out(state, fmt, ap);
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_out(state, fmt, ap);
+        va_end(ap);
+        return ret;
+    }
 #endif
 
-        ret = vprintf (fmt, ap);
-        printf ("\n");
-        va_end (ap);
+    ret = vprintf(fmt, ap);
+    printf("\n");
+    va_end(ap);
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_rpc_init (struct cli_state *state)
+cli_quotad_clnt_rpc_init(void)
 {
-        struct rpc_clnt         *rpc = NULL;
-        struct rpc_clnt_config  rpc_cfg = {0,};
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-        int                     port = CLI_GLUSTERD_PORT;
-        xlator_t                *this = NULL;
-
-
-        this = THIS;
-        cli_rpc_prog = &cli3_1_prog;
-        options = dict_new ();
-        if (!options)
-                goto out;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *rpc_opts = NULL;
+    int ret = -1;
+
+    rpc_opts = dict_new();
+    if (!rpc_opts) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(rpc_opts, "transport.address-family", "unix");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport-type", "socket");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport.socket.connect-path",
+                       "/var/run/gluster/quotad.socket");
+    if (ret)
+        goto out;
+
+    rpc = cli_quotad_clnt_init(THIS, rpc_opts);
+    if (!rpc)
+        goto out;
+
+    global_quotad_rpc = rpc;
+out:
+    if (rpc_opts) {
+        dict_unref(rpc_opts);
+    }
+    return rpc;
+}
 
-        ret = dict_set_str (options, "remote-host", state->remote_host);
+struct rpc_clnt *
+cli_rpc_init(struct cli_state *state)
+{
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = CLI_GLUSTERD_PORT;
+    xlator_t *this = NULL;
+#ifdef IPV6_DEFAULT
+    char *addr_family = "inet6";
+#else
+    char *addr_family = "inet";
+#endif
+
+    this = THIS;
+    cli_rpc_prog = &cli_prog;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    /* If address family specified in CLI */
+    if (state->address_family) {
+        addr_family = state->address_family;
+    }
+
+    /* Connect to glusterd using the specified method, giving preference
+     * to a unix socket connection.  If nothing is specified, connect to
+     * the default glusterd socket.
+     */
+    if (state->glusterd_sock) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to glusterd using "
+               "sockfile %s",
+               state->glusterd_sock);
+        ret = rpc_transport_unix_options_build(options, state->glusterd_sock,
+                                               0);
         if (ret)
-                goto out;
+            goto out;
+    } else if (state->remote_host) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to remote glusterd at "
+               "%s",
+               state->remote_host);
+
+        ret = dict_set_str(options, "remote-host", state->remote_host);
+        if (ret)
+            goto out;
 
         if (state->remote_port)
-                port = state->remote_port;
+            port = state->remote_port;
 
-        rpc_cfg.remote_host = state->remote_host;
-        rpc_cfg.remote_port = port;
-
-        ret = dict_set_int32 (options, "remote-port", port);
+        ret = dict_set_int32(options, "remote-port", port);
         if (ret)
-                goto out;
+            goto out;
 
-        ret = dict_set_str (options, "transport.address-family", "inet");
+        ret = dict_set_str(options, "transport.address-family", addr_family);
         if (ret)
-                goto out;
+            goto out;
+    } else {
+        gf_log("cli", GF_LOG_DEBUG,
+               "Connecting to glusterd using "
+               "default socket");
+        ret = rpc_transport_unix_options_build(options,
+                                               DEFAULT_GLUSTERD_SOCKFILE, 0);
+        if (ret)
+            goto out;
+    }
 
-        rpc = rpc_clnt_init (&rpc_cfg, options, this->ctx, this->name);
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
 
-        if (rpc) {
-                ret = rpc_clnt_register_notify (rpc, cli_rpc_notify, this);
-        }
+    ret = rpc_clnt_register_notify(rpc, cli_rpc_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    ret = rpc_clnt_start(rpc);
 out:
-        return rpc;
+    if (options)
+        dict_unref(options);
+
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
+    return rpc;
 }
 
 cli_local_t *
-cli_local_get ()
+cli_local_get()
 {
-        cli_local_t     *local = NULL;
+    cli_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), cli_mt_cli_local_t);
+    local = GF_CALLOC(1, sizeof(*local), cli_mt_cli_local_t);
+    LOCK_INIT(&local->lock);
+    INIT_LIST_HEAD(&local->dict_list);
 
-        return local;
+    return local;
 }
 
 void
-cli_local_wipe (cli_local_t *local)
+cli_local_wipe(cli_local_t *local)
 {
-        if (local) {
-                GF_FREE (local);
-        }
-
-        return;
+    if (local) {
+        GF_FREE(local->get_vol.volname);
+        if (local->dict)
+            dict_unref(local->dict);
+        GF_FREE(local);
+    }
+
+    return;
 }
 
-void
-cli_path_strip_trailing_slashes (char *path)
+struct cli_state *global_state;
+
+int
+main(int argc, char *argv[])
 {
-        int i = 0;
-        int len = 0;
+    struct cli_state state = {
+        0,
+    };
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
 
-        if (!path)
-                return;
+    mem_pools_init();
 
-        len = strlen (path);
-        for (i = len - 1; i > 0 ; i--) {
-                if (path[i] != '/')
-                        break;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        return ENOMEM;
 
-        }
+#ifdef DEBUG
+    gf_mem_acct_enable_set(ctx);
+#endif
 
-        if (i < (len - 1))
-                path[i + 1] = '\0';
-}
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        return ret;
 
-struct cli_state *global_state;
+    THIS->ctx = ctx;
 
-int
-main (int argc, char *argv[])
-{
-        struct cli_state   state = {0, };
-        int                ret = -1;
-        glusterfs_ctx_t   *ctx = NULL;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = glusterfs_globals_init ();
-        if (ret)
-                return ret;
+    cli_default_conn_timeout = 120;
+    cli_ten_minutes_timeout = 600;
 
-        ctx = glusterfs_ctx_get ();
-        if (!ctx)
-                return ENOMEM;
+    ret = cli_state_init(&state);
+    if (ret)
+        goto out;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto out;
+    state.ctx = ctx;
+    global_state = &state;
 
-        ret = cli_state_init (&state);
-        if (ret)
-                goto out;
+    ret = parse_cmdline(argc, argv, &state);
+    if (ret)
+        goto out;
 
-        state.ctx = ctx;
-        global_state = &state;
+    ret = logging_init(ctx, &state);
+    if (ret)
+        goto out;
 
-        ret = parse_cmdline (argc, argv, &state);
-        if (ret)
-                goto out;
+    gf_log("cli", GF_LOG_INFO, "Started running %s with version %s", argv[0],
+           PACKAGE_VERSION);
 
-        if (geteuid ()) {
-                printf ("Only super user can run this command\n");
-                return EPERM;
-        }
+    global_rpc = cli_rpc_init(&state);
+    if (!global_rpc)
+        goto out;
 
-        global_rpc = cli_rpc_init (&state);
-        if (!global_rpc)
-                goto out;
+    global_quotad_rpc = cli_quotad_clnt_rpc_init();
+    if (!global_quotad_rpc)
+        goto out;
 
-        ret = logging_init (ctx);
-        if (ret)
-                goto out;
+    ret = cli_cmds_register(&state);
+    if (ret)
+        goto out;
 
-        ret = cli_cmds_register (&state);
-        if (ret)
-                goto out;
+    ret = cli_input_init(&state);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_cond_init ();
-        if (ret)
-                goto out;
+    ret = gf_event_dispatch(ctx->event_pool);
 
-        ret = cli_input_init (&state);
-        if (ret)
-                goto out;
+out:
+    //        glusterfs_ctx_destroy (ctx);
 
-        ret = event_dispatch (ctx->event_pool);
+    mem_pools_fini();
 
-out:
-//        glusterfs_ctx_destroy (ctx);
+    return ret;
+}
 
-        return ret;
+void
+cli_print_line(int len)
+{
+    GF_ASSERT(len > 0);
+
+    while (len--)
+        printf("-");
+
+    printf("\n");
+}
+
+void
+print_quota_list_header(int type)
+{
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        cli_out(
+            "                  Path                   Hard-limit "
+            " Soft-limit      Used  Available  Soft-limit "
+            "exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "---------------------");
+    } else {
+        cli_out(
+            "                  Path                   Hard-limit  "
+            " Soft-limit      Files       Dirs     Available  "
+            "Soft-limit exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "-------------------------------------");
+    }
+}
+
+void
+print_quota_list_empty(char *path, int type)
+{
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        cli_out("%-40s %7s %9s %10s %7s %15s %20s", path, "N/A", "N/A", "N/A",
+                "N/A", "N/A", "N/A");
+    else
+        cli_out("%-40s %9s %9s %12s %10s %10s %15s %20s", path, "N/A", "N/A",
+                "N/A", "N/A", "N/A", "N/A", "N/A");
 }
diff --git a/cli/src/cli.h b/cli/src/cli.h
index c0fb5a7e5a4..c0d933e8f8a 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -1,217 +1,516 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_H__
 #define __CLI_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpc-clnt.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "protocol-common.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/quota-common-utils.h>
 
-#define DEFAULT_EVENT_POOL_SIZE            16384
-#define CLI_GLUSTERD_PORT                  24007
-#define CLI_DEFAULT_CONN_TIMEOUT             120
-#define CLI_DEFAULT_CMD_TIMEOUT              120
+#include "cli1-xdr.h"
+#include "gd-common-utils.h"
+
+#if (HAVE_LIB_XML)
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+#endif
+
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define CLI_GLUSTERD_PORT 24007
+#define DEFAULT_CLI_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
+#define CLI_VOL_STATUS_BRICK_LEN 43
+#define CLI_TAB_LENGTH 8
+#define CLI_BRICK_STATUS_LINE_LEN 78
+
+/* Geo-rep command positional arguments' index  */
+#define GEO_REP_CMD_INDEX 1
+#define GEO_REP_CMD_CONFIG_INDEX 4
 
 enum argp_option_keys {
-	ARGP_DEBUG_KEY = 133,
-	ARGP_PORT_KEY = 'p',
+    ARGP_DEBUG_KEY = 133,
+    ARGP_PORT_KEY = 'p',
 };
 
-#define GLUSTER_MODE_SCRIPT    (1 << 0)
+extern int cli_default_conn_timeout;
+extern int cli_ten_minutes_timeout;
+
+typedef enum {
+    COLD_BRICK_COUNT,
+    COLD_TYPE,
+    COLD_DIST_COUNT,
+    COLD_REPLICA_COUNT,
+    COLD_ARBITER_COUNT,
+    COLD_DISPERSE_COUNT,
+    COLD_REDUNDANCY_COUNT,
+    HOT_BRICK_COUNT,
+    HOT_TYPE,
+    HOT_REPLICA_COUNT,
+    MAX
+} values;
+
+#define GLUSTER_MODE_SCRIPT (1 << 0)
 #define GLUSTER_MODE_ERR_FATAL (1 << 1)
+#define GLUSTER_MODE_XML (1 << 2)
+#define GLUSTER_MODE_WIGNORE (1 << 3)
+#define GLUSTER_MODE_WIGNORE_PARTITION (1 << 4)
+#define GLUSTER_MODE_GLFSHEAL_NOLOG (1 << 5)
+
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path)             \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list%s", volname, path); \
+    } while (0)
 
 struct cli_state;
 struct cli_cmd_word;
 struct cli_cmd_tree;
+struct cli_cmd;
 
-typedef int (cli_cmd_cbk_t)(struct cli_state *state,
-                            struct cli_cmd_word *word,
-                            const char **words,
-                            int wordcount);
-typedef int (cli_cmd_match_t)(struct cli_cmd_word *word);
-typedef int (cli_cmd_filler_t)(struct cli_cmd_word *word);
+extern char *cli_vol_status_str[];
+extern char *cli_vol_task_status_str[];
+
+typedef int(cli_cmd_cbk_t)(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount);
+typedef void(cli_cmd_reg_cbk_t)(struct cli_cmd *this);
+
+typedef int(cli_cmd_match_t)(struct cli_cmd_word *word);
+typedef int(cli_cmd_filler_t)(struct cli_cmd_word *word);
 
 struct cli_cmd_word {
-        struct cli_cmd_tree   *tree;
-        const char            *word;
-        cli_cmd_filler_t      *filler;
-        cli_cmd_match_t       *match;
-        cli_cmd_cbk_t         *cbkfn;
-        const char            *desc;
-        const char            *pattern;
-        int                    nextwords_cnt;
-        struct cli_cmd_word  **nextwords;
+    struct cli_cmd_tree *tree;
+    const char *word;
+    cli_cmd_filler_t *filler;
+    cli_cmd_match_t *match;
+    cli_cmd_cbk_t *cbkfn;
+    const char *desc;
+    const char *pattern;
+    int nextwords_cnt;
+    struct cli_cmd_word **nextwords;
 };
 
-
 struct cli_cmd_tree {
-        struct cli_state      *state;
-        struct cli_cmd_word    root;
+    struct cli_state *state;
+    struct cli_cmd_word root;
 };
 
-
 struct cli_state {
-        int                   argc;
-        char                **argv;
+    int argc;
+    char **argv;
+
+    char debug;
 
-        char                  debug;
+    /* for events dispatching */
+    glusterfs_ctx_t *ctx;
 
-        /* for events dispatching */
-        glusterfs_ctx_t      *ctx;
+    /* registry of known commands */
+    struct cli_cmd_tree tree;
 
-        /* registry of known commands */
-        struct cli_cmd_tree   tree;
+    /* the thread which "executes" the command in non-interactive mode */
+    /* also the thread which reads from stdin in non-readline mode */
+    pthread_t input;
 
-        /* the thread which "executes" the command in non-interactive mode */
-        /* also the thread which reads from stdin in non-readline mode */
-        pthread_t             input;
+    /* terminal I/O */
+    const char *prompt;
+    int rl_enabled;
+    int rl_async;
+    int rl_processing;
 
-        /* terminal I/O */
-        const char           *prompt;
-        int                   rl_enabled;
-        int                   rl_async;
-        int                   rl_processing;
+    /* autocompletion state */
+    char **matches;
+    char **matchesp;
 
-        /* autocompletion state */
-        char                **matches;
-        char                **matchesp;
+    char *remote_host;
+    int remote_port;
+    int mode;
+    int await_connected;
 
-        char                 *remote_host;
-        int                   remote_port;
-        int                   mode;
+    char *log_file;
+    gf_loglevel_t log_level;
+
+    char *glusterd_sock;
+    char *address_family;
 };
 
 struct cli_local {
-        union {
-                struct {
-                        dict_t  *dict;
-                } create_vol;
-
-                struct {
-                        char    *volname;
-                        int     flags;
-                } start_vol;
-
-                struct {
-                        char    *volname;
-                        int     flags;
-                } stop_vol;
-
-                struct {
-                        char    *volname;
-                } delete_vol;
-
-                struct {
-                        char    *volname;
-                        int      cmd;
-                } defrag_vol;
-
-                struct {
-                        char    *volname;
-                        dict_t  *dict;
-                } replace_brick;
-
-                struct {
-                        char    *volname;
-                        int     flags;
-                } get_vol;
-        } u;
+    struct {
+        char *volname;
+        int flags;
+    } get_vol;
+
+    dict_t *dict;
+    const char **words;
+    /* Marker for volume status all */
+    gf_boolean_t all;
+#if (HAVE_LIB_XML)
+    xmlTextWriterPtr writer;
+    xmlDocPtr doc;
+    int vol_count;
+#endif
+    gf_lock_t lock;
+    struct list_head dict_list;
+};
+
+struct cli_volume_status {
+    int port;
+    int rdma_port;
+    int online;
+    uint64_t block_size;
+    uint64_t total_inodes;
+    uint64_t free_inodes;
+    char *brick;
+    char *pid_str;
+    char *free;
+    char *total;
+    char *fs_name;
+    char *mount_options;
+    char *device;
+    char *inode_size;
 };
 
+struct snap_config_opt_vals_ {
+    char *op_name;
+    char *question;
+};
+
+typedef struct cli_volume_status cli_volume_status_t;
+
 typedef struct cli_local cli_local_t;
 
-typedef ssize_t (*cli_serialize_t) (struct iovec outmsg, void *args);
+typedef ssize_t (*cli_serialize_t)(struct iovec outmsg, void *args);
 
 extern struct cli_state *global_state; /* use only in readline callback */
 
-int cli_cmd_register (struct cli_cmd_tree *tree, const char *template,
-                      cli_cmd_cbk_t cbk, const char *desc);
-int cli_cmds_register (struct cli_state *state);
+extern struct rpc_clnt *global_quotad_rpc;
+
+extern struct rpc_clnt *global_rpc;
+
+extern rpc_clnt_prog_t *cli_rpc_prog;
+
+typedef const char *(*cli_selector_t)(void *wcon);
+
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val);
+
+void *
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel);
+
+int
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd);
+int
+cli_cmds_register(struct cli_state *state);
 
-int cli_input_init (struct cli_state *state);
+int
+cli_input_init(struct cli_state *state);
 
-int cli_cmd_process (struct cli_state *state, int argc, char *argv[]);
-int cli_cmd_process_line (struct cli_state *state, const char *line);
+int
+cli_cmd_process(struct cli_state *state, int argc, char *argv[]);
+int
+cli_cmd_process_line(struct cli_state *state, const char *line);
+
+int
+cli_rl_enable(struct cli_state *state);
+int
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap);
 
-int cli_rl_enable (struct cli_state *state);
-int cli_rl_out (struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_usage_out(const char *usage);
 
-int cli_usage_out (const char *usage);
-int cli_out (const char *fmt, ...);
+int
+_cli_out(const char *fmt, ...);
+int
+_cli_err(const char *fmt, ...);
+
+#define cli_out(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_out(fmt);                                                         \
+                                                                               \
+    } while (0)
+
+#define cli_err(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_err(fmt);                                                         \
+                                                                               \
+    } while (0)
+
+#define usage()                                                                \
+    do {                                                                       \
+        cli_out(                                                               \
+            " Usage: gluster [options] <help> <peer>"                          \
+            " <pool> <volume>\n"                                               \
+            " Options:\n"                                                      \
+            " --help  Shows the help information\n"                            \
+            " --version  Shows the version\n"                                  \
+            " --print-logdir  Shows the log directory\n"                       \
+            " --print-statedumpdir Shows the state dump directory\n");         \
+                                                                               \
+    } while (0)
 
 int
-cli_submit_request (void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    cli_serialize_t sfunc, xlator_t *this,
-                    fop_cbk_fn_t cbkfn);
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 int32_t
-cli_cmd_volume_create_parse (const char **words, int wordcount,
-                             dict_t **options);
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **bricks);
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount,
-                          dict_t **options);
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **opt, char **errstr);
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options);
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options);
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **opt);
+
+int32_t
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr);
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr);
+
+int32_t
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr);
+
+int32_t
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *type);
+
+int32_t
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *command);
+
+int32_t
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
                                    dict_t **options);
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options);
+
+int32_t
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options);
 
-cli_local_t * cli_local_get ();
+int32_t
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options);
+int32_t
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options);
+
+cli_local_t *
+cli_local_get();
 
 void
-cli_local_wipe (cli_local_t *local);
+cli_local_wipe(cli_local_t *local);
+
+gf_boolean_t
+cli_cmd_connected();
+
+int32_t
+cli_cmd_await_connected(unsigned timeout);
+
+int32_t
+cli_cmd_broadcast_connected(gf_boolean_t status);
+
+int
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data);
+
+int32_t
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options);
+int32_t
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_await_connected ();
+cli_cmd_log_level_parse(const char **words, int wordcount, dict_t **options);
+
+int32_t
+cli_cmd_volume_status_parse(const char **words, int wordcount,
+                            dict_t **options);
+
+int
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options);
 
+int
+cli_cmd_volume_defrag_parse(const char **words, int wordcount,
+                            dict_t **options);
+
+int
+cli_print_brick_status(cli_volume_status_t *status);
+
+void
+cli_print_detailed_status(cli_volume_status_t *status);
+
+int
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status);
+
+void
+cli_print_line(int len);
+
+int
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr);
+
+int
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr);
+
+int
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
+
+int
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr);
+
+int
+cli_xml_output_vol_status_end(cli_local_t *local);
+
+int
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict);
+
+int
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
+
+int
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr);
+
+int
+cli_xml_output_vol_info_end(cli_local_t *local);
+
+int
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict);
+
+int
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr);
+int
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local);
+
+int
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr);
+
+int
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set);
+
+int
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set);
+
+int
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
+
+int
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op);
+
+int
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr);
+
+int
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr);
+
+int
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr);
+
+int
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr);
+int
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict);
+
+int
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp);
+
+int
+cli_xml_snapshot_begin_composite_op(cli_local_t *local);
+
+int
+cli_xml_snapshot_end_composite_op(cli_local_t *local);
+
+int
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
+int
+cli_xml_output_snap_delete_end(cli_local_t *local);
+
+int
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
+int
+cli_xml_output_snap_status_end(cli_local_t *local);
+int
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
+int
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict,
+                                    char *key);
 int32_t
-cli_cmd_broadcast_connected ();
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state);
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data);
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
+
 void
-cli_path_strip_trailing_slashes (char *path);
+print_quota_list_header(int type);
+
+void
+print_quota_list_empty(char *path, int type);
+
+int
+gf_gsync_status_t_comparator(const void *p, const void *q);
 #endif /* __CLI_H__ */
diff --git a/cli/src/cli3_1-cops.c b/cli/src/cli3_1-cops.c
deleted file mode 100644
index 3003469cb8e..00000000000
--- a/cli/src/cli3_1-cops.c
+++ /dev/null
@@ -1,2431 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "cli.h"
-#include "compat-errno.h"
-#include "cli-cmd.h"
-#include <sys/uio.h>
-
-#include "cli1-xdr.h"
-#include "cli1.h"
-#include "protocol-common.h"
-#include "cli-mem-types.h"
-#include "compat.h"
-
-#include "glusterfs3.h"
-#include "portmap.h"
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-extern int      cli_op_ret;
-
-char *cli_volume_type[] = {"Distribute",
-                           "Stripe",
-                           "Replicate",
-                           "Distributed-Stripe",
-                           "Distributed-Replicate",
-};
-
-
-char *cli_volume_status[] = {"Created",
-                             "Started",
-                             "Stopped"
-};
-
-int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data);
-
-
-rpc_clnt_prog_t cli_handshake_prog = {
-        .progname  = "cli handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-};
-
-rpc_clnt_prog_t cli_pmap_prog = {
-        .progname   = "cli portmap",
-        .prognum    = GLUSTER_PMAP_PROGRAM,
-        .progver    = GLUSTER_PMAP_VERSION,
-};
-
-
-int
-gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
-{
-        gf1_cli_probe_rsp    rsp   = {0,};
-        int                   ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_probe_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to probe");
-	 if (!rsp.op_ret) {
-	 	switch (rsp.op_errno) {
-		 	case GF_PROBE_SUCCESS:
-		      		cli_out ("Probe successful");
-		      		break;
-	 	 	case GF_PROBE_LOCALHOST:
-		      		cli_out ("Probe on localhost not needed");
-		      		break;
-			case GF_PROBE_FRIEND:
-				cli_out ("Probe on host %s port %d already"
-					 " in peer list", rsp.hostname, rsp.port);
-				break;
-		 	default:
-		      		cli_out ("Probe returned with unknown errno %d",
-					rsp.op_errno);
-		      		break;
-	 	}
-	 }
-
-        if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                cli_out ("%s is already part of "
-                                         "another cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                cli_out ("Atleast one volume on %s conflicts "
-                                         "with existing volumes in the "
-                                         "cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_UNKNOWN_PEER:
-                                cli_out ("%s responded with 'unknown peer' error, "
-                                         "this could happen if %s doesn't have"
-                                         " localhost in its peer database",
-                                         rsp.hostname, rsp.hostname);
-                                break;
-                        case GF_PROBE_ADD_FAILED:
-                                cli_out ("Failed to add peer information "
-                                         "on %s" , rsp.hostname);
-                                break;
-
-                        default:
-                                cli_out ("Probe unsuccessful\nProbe returned "
-                                         "with unknown errno %d", rsp.op_errno);
-                                break;
-                }
-                gf_log ("glusterd",GF_LOG_ERROR,"Probe failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
-        }
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf1_cli_deprobe_rsp    rsp   = {0,};
-        int                   ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_deprobe_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to deprobe");
-        if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_DEPROBE_LOCALHOST:
-                                cli_out ("%s is localhost",
-                                         rsp.hostname);
-                                break;
-                        case GF_DEPROBE_NOT_FRIEND:
-                                cli_out ("%s is not part of cluster",
-                                         rsp.hostname);
-                                break;
-                        case GF_DEPROBE_BRICK_EXIST:
-                                cli_out ("Brick(s) with the peer %s exist in "
-                                         "cluster", rsp.hostname);
-                                break;
-                        default:
-                                cli_out ("Detach unsuccessful\nDetach returned "
-                                         "with unknown errno %d",
-                                         rsp.op_errno);
-                                break;
-                }
-                gf_log ("glusterd",GF_LOG_ERROR,"Detach failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
-        } else {
-                cli_out ("Detach successful");
-        }
-
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_peer_list_rsp      rsp   = {0,};
-        int                        ret   = 0;
-        dict_t                     *dict = NULL;
-        char                       *uuid_buf = NULL;
-        char                       *hostname_buf = NULL;
-        int32_t                    i = 1;
-        char                       key[256] = {0,};
-        char                       *state = NULL;
-        int32_t                    port = 0;
-        int32_t                    connected = 0;
-        char                       *connected_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_peer_list_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to list: %d",
-                rsp.op_ret);
-
-        ret = rsp.op_ret;
-
-        if (!rsp.op_ret) {
-
-                if (!rsp.friends.friends_len) {
-                        cli_out ("No peers present");
-                        ret = 0;
-                        goto out;
-                }
-
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.friends.friends_val,
-                                        rsp.friends.friends_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "count", &count);
-
-                if (ret) {
-                        goto out;
-                }
-
-                cli_out ("Number of Peers: %d", count);
-
-                while ( i <= count) {
-                        snprintf (key, 256, "friend%d.uuid", i);
-                        ret = dict_get_str (dict, key, &uuid_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.hostname", i);
-                        ret = dict_get_str (dict, key, &hostname_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.connected", i);
-                        ret = dict_get_int32 (dict, key, &connected);
-                        if (ret)
-                                goto out;
-                        if (connected)
-                                connected_str = "Connected";
-                        else
-                                connected_str = "Disconnected";
-
-                        snprintf (key, 256, "friend%d.port", i);
-                        ret = dict_get_int32 (dict, key, &port);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.state", i);
-                        ret = dict_get_str (dict, key, &state);
-                        if (ret)
-                                goto out;
-
-                        if (!port) {
-                                cli_out ("\nHostname: %s\nUuid: %s\nState: %s "
-                                         "(%s)",
-                                         hostname_buf, uuid_buf, state,
-                                         connected_str);
-                        } else {
-                                cli_out ("\nHostname: %s\nPort: %d\nUuid: %s\n"
-                                         "State: %s (%s)", hostname_buf, port,
-                                         uuid_buf, state, connected_str);
-                        }
-                        i++;
-                }
-        } else {
-                ret = -1;
-                goto out;
-        }
-
-
-        ret = 0;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (ret)
-                cli_out ("Command Execution Failed");
-
-        if (dict)
-                dict_destroy (dict);
-
-        return ret;
-}
-
-void
-cli_out_options ( char *substr, char *optstr, char *valstr)
-{
-        char                    *ptr1 = NULL;
-        char                    *ptr2 = NULL;
-
-        ptr1 = substr;
-        ptr2 = optstr;
-
-        while (ptr1)
-        {
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
-                if (!ptr1)
-                        return;
-                if (!ptr2)
-                        return;
-        }
-
-        if (*ptr2 == '\0')
-                return;
-        cli_out ("%s: %s",ptr2 , valstr);
-}
-
-
-int
-gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_get_vol_rsp        rsp   = {0,};
-        int                        ret   = 0;
-        dict_t                     *dict = NULL;
-        char                       *volname = NULL;
-        int32_t                    i = 0;
-        char                       key[1024] = {0,};
-        int32_t                    status = 0;
-        int32_t                    type = 0;
-        int32_t                    brick_count = 0;
-        int32_t                    sub_count = 0;
-        int32_t                    vol_type = 0;
-        char                       *brick = NULL;
-        int32_t                    j = 1;
-        cli_local_t                *local = NULL;
-        int32_t                    transport = 0;
-        data_pair_t                *pairs = NULL;
-        char                       *ptr = NULL;
-        data_t                     *value = NULL;
-        int                        opt_count = 0;
-        int                        k = 0;
- 
-
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_get_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to get vol: %d",
-                rsp.op_ret);
-
-        if (!rsp.op_ret) {
-
-                if (!rsp.volumes.volumes_len) {
-                        cli_out ("No volumes present");
-                        ret = 0;
-                        goto out;
-                }
-
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.volumes.volumes_val,
-                                        rsp.volumes.volumes_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "count", &count);
-
-                if (ret) {
-                        goto out;
-                }
-
-                local = ((call_frame_t *)myframe)->local;
-                //cli_out ("Number of Volumes: %d", count);
-
-                if (!count && (local->u.get_vol.flags ==
-                                        GF_CLI_GET_NEXT_VOLUME)) {
-                        local->u.get_vol.volname = NULL;
-                        ret = 0;
-                        goto out;
-                } else if (!count && (local->u.get_vol.flags ==
-                                        GF_CLI_GET_VOLUME)) {
-                        cli_out ("Volume %s not present",
-                                  local->u.get_vol.volname);
-                        ret = 0;
-                        goto out;
-                }
-
-                while ( i < count) {
-                        cli_out ("");
-                        snprintf (key, 256, "volume%d.name", i);
-                        ret = dict_get_str (dict, key, &volname);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "volume%d.type", i);
-                        ret = dict_get_int32 (dict, key, &type);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "volume%d.status", i);
-                        ret = dict_get_int32 (dict, key, &status);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "volume%d.brick_count", i);
-                        ret = dict_get_int32 (dict, key, &brick_count);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "volume%d.sub_count", i);
-                        ret = dict_get_int32 (dict, key, &sub_count);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "volume%d.transport", i);
-                        ret = dict_get_int32 (dict, key, &transport);
-                        if (ret)
-                                goto out;
-
-                        vol_type = type;
-
-                        // Stripe
-                        if ((type == 1) && (sub_count < brick_count))
-                                vol_type = 3;
-
-                        // Replicate
-                        if ((type == 2) && (sub_count < brick_count))
-                                vol_type = 4;
-
-                        cli_out ("Volume Name: %s", volname);
-                        cli_out ("Type: %s", cli_volume_type[vol_type]);
-                        cli_out ("Status: %s", cli_volume_status[status], brick_count);
-                        if ((sub_count > 1) && (brick_count > sub_count))
-                                cli_out ("Number of Bricks: %d x %d = %d",
-                                         brick_count / sub_count, sub_count,
-                                         brick_count);
-                        else
-                                cli_out ("Number of Bricks: %d", brick_count);
-
-                        cli_out ("Transport-type: %s", ((transport == 0)?
-                                 "tcp" : "rdma"));
-                        j = 1;
-
-
-                        GF_FREE (local->u.get_vol.volname);
-                        local->u.get_vol.volname = gf_strdup (volname);
-
-                        if (brick_count)
-                                cli_out ("Bricks:");
-
-                        while ( j <= brick_count) {
-                                snprintf (key, 1024, "volume%d.brick%d",
-                                          i, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-                                cli_out ("Brick%d: %s", j, brick);
-                                j++;
-                        }
-                        pairs = dict->members_list;
-                        if (!pairs) {
-                                ret = -1;
-                                goto out;
-                        }
-                        
-                        snprintf (key, 256, "volume%d.opt_count",i);
-                        ret = dict_get_int32 (dict, key, &opt_count);
-                        if (ret)
-                            goto out;
-
-                        if (!opt_count)
-                            goto out;
-
-                        cli_out ("Options Reconfigured:");
-                        k = 0;
-                        while ( k < opt_count) {
-
-                                snprintf (key, 256, "volume%d.option.",i);
-                                while (pairs) {
-                                        ptr = strstr (pairs->key, "option.");
-                                        if (ptr) {
-                                                value = pairs->value;
-                                                if (!value) {
-                                                        ret = -1;
-                                                        goto out;
-                                                }
-                                                cli_out_options (key, pairs->key,
-                                                                 value->data);
-                                        }
-                                        pairs = pairs->next;
-                                }
-                                k++;
-                       }
-
-                       i++;
-                }
-
-
-        } else {
-                ret = -1;
-                goto out;
-        }
-
-
-        ret = 0;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (ret)
-                cli_out ("Command Execution Failed");
-
-        if (dict)
-                dict_destroy (dict);
-
-        gf_log ("", GF_LOG_NORMAL, "Returning: %d", ret);
-        return ret;
-}
-
-int
-gf_cli3_1_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_create_vol_rsp  rsp   = {0,};
-        int                     ret   = 0;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        dict_t                  *dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        local = ((call_frame_t *) (myframe))->local;
-        ((call_frame_t *) (myframe))->local = NULL;
-
-        ret = gf_xdr_to_cli_create_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        dict = local->u.create_vol.dict;
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to create volume");
-	if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-	        cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Creation of volume %s has been %s", volname,
-                                (rsp.op_ret) ? "unsuccessful":
-                                "successful. Please start the volume to "
-                                "access data.");
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.volname)
-                free (rsp.volname);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        return ret;
-}
-
-int
-gf_cli3_1_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_delete_vol_rsp  rsp   = {0,};
-        int                     ret   = 0;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_delete_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        frame = myframe;
-        local = frame->local;
-        frame->local = NULL;
-
-        if (local)
-                volname = local->u.delete_vol.volname;
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to delete volume");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out (rsp.op_errstr);
-        else
-                cli_out ("Deleting volume %s has been %s", volname,
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.volname)
-                free (rsp.volname);
-        gf_log ("", GF_LOG_NORMAL, "Returning with %d", ret);
-        return ret;
-}
-
-int
-gf_cli3_1_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_start_vol_rsp   rsp   = {0,};
-        int                     ret   = 0;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_start_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        frame = myframe;
-
-        if (frame) {
-                local = frame->local;
-                frame->local = NULL;
-        }
-
-        if (local)
-                volname = local->u.start_vol.volname;
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to start volume");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Starting volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.volname)
-                free (rsp.volname);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        return ret;
-}
-
-int
-gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_stop_vol_rsp  rsp   = {0,};
-        int                   ret   = 0;
-        cli_local_t           *local = NULL;
-        char                  *volname = NULL;
-        call_frame_t          *frame = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_stop_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        frame = myframe;
-
-        if (frame)
-                local = frame->local;
-
-        if (local)
-                volname = local->u.start_vol.volname;
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to stop volume");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out (rsp.op_errstr);
-        else
-                cli_out ("Stopping volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        if (rsp.volname)
-                free (rsp.volname);
-        return ret;
-}
-
-int
-gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_defrag_vol_rsp  rsp     = {0,};
-        cli_local_t            *local   = NULL;
-        char                   *volname = NULL;
-        call_frame_t           *frame   = NULL;
-        int                     cmd     = 0;
-        int                     ret     = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_defrag_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        frame = myframe;
-
-        if (frame)
-                local = frame->local;
-
-        if (local) {
-                volname = local->u.defrag_vol.volname;
-                cmd = local->u.defrag_vol.cmd;
-        }
-        if (cmd == GF_DEFRAG_CMD_START) {
-                cli_out ("starting rebalance on volume %s has been %s", volname,
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-                if (rsp.op_ret && rsp.op_errno == EEXIST)
-                       cli_out ("Rebalance already started on volume %s", 
-                                volname);
-        }
-        if (cmd == GF_DEFRAG_CMD_STOP) {
-                if (rsp.op_ret == -1)
-                        cli_out ("rebalance volume %s stop failed", volname);
-                else
-                        cli_out ("stopped rebalance process of volume %s \n"
-                                 "(after rebalancing %"PRId64" files totaling "
-                                 "%"PRId64" bytes)", volname, rsp.files, rsp.size);
-        }
-        if (cmd == GF_DEFRAG_CMD_STATUS) {
-                if (rsp.op_ret == -1)
-                        cli_out ("failed to get the status of rebalance process");
-                else {
-                        char *status = "unknown";
-                        if (rsp.op_errno == 0)
-                                status = "not started";
-                        if (rsp.op_errno == 1)
-                                status = "step 1: layout fix in progress";
-                        if (rsp.op_errno == 2)
-                                status = "step 2: data migration in progress";
-                        if (rsp.op_errno == 3)
-                                status = "stopped";
-                        if (rsp.op_errno == 4)
-                                status = "completed";
-                        if (rsp.op_errno == 5)
-                                status = "failed";
-
-                        if (rsp.files && (rsp.op_errno == 1)) {
-                                cli_out ("rebalance %s: fixed layout %"PRId64,
-                                         status, rsp.files);
-                        } else if (rsp.files) {
-                                cli_out ("rebalance %s: rebalanced %"PRId64
-                                         " files of size %"PRId64" (total files"
-                                         " scanned %"PRId64")", status,
-                                         rsp.files, rsp.size, rsp.lookedup_files);
-                        } else {
-                                cli_out ("rebalance %s", status);
-                        }
-                }
-        }
-
-        if (volname)
-                GF_FREE (volname);
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_rename_vol_rsp  rsp   = {0,};
-        int                     ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_rename_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to probe");
-        cli_out ("Rename volume %s", (rsp.op_ret) ? "unsuccessful":
-                                        "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_reset_vol_rsp  rsp   = {0,};
-        int                  ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_reset_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to reset");
-
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("reset volume %s", (rsp.op_ret) ? "unsuccessful":
-                                "successful");
-
-        ret = rsp.op_ret;
-
-out:
-                cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-void
-_cli_out_options (dict_t *this, char *key, data_t *value, void *count)
-{
-
-        (*((int *) count))++;
-        cli_out ("%s  -  %s", key, value->data);
-}
-
-int
-gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_set_vol_rsp  rsp   = {0,};
-        int                  ret   = 0;
-        dict_t              *dict  = NULL;
-        int                  dict_count = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_set_vol_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        if (rsp.op_ret == 1) { // if the command was volume set <vol> history
-
-                if (!rsp.dict.dict_len) {
-                        cli_out ("No volumes present");
-                        ret = 0;
-                        goto out;
-                }
-
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                        goto out;
-                }
-                cli_out ("Options:");
-                dict_foreach (dict, _cli_out_options, &dict_count);
-                if (!dict_count)
-                        cli_out ("No Options Reconfigured!!");
-                goto out;
-        }
-
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to set");
-
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Set volume %s", (rsp.op_ret) ? "unsuccessful":
-                                                         "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_add_brick_rsp       rsp   = {0,};
-        int                         ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_add_brick_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to add brick");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Add Brick %s", (rsp.op_ret) ? "unsuccessful":
-                                                        "successful");
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (rsp.volname)
-                free (rsp.volname);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        return ret;
-}
-
-
-int
-gf_cli3_1_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_remove_brick_rsp        rsp   = {0,};
-        int                             ret   = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_remove_brick_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to remove brick");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Remove Brick %s", (rsp.op_ret) ? "unsuccessful":
-                                                           "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        if (rsp.volname)
-                free (rsp.volname);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        return ret;
-}
-
-
-
-int
-gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_replace_brick_rsp        rsp              = {0,};
-        int                              ret              = 0;
-        cli_local_t                     *local            = NULL;
-        call_frame_t                    *frame            = NULL;
-        dict_t                          *dict             = NULL;
-        char                            *src_brick        = NULL;
-        char                            *dst_brick        = NULL;
-        char                            *status_reply     = NULL;
-        gf1_cli_replace_op               replace_op       = 0;
-        char                            *rb_operation_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = (call_frame_t *) myframe;
-
-        ret = gf_xdr_to_cli_replace_brick_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        local = frame->local;
-        GF_ASSERT (local);
-        dict = local->u.replace_brick.dict;
-
-        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        switch (replace_op) {
-        case GF_REPLACE_OP_START:
-                if (rsp.op_ret)
-                        rb_operation_str = "replace-brick failed to start";
-                else
-                        rb_operation_str = "replace-brick started successfully";
-                break;
-
-        case GF_REPLACE_OP_STATUS:
-
-                status_reply = rsp.status;
-                if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick status unknown";
-                else
-                        rb_operation_str = status_reply;
-
-                break;
-
-        case GF_REPLACE_OP_PAUSE:
-                if (rsp.op_ret)
-                        rb_operation_str = "replace-brick pause failed";
-                else
-                        rb_operation_str = "replace-brick paused successfully";
-                break;
-
-        case GF_REPLACE_OP_ABORT:
-                if (rsp.op_ret)
-                        rb_operation_str = "replace-brick abort failed";
-                else
-                        rb_operation_str = "replace-brick aborted successfully";
-                break;
-
-        case GF_REPLACE_OP_COMMIT:
-        case GF_REPLACE_OP_COMMIT_FORCE:
-                ret = dict_get_str (dict, "src-brick", &src_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dict_get on src-brick failed");
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, "dst-brick", &dst_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dict_get on dst-brick failed");
-                        goto out;
-                }
-
-
-                if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick commit failed";
-                else
-                        rb_operation_str = "replace-brick commit successful";
-
-                break;
-
-        default:
-                gf_log ("", GF_LOG_DEBUG,
-                        "Unknown operation");
-                break;
-        }
-
-        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = rsp.op_errstr;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to replace brick");
-        cli_out ("%s",
-                 rb_operation_str ? rb_operation_str : "Unknown operation");
-
-        ret = rsp.op_ret;
-
-out:
-        if (local) {
-                dict_unref (local->u.replace_brick.dict);
-                GF_FREE (local->u.replace_brick.volname);
-                cli_local_wipe (local);
-        }
-
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-static int
-gf_cli3_1_log_filename_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
-{
-        gf1_cli_log_filename_rsp        rsp   = {0,};
-        int                             ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_log_filename_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log filename");
-
-        if (rsp.op_ret && strcmp (rsp.errstr, ""))
-                cli_out (rsp.errstr);
-        else
-                cli_out ("log filename : %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-static int
-gf_cli3_1_log_locate_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        gf1_cli_log_locate_rsp rsp   = {0,};
-        int                    ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_log_locate_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log locate");
-        cli_out ("log file location: %s", rsp.path);
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-static int
-gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        gf1_cli_log_rotate_rsp rsp   = {0,};
-        int                    ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_log_rotate_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log rotate");
-
-        if (rsp.op_ret && strcmp (rsp.errstr, ""))
-                cli_out (rsp.errstr);
-        else
-                cli_out ("log rotate %s", (rsp.op_ret) ? "unsuccessful":
-                                                         "successful");
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-static int
-gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
-{
-        gf1_cli_sync_volume_rsp        rsp   = {0,};
-        int                            ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_sync_volume_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out (rsp.op_errstr);
-        else
-                cli_out ("volume sync: %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_getspec_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf_getspec_rsp          rsp   = {0,};
-        int                     ret   = 0;
-        char                   *spec  = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_getspec_rsp (*iov, &rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to getspec");
-
-        spec = GF_MALLOC (rsp.op_ret + 1, cli_mt_char);
-        if (!spec) {
-                gf_log("", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-        memcpy (spec, rsp.spec, rsp.op_ret);
-        spec[rsp.op_ret] = '\0';
-        cli_out ("%s", spec);
-        GF_FREE (spec);
-
-        ret = 0;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int
-gf_cli3_1_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
-{
-        pmap_port_by_brick_rsp rsp = {0,};
-        int                     ret   = 0;
-        char                   *spec  = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_pmap_port_by_brick_rsp (*iov, &rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_NORMAL, "Received resp to pmap b2p");
-
-        cli_out ("%d", rsp.port);
-        GF_FREE (spec);
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-
-int32_t
-gf_cli3_1_probe (call_frame_t *frame, xlator_t *this,
-                 void *data)
-{
-        gf1_cli_probe_req  req      = {0,};
-        int                ret      = 0;
-        dict_t            *dict     = NULL;
-        char              *hostname = NULL;
-        int                port     = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
-
-        req.hostname = hostname;
-        req.port     = port;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_PROBE, NULL, gf_xdr_from_cli_probe_req,
-                              this, gf_cli3_1_probe_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-gf_cli3_1_deprobe (call_frame_t *frame, xlator_t *this,
-                   void *data)
-{
-        gf1_cli_deprobe_req  req      = {0,};
-        int                  ret      = 0;
-        dict_t              *dict     = NULL;
-        char                *hostname = NULL;
-        int                  port     = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
-
-        req.hostname = hostname;
-        req.port     = port;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_DEPROBE, NULL,
-                              gf_xdr_from_cli_deprobe_req,
-                              this, gf_cli3_1_deprobe_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-gf_cli3_1_list_friends (call_frame_t *frame, xlator_t *this,
-                        void *data)
-{
-        gf1_cli_peer_list_req   req = {0,};
-        int                     ret = 0;
-
-        if (!frame || !this) {
-                ret = -1;
-                goto out;
-        }
-
-        req.flags = GF_CLI_LIST_ALL;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_LIST_FRIENDS, NULL,
-                              gf_xdr_from_cli_peer_list_req,
-                              this, gf_cli3_1_list_friends_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
-                           void *data)
-{
-
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        cli_local_t                     *local = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-
-        ret = gf_cli3_1_get_volume (frame, this, data);
-
-        local = frame->local;
-
-        if (!local || !local->u.get_vol.volname) {
-                cli_out ("No volumes present");
-                goto out;
-        }
-
-        ctx->volname = local->u.get_vol.volname;
-
-        while (ctx->volname) {
-                ret = gf_cli3_1_get_volume (frame, this, ctx);
-                if (ret)
-                        goto out;
-                ctx->volname = local->u.get_vol.volname;
-        }
-
-out:
-        return ret;
-}
-
-int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf1_cli_get_vol_req             req = {0,};
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        dict_t                          *dict = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-        req.flags = ctx->flags;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (ctx->volname) {
-                ret = dict_set_str (dict, "volname", ctx->volname);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_GET_VOLUME, NULL,
-                              gf_xdr_from_cli_get_vol_req,
-                              this, gf_cli3_1_get_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-
-int32_t
-gf_cli3_1_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_create_vol_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_ref ((dict_t *)data);
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "type", (int32_t *)&req.type);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "count", &req.count);
-        if (ret)
-                goto out;
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.bricks.bricks_val,
-                                           (size_t *)&req.bricks.bricks_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        local = cli_local_get ();
-
-        if (local) {
-                local->u.create_vol.dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_CREATE_VOLUME, NULL,
-                              gf_xdr_from_cli_create_vol_req,
-                              this, gf_cli3_1_create_volume_cbk);
-
-
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        if (dict)
-                dict_unref (dict);
-
-        if (req.bricks.bricks_val) {
-                GF_FREE (req.bricks.bricks_val);
-        }
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_delete_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_delete_vol_req  req = {0,};
-        int                     ret = 0;
-        cli_local_t             *local = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        local = cli_local_get ();
-
-        if (local) {
-                local->u.delete_vol.volname = data;
-                frame->local = local;
-        }
-
-        req.volname = data;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_DELETE_VOLUME, NULL,
-                              gf_xdr_from_cli_delete_vol_req,
-                              this, gf_cli3_1_delete_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_start_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_start_vol_req   *req = NULL;
-        int                     ret = 0;
-        cli_local_t             *local = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        req = data;
-        local = cli_local_get ();
-
-        if (local) {
-                local->u.start_vol.volname = req->volname;
-                local->u.start_vol.flags = req->flags;
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_START_VOLUME, NULL,
-                              gf_xdr_from_cli_start_vol_req,
-                              this, gf_cli3_1_start_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_stop_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_stop_vol_req   req = {0,};
-        int                    ret = 0;
-        cli_local_t            *local = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        req = *((gf1_cli_stop_vol_req*)data);
-        local = cli_local_get ();
-
-        if (local) {
-                local->u.stop_vol.volname = req.volname;
-                local->u.stop_vol.flags = req.flags;
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_STOP_VOLUME, NULL,
-                              gf_xdr_from_cli_stop_vol_req,
-                              this, gf_cli3_1_stop_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_defrag_vol_req  req     = {0,};
-        int                     ret     = 0;
-        cli_local_t            *local   = NULL;
-        char                   *volname = NULL;
-        char                   *cmd_str = NULL;
-        dict_t                 *dict    = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log ("", GF_LOG_DEBUG, "error");
-
-        ret = dict_get_str (dict, "command", &cmd_str);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "error");
-                goto out;
-        }
-
-        if (strncasecmp (cmd_str, "start", 6) == 0) {
-                req.cmd = GF_DEFRAG_CMD_START;
-        } else if (strncasecmp (cmd_str, "stop", 5) == 0) {
-                req.cmd = GF_DEFRAG_CMD_STOP;
-        } else if (strncasecmp (cmd_str, "status", 7) == 0) {
-                req.cmd = GF_DEFRAG_CMD_STATUS;
-        }
-
-
-        local = cli_local_get ();
-
-        if (local) {
-                local->u.defrag_vol.volname = gf_strdup (volname);
-                local->u.defrag_vol.cmd = req.cmd;
-                frame->local = local;
-        }
-
-        req.volname = volname;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_DEFRAG_VOLUME, NULL,
-                              gf_xdr_from_cli_defrag_vol_req,
-                              this, gf_cli3_1_defrag_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_rename_vol_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "old-volname", &req.old_volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "new-volname", &req.new_volname);
-
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_RENAME_VOLUME, NULL,
-                              gf_xdr_from_cli_rename_vol_req,
-                              this, gf_cli3_1_rename_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_reset_volume (call_frame_t *frame, xlator_t *this, 
-                        void *data)
-{
-        gf1_cli_reset_vol_req     req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                               GD_MGMT_CLI_RESET_VOLUME, NULL,
-                               gf_xdr_from_cli_reset_vol_req,
-                               this, gf_cli3_1_reset_volume_cbk);
-
-out:
-                gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_set_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_set_vol_req     req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_SET_VOLUME, NULL,
-                              gf_xdr_from_cli_set_vol_req,
-                              this, gf_cli3_1_set_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_add_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_add_brick_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "count", &req.count);
-        if (ret)
-                goto out;
-
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.bricks.bricks_val,
-                                           (size_t *)&req.bricks.bricks_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_ADD_BRICK, NULL,
-                              gf_xdr_from_cli_add_brick_req,
-                              this, gf_cli3_1_add_brick_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        if (req.bricks.bricks_val) {
-                GF_FREE (req.bricks.bricks_val);
-        }
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_remove_brick_req  req = {0,};
-        int                       ret = 0;
-        dict_t                    *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "count", &req.count);
-
-        if (ret)
-                goto out;
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.bricks.bricks_val,
-                                           (size_t *)&req.bricks.bricks_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_REMOVE_BRICK, NULL,
-                              gf_xdr_from_cli_remove_brick_req,
-                              this, gf_cli3_1_remove_brick_cbk);
-
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        if (req.bricks.bricks_val) {
-                GF_FREE (req.bricks.bricks_val);
-        }
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf1_cli_replace_brick_req   req        = {0,};
-        int                         ret        = 0;
-        cli_local_t                *local      = NULL;
-        dict_t                     *dict       = NULL;
-        char                       *src_brick  = NULL;
-        char                       *dst_brick  = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-	dict = data;
-
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto out;
-        }
-
-        local->u.replace_brick.dict = dict_ref (dict);
-        frame->local                = local;
-
-        ret = dict_get_int32 (dict, "operation", (int32_t *)&req.op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &req.volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on volname failed");
-                goto out;
-        }
-
-        local->u.replace_brick.volname = gf_strdup (req.volname);
-        if (!local->u.replace_brick.volname) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on src-brick failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on dst-brick failed");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Recevied command replace-brick %s with "
-                "%s with operation=%d", src_brick,
-                dst_brick, req.op);
-
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.bricks.bricks_val,
-                                           (size_t *)&req.bricks.bricks_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_REPLACE_BRICK, NULL,
-                              gf_xdr_from_cli_replace_brick_req,
-                              this, gf_cli3_1_replace_brick_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        if (req.bricks.bricks_val) {
-                GF_FREE (req.bricks.bricks_val);
-        }
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_log_filename (call_frame_t *frame, xlator_t *this,
-                        void *data)
-{
-        gf1_cli_log_filename_req  req = {0,};
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                req.brick = "";
-
-        ret = dict_get_str (dict, "path", &req.path);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_LOG_FILENAME, NULL,
-                              gf_xdr_from_cli_log_filename_req,
-                              this, gf_cli3_1_log_filename_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-
-int32_t
-gf_cli3_1_log_locate (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf1_cli_log_locate_req  req = {0,};
-        int                     ret = 0;
-        dict_t                 *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                req.brick = "";
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_LOG_LOCATE, NULL,
-                              gf_xdr_from_cli_log_locate_req,
-                              this, gf_cli3_1_log_locate_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_log_rotate (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf1_cli_log_locate_req  req = {0,};
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &req.volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                req.brick = "";
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_LOG_ROTATE, NULL,
-                              gf_xdr_from_cli_log_rotate_req,
-                              this, gf_cli3_1_log_rotate_cbk);
-
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_sync_volume (call_frame_t *frame, xlator_t *this,
-                       void *data)
-{
-        int               ret = 0;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = cli_cmd_submit ((gf1_cli_sync_volume_req*)data, frame,
-                              cli_rpc_prog, GD_MGMT_CLI_SYNC_VOLUME,
-                              NULL, gf_xdr_from_cli_sync_volume_req,
-                              this, gf_cli3_1_sync_volume_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_getspec (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_getspec_req          req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volid", &req.key);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_submit (&req, frame, &cli_handshake_prog,
-                              GF_HNDSK_GETSPEC, NULL,
-                              xdr_from_getspec_req,
-                              this, gf_cli3_1_getspec_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-gf_cli3_1_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
-{
-        pmap_port_by_brick_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_submit (&req, frame, &cli_pmap_prog,
-                              GF_PMAP_PORTBYBRICK, NULL,
-                              xdr_from_pmap_port_by_brick_req,
-                              this, gf_cli3_1_pmap_b2p_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-static int
-gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf1_cli_fsm_log_rsp        rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                     *dict = NULL;
-        int                        tr_count = 0;
-        char                       key[256] = {0};
-        int                        i = 0;
-        char                       *old_state = NULL;
-        char                       *new_state = NULL;
-        char                       *event = NULL;
-        char                       *time = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = gf_xdr_to_cli_fsm_log_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        cli_out (rsp.op_errstr);
-                } else if (rsp.op_ret) {
-                        cli_out ("fsm log unsuccessful");
-                }
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.fsm_log.fsm_log_val,
-                                rsp.fsm_log.fsm_log_len,
-                                &dict);
-
-        if (ret) {
-                cli_out ("bad response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &tr_count);
-        if (tr_count)
-                cli_out("number of transitions: %d", tr_count);
-        else
-                cli_out("No transitions");
-        for (i = 0; i < tr_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-old-state", i);
-                ret = dict_get_str (dict, key, &old_state);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-event", i);
-                ret = dict_get_str (dict, key, &event);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-new-state", i);
-                ret = dict_get_str (dict, key, &new_state);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-time", i);
-                ret = dict_get_str (dict, key, &time);
-                if (ret)
-                        goto out;
-                cli_out ("Old State: [%s]\n"
-                         "New State: [%s]\n"
-                         "Event    : [%s]\n"
-                         "timestamp: [%s]\n", old_state, new_state, event, time);
-        }
-
-        ret = rsp.op_ret;
-
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int32_t
-gf_cli3_1_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
-{
-        int                        ret = -1;
-        gf1_cli_fsm_log_req        req = {0,};
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
-
-        if (!frame || !this || !data)
-                goto out;
-        req.name = data;
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GD_MGMT_CLI_FSM_LOG, NULL,
-                              gf_xdr_from_cli_fsm_log_req,
-                              this, gf_cli3_1_fsm_log_cbk);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-struct rpc_clnt_procedure gluster3_1_cli_actors[GF1_CLI_MAXVALUE] = {
-        [GF1_CLI_NULL]        = {"NULL", NULL },
-        [GF1_CLI_PROBE]  = { "PROBE_QUERY",  gf_cli3_1_probe},
-        [GF1_CLI_DEPROBE]  = { "DEPROBE_QUERY",  gf_cli3_1_deprobe},
-        [GF1_CLI_LIST_FRIENDS]  = { "LIST_FRIENDS",  gf_cli3_1_list_friends},
-        [GF1_CLI_CREATE_VOLUME] = {"CREATE_VOLUME", gf_cli3_1_create_volume},
-        [GF1_CLI_DELETE_VOLUME] = {"DELETE_VOLUME", gf_cli3_1_delete_volume},
-        [GF1_CLI_START_VOLUME] = {"START_VOLUME", gf_cli3_1_start_volume},
-        [GF1_CLI_STOP_VOLUME] = {"STOP_VOLUME", gf_cli3_1_stop_volume},
-        [GF1_CLI_RENAME_VOLUME] = {"RENAME_VOLUME", gf_cli3_1_rename_volume},
-        [GF1_CLI_DEFRAG_VOLUME] = {"DEFRAG_VOLUME", gf_cli3_1_defrag_volume},
-        [GF1_CLI_GET_VOLUME] = {"GET_VOLUME", gf_cli3_1_get_volume},
-        [GF1_CLI_GET_NEXT_VOLUME] = {"GET_NEXT_VOLUME", gf_cli3_1_get_next_volume},
-        [GF1_CLI_SET_VOLUME] = {"SET_VOLUME", gf_cli3_1_set_volume},
-        [GF1_CLI_ADD_BRICK] = {"ADD_BRICK", gf_cli3_1_add_brick},
-        [GF1_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", gf_cli3_1_remove_brick},
-        [GF1_CLI_REPLACE_BRICK] = {"REPLACE_BRICK", gf_cli3_1_replace_brick},
-        [GF1_CLI_LOG_FILENAME] = {"LOG FILENAME", gf_cli3_1_log_filename},
-        [GF1_CLI_LOG_LOCATE] = {"LOG LOCATE", gf_cli3_1_log_locate},
-        [GF1_CLI_LOG_ROTATE] = {"LOG ROTATE", gf_cli3_1_log_rotate},
-        [GF1_CLI_GETSPEC] = {"GETSPEC", gf_cli3_1_getspec},
-        [GF1_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli3_1_pmap_b2p},
-        [GF1_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", gf_cli3_1_sync_volume},
-        [GF1_CLI_RESET_VOLUME] = {"RESET_VOLUME", gf_cli3_1_reset_volume},
-        [GF1_CLI_FSM_LOG] = {"FSM_LOG", gf_cli3_1_fsm_log}
-};
-
-struct rpc_clnt_program cli3_1_prog = {
-        .progname = "CLI 3.1",
-        .prognum  = GLUSTER3_1_CLI_PROGRAM,
-        .progver  = GLUSTER3_1_CLI_VERSION,
-        .proctable  = gluster3_1_cli_actors,
-        .numproc  = GLUSTER3_1_CLI_PROCCNT,
-};
diff --git a/cli/src/input.c b/cli/src/input.c
index 5c38efe13e7..5ac1a20edb1 100644
--- a/cli/src/input.c
+++ b/cli/src/input.c
@@ -1,107 +1,93 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "cli.h"
 #include "cli-mem-types.h"
 
 #define CMDBUFSIZ 1024
 
 void *
-cli_batch (void *d)
+cli_batch(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = d;
+    state = d;
 
-        ret = cli_cmd_process (state, state->argc, state->argv);
+    ret = cli_cmd_process(state, state->argc, state->argv);
 
-        gf_log ("", GF_LOG_NORMAL, "Exiting with: %d", ret);
-        exit (ret);
+    gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+    exit(-ret);
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_input (void *d)
+cli_input(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
-        char              cmdbuf[CMDBUFSIZ];
-        char             *cmd = NULL;
-        size_t            len = 0;
-
-        state = d;
-
-        for (;;) {
-                printf ("%s", state->prompt);
-
-                cmd = fgets (cmdbuf, CMDBUFSIZ, stdin);
-                if (!cmd)
-                        break;
-                len = strlen(cmd);
-                if (len > 0 && cmd[len - 1] == '\n') //strip trailing \n
-                        cmd[len - 1] = '\0';
-                ret = cli_cmd_process_line (state, cmd);
-                if (ret == -1 && state->mode & GLUSTER_MODE_ERR_FATAL)
-                        break;
-        }
-
-        exit (ret);
-
-        return NULL;
+    struct cli_state *state = NULL;
+    int ret = 0;
+    char cmdbuf[CMDBUFSIZ];
+    char *cmd = NULL;
+    size_t len = 0;
+
+    state = d;
+
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        printf("%s", state->prompt);
+
+        cmd = fgets(cmdbuf, CMDBUFSIZ, stdin);
+        if (!cmd)
+            break;
+        len = strlen(cmd);
+        if (len > 0 && cmd[len - 1] == '\n')  // strip trailing \n
+            cmd[len - 1] = '\0';
+        ret = cli_cmd_process_line(state, cmd);
+        if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
+            break;
+    }
+
+    exit(-ret);
+
+    return NULL;
 }
 
-
 int
-cli_input_init (struct cli_state *state)
+cli_input_init(struct cli_state *state)
 {
-        int  ret = 0;
+    int ret = 0;
 
-        if (state->argc) {
-                ret = pthread_create (&state->input, NULL, cli_batch, state);
-                return ret;
-        }
+    if (state->argc) {
+        ret = pthread_create(&state->input, NULL, cli_batch, state);
+        return ret;
+    }
 
-        if (isatty (STDIN_FILENO)) {
-                state->prompt = "gluster> ";
+    if (isatty(STDIN_FILENO)) {
+        state->prompt = "gluster> ";
 
-                cli_rl_enable (state);
-        } else {
-                state->prompt = "";
-                state->mode = GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
-        }
+        cli_rl_enable(state);
+    } else {
+        state->prompt = "";
+        state->mode |= GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
+    }
 
-        if (!state->rl_enabled)
-                ret = pthread_create (&state->input, NULL, cli_input, state);
+    if (!state->rl_enabled)
+        ret = pthread_create(&state->input, NULL, cli_input, state);
 
-        return ret;
+    return ret;
 }
diff --git a/cli/src/registry.c b/cli/src/registry.c
index 166a148976f..85f7686ade1 100644
--- a/cli/src/registry.c
+++ b/cli/src/registry.c
@@ -1,27 +1,12 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #include <stdio.h>
 #include <string.h>
@@ -30,20 +15,18 @@
 #include "cli.h"
 #include "cli-cmd.h"
 
-
 static int
-__is_spc (int ch)
+__is_spc(int ch)
 {
-        if (ch == ' ')
-                return 1;
-        return 0;
+    if (ch == ' ')
+        return 1;
+    return 0;
 }
 
-
 static int
-__is_div (int ch)
+__is_div(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
         case ')':
         case '<':
@@ -53,337 +36,356 @@ __is_div (int ch)
         case '{':
         case '}':
         case '|':
-                return 1;
-        }
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 static int
-__is_word (const char *word)
+__is_word(const char *word)
 {
-        return (!__is_div (*word) && !__is_spc (*word));
+    return (!__is_div(*word) && !__is_spc(*word));
 }
 
-
 int
-counter_char (int ch)
+counter_char(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
-                return ')';
+            return ')';
         case '<':
-                return '>';
+            return '>';
         case '[':
-                return ']';
+            return ']';
         case '{':
-                return '}';
-        }
+            return '}';
+    }
 
-        return -1;
+    return -1;
 }
 
-
 const char *
-__is_template_balanced (const char *template)
+__is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
-        int   ch = 0;
-
-        trav = template;
-
-        while (*trav) {
-                ch = *trav;
-
-                switch (ch) {
-                case '<':
-                case '(':
-                case '[':
-                        trav = __is_template_balanced (trav+1);
-                        if (!trav)
-                                return NULL;
-                        if (*trav != counter_char (ch))
-                                return NULL;
-                        break;
-                case '>':
-                case ')':
-                case ']':
-                        return trav;
-                }
+    const char *trav = NULL;
+    int ch = 0;
+
+    trav = template;
+
+    while (*trav) {
+        ch = *trav;
 
-                trav++;
+        switch (ch) {
+            case '<':
+            case '(':
+            case '[':
+                trav = __is_template_balanced(trav + 1);
+                if (!trav)
+                    return NULL;
+                if (*trav != counter_char(ch))
+                    return NULL;
+                break;
+            case '>':
+            case ')':
+            case ']':
+                return trav;
         }
 
-        return trav;
-}
+        trav++;
+    }
 
+    return trav;
+}
 
 int
-is_template_balanced (const char *template)
+is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
+    const char *trav = NULL;
 
-        trav = __is_template_balanced (template);
-        if (!trav || *trav)
-                return -1;
+    trav = __is_template_balanced(template);
+    if (!trav || *trav)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
-
 int
-cli_cmd_token_count (const char *template)
+cli_cmd_token_count(const char *template)
 {
-        int         count = 0;
-        const char *trav = NULL;
-        int         is_alnum = 0;
-
-        for (trav = template; *trav; trav++) {
-                switch (*trav) {
-                case '<':
-                case '>':
-                case '(':
-                case ')':
-                case '[':
-                case ']':
-                case '{':
-                case '}':
-                case '|':
-                        count++;
-                        /* fall through */
-                case ' ':
-                        is_alnum = 0;
-                        break;
-                default:
-                        if (!is_alnum) {
-                                is_alnum = 1;
-                                count++;
-                        }
+    int count = 0;
+    const char *trav = NULL;
+    int is_alnum = 0;
+
+    for (trav = template; *trav; trav++) {
+        switch (*trav) {
+            case '<':
+            case '>':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '|':
+                count++;
+                /* fall through */
+            case ' ':
+                is_alnum = 0;
+                break;
+            default:
+                if (!is_alnum) {
+                    is_alnum = 1;
+                    count++;
                 }
         }
+    }
 
-        return count + 1;
+    return count + 1;
 }
 
-
 void
-cli_cmd_tokens_destroy (char **tokens)
+cli_cmd_tokens_destroy(char **tokens)
 {
-        char **tokenp = NULL;
+    char **tokenp = NULL;
 
-        if (!tokens)
-                return;
+    if (!tokens)
+        return;
 
-        tokenp = tokens;
-        while (*tokenp) {
-                free (*tokenp);
-                tokenp++;
-        }
+    tokenp = tokens;
+    while (*tokenp) {
+        free(*tokenp);
+        tokenp++;
+    }
 
-        free (tokens);
+    free(tokens);
 }
 
-
 int
-cli_cmd_tokens_fill (char **tokens, const char *template)
+cli_cmd_tokens_fill(char **tokens, const char *template)
 {
-        const char  *trav = NULL;
-        char       **tokenp = NULL;
-        char        *token = NULL;
-        int          ret = 0;
-        int          ch = 0;
+    const char *trav = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    int ret = 0;
+    int ch = 0;
 
-        tokenp = tokens;
+    tokenp = tokens;
 
-        for (trav = template; *trav; trav++) {
-                ch = *trav;
+    for (trav = template; *trav; trav++) {
+        ch = *trav;
 
-                if (__is_spc (ch))
-                        continue;
+        if (__is_spc(ch))
+            continue;
 
-                if (__is_div (ch)) {
-                        token = calloc (2, 1);
-                        if (!token)
-                                return -1;
-                        token[0] = ch;
+        if (__is_div(ch)) {
+            token = calloc(2, 1);
+            if (!token)
+                return -1;
+            token[0] = ch;
 
-                        *tokenp = token;
-                        tokenp++;
+            *tokenp = token;
+            tokenp++;
 
-                        continue;
-                }
+            continue;
+        }
 
-                token = strdup (trav);
-                *tokenp = token;
-                tokenp++;
+        token = strdup(trav);
+        *tokenp = token;
+        tokenp++;
 
-                for (token++; *token; token++) {
-                        if (__is_spc (*token) || __is_div (*token)) {
-                                *token = 0;
-                                break;
-                        }
-                        trav++;
-                }
+        for (token++; *token; token++) {
+            if (__is_spc(*token) || __is_div(*token)) {
+                *token = 0;
+                break;
+            }
+            trav++;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 char **
-cli_cmd_tokenize (const char *template)
+cli_cmd_tokenize(const char *template)
 {
-        char **tokens = NULL;
-        int    ret = 0;
-        int    count = 0;
+    char **tokens = NULL;
+    int ret = 0;
+    int count = 0;
 
-        ret = is_template_balanced (template);
-        if (ret)
-                return NULL;
+    ret = is_template_balanced(template);
+    if (ret)
+        return NULL;
 
-        count = cli_cmd_token_count (template);
-        if (count <= 0)
-                return NULL;
+    count = cli_cmd_token_count(template);
+    if (count <= 0)
+        return NULL;
 
-        tokens = calloc (count + 1, sizeof (char *));
-        if (!tokens)
-                return NULL;
+    tokens = calloc(count + 1, sizeof(char *));
+    if (!tokens)
+        return NULL;
 
-        ret = cli_cmd_tokens_fill (tokens, template);
-        if (ret)
-                goto err;
+    ret = cli_cmd_tokens_fill(tokens, template);
+    if (ret)
+        goto err;
 
-        return tokens;
+    return tokens;
 err:
-        cli_cmd_tokens_destroy (tokens);
-        return NULL;
+    cli_cmd_tokens_destroy(tokens);
+    return NULL;
 }
 
-
-struct cli_cmd_word *
-cli_cmd_nextword (struct cli_cmd_word *word, const char *token)
+void *
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel)
 {
-        struct cli_cmd_word    *next = NULL;
-        struct cli_cmd_word   **trav = NULL;
-        int                     ret = 0;
-
-        if (!word->nextwords)
-                return NULL;
-
-        for (trav = word->nextwords; (next = *trav); trav++) {
-                if (next->match) {
-//                        ret = next->match ();
-                } else {
-                        ret = strcmp (next->word, token);
-                }
+    void **wcon = NULL;
+    char *w = NULL;
+    unsigned mn = 0;
+    void *ret = NULL;
 
-                if (ret == 0)
-                        break;
-        }
+    if (!choices || !tok || !*tok)
+        return NULL;
 
-        return next;
+    for (wcon = choices; *wcon; wcon++) {
+        w = strtail((char *)sel(*wcon), tok);
+        if (!w)
+            /* no match */
+            continue;
+        if (!*w)
+            /* exact match */
+            return *wcon;
+
+        ret = *wcon;
+        mn++;
+    }
+
+#ifdef FORCE_MATCH_EXACT
+    return NULL;
+#else
+    return (mn == 1) ? ret : NULL;
+#endif
 }
 
+static const char *
+sel_cmd_word(void *wcon)
+{
+    return ((struct cli_cmd_word *)wcon)->word;
+}
 
 struct cli_cmd_word *
-cli_cmd_newword (struct cli_cmd_word *word, const char *token)
+cli_cmd_nextword(struct cli_cmd_word *word, const char *token)
 {
-        struct cli_cmd_word **nextwords = NULL;
-        struct cli_cmd_word  *nextword = NULL;
+    return (struct cli_cmd_word *)cli_getunamb(token, (void **)word->nextwords,
+                                               sel_cmd_word);
+}
 
-        nextwords = realloc (word->nextwords,
-                             (word->nextwords_cnt + 2) * sizeof (*nextwords));
-        if (!nextwords)
-                return NULL;
+struct cli_cmd_word *
+cli_cmd_newword(struct cli_cmd_word *word, const char *token)
+{
+    struct cli_cmd_word **nextwords = NULL;
+    struct cli_cmd_word *nextword = NULL;
 
-        word->nextwords = nextwords;
+    nextwords = realloc(word->nextwords,
+                        (word->nextwords_cnt + 2) * sizeof(*nextwords));
+    if (!nextwords)
+        return NULL;
 
-        nextword = calloc (1, sizeof (*nextword));
-        if (!nextword)
-                return NULL;
+    word->nextwords = nextwords;
 
-        nextword->word = strdup (token);
-        if (!nextword->word) {
-                free (nextword);
-                return NULL;
-        }
+    nextword = calloc(1, sizeof(*nextword));
+    if (!nextword)
+        return NULL;
 
-        nextword->tree = word->tree;
-        nextwords[word->nextwords_cnt++] = nextword;
-        nextwords[word->nextwords_cnt] = NULL;
+    nextword->word = strdup(token);
+    if (!nextword->word) {
+        free(nextword);
+        return NULL;
+    }
 
-        return nextword;
-}
+    nextword->tree = word->tree;
+    nextwords[word->nextwords_cnt++] = nextword;
+    nextwords[word->nextwords_cnt] = NULL;
 
+    return nextword;
+}
 
 int
-cli_cmd_ingest (struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
-                const char *desc, const char *pattern)
+cli_cmd_ingest(struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
+               const char *desc, const char *pattern)
 {
-        int                    ret = 0;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
+    int ret = 0;
+    char **tokenp = NULL;
+    char *token = NULL;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
 
-        word = &tree->root;
+    word = &tree->root;
 
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!__is_word (token))
-                        break;
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!__is_word(token))
+            break;
 
-                next = cli_cmd_nextword (word, token);
-                if (!next)
-                        next = cli_cmd_newword (word, token);
-
-                word = next;
-                if (!word)
-                        break;
-        }
+        next = cli_cmd_nextword(word, token);
+        if (!next)
+            next = cli_cmd_newword(word, token);
 
+        word = next;
         if (!word)
-                return -1;
+            break;
+    }
 
-        if (word->cbkfn) {
-                /* warning - command already registered */
-        }
+    if (!word)
+        return -1;
 
-        word->cbkfn = cbkfn;
-        word->desc  = desc;
-        word->pattern = pattern;
+    if (word->cbkfn) {
+        /* warning - command already registered */
+    }
 
-        /* end of static strings in command template */
+    word->cbkfn = cbkfn;
+    word->desc = desc;
+    word->pattern = pattern;
 
-        /* TODO: autocompletion beyond this point is just "nice to have" */
+    /* end of static strings in command template */
 
-        return ret;
-}
+    /* TODO: autocompletion beyond this point is just "nice to have" */
 
+    return ret;
+}
 
 int
-cli_cmd_register (struct cli_cmd_tree *tree, const char *template,
-                  cli_cmd_cbk_t cbk, const char *desc)
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd)
 {
-        char **tokens = NULL;
-        int    ret = 0;
+    char **tokens = NULL;
+    int ret = 0;
 
-        if (!template)
-                return -1;
+    GF_ASSERT(cmd);
 
-        tokens = cli_cmd_tokenize (template);
-        if (!tokens)
-                return -1;
+    if (cmd->reg_cbk)
+        cmd->reg_cbk(cmd);
 
-        ret = cli_cmd_ingest (tree, tokens, cbk, desc, template);
-        if (ret)
-                goto err;
+    if (cmd->disable) {
+        ret = 0;
+        goto out;
+    }
 
-        return 0;
-err:
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    tokens = cli_cmd_tokenize(cmd->pattern);
+    if (!tokens) {
+        ret = -1;
+        goto out;
+    }
 
-        return ret;
-}
+    ret = cli_cmd_ingest(tree, tokens, cmd->cbk, cmd->desc, cmd->pattern);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
+out:
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
diff --git a/commit.sh b/commit.sh
deleted file mode 100755
index 9cc55c52e8d..00000000000
--- a/commit.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-export EDITOR="emacs"
-git commit -a -e "$@"
diff --git a/configure.ac b/configure.ac
index fedde7582cd..e2d6fd66cec 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,135 +1,455 @@
-dnl Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-dnl This file is part of GlusterFS.
+dnl  Copyright (c) 2006-2016 Red Hat, Inc. <http://www.redhat.com>
+dnl  This file is part of GlusterFS.
 dnl
-dnl GlusterFS is free software; you can redistribute it and/or modify
-dnl it under the terms of the GNU Affero General Public License as published by
-dnl the Free Software Foundation; either version 3 of the License, or
-dnl (at your option) any later version.
-dnl
-dnl GlusterFS is distributed in the hope that it will be useful,
-dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
-dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-dnl GNU Affero General Public License for more details.
-dnl
-dnl You should have received a copy of the GNU Affero General Public License
-dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
+dnl  This file is licensed to you under your choice of the GNU Lesser
+dnl  General Public License, version 3 or any later version (LGPLv3 or
+dnl  later), or the GNU General Public License, version 2 (GPLv2), in all
+dnl  cases as published by the Free Software Foundation.
+
+AC_INIT([glusterfs],
+        [m4_esyscmd([build-aux/pkg-version --version])],
+        [gluster-users@gluster.org],,[https://github.com/gluster/glusterfs.git])
+
+AC_SUBST([PACKAGE_RELEASE],
+         [m4_esyscmd([build-aux/pkg-version --release])])
 
-AC_INIT([glusterfs],[3.1.0git],[gluster-users@gluster.org])
+AM_INIT_AUTOMAKE([tar-pax foreign])
 
-AM_INIT_AUTOMAKE
+# Removes warnings when using automake 1.14 around (...but option 'subdir-objects' is disabled )
+#but libglusterfs fails to build with contrib (Then are not set up that way?)
+#AM_INIT_AUTOMAKE([subdir-objects])
 
-AM_CONFIG_HEADER([config.h])
+m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
+
+AC_CONFIG_HEADERS([config.h site.h])
 
 AC_CONFIG_FILES([Makefile
-		libglusterfs/Makefile 
-		libglusterfs/src/Makefile
-		glusterfsd/Makefile
-		glusterfsd/src/Makefile
+                libglusterfs/Makefile
+                libglusterfs/src/Makefile
+                libglusterd/Makefile
+                libglusterd/src/Makefile
+                geo-replication/src/peer_gsec_create
+                geo-replication/src/peer_mountbroker
+                geo-replication/src/peer_mountbroker.py
+                geo-replication/src/peer_georep-sshkey.py
+                extras/peer_add_secret_pub
+                geo-replication/syncdaemon/conf.py
+                geo-replication/gsyncd.conf
+                extras/snap_scheduler/conf.py
+                glusterfsd/Makefile
+                glusterfsd/src/Makefile
                 rpc/Makefile
                 rpc/rpc-lib/Makefile
                 rpc/rpc-lib/src/Makefile
                 rpc/rpc-transport/Makefile
                 rpc/rpc-transport/socket/Makefile
                 rpc/rpc-transport/socket/src/Makefile
-                rpc/rpc-transport/rdma/Makefile
-                rpc/rpc-transport/rdma/src/Makefile
                 rpc/xdr/Makefile
                 rpc/xdr/src/Makefile
-		xlators/Makefile
-		xlators/mount/Makefile
-		xlators/mount/fuse/Makefile
-		xlators/mount/fuse/src/Makefile
-		xlators/mount/fuse/utils/mount.glusterfs
-		xlators/mount/fuse/utils/mount_glusterfs
-		xlators/mount/fuse/utils/Makefile
-		xlators/storage/Makefile
-		xlators/storage/posix/Makefile
-		xlators/storage/posix/src/Makefile
-		xlators/cluster/Makefile
-		xlators/cluster/afr/Makefile
-		xlators/cluster/afr/src/Makefile
-		xlators/cluster/stripe/Makefile
-		xlators/cluster/stripe/src/Makefile
-		xlators/cluster/dht/Makefile
-		xlators/cluster/dht/src/Makefile
-		xlators/performance/Makefile
-		xlators/performance/write-behind/Makefile
-		xlators/performance/write-behind/src/Makefile
-		xlators/performance/read-ahead/Makefile
-		xlators/performance/read-ahead/src/Makefile
-		xlators/performance/io-threads/Makefile
-		xlators/performance/io-threads/src/Makefile
-		xlators/performance/io-cache/Makefile
-		xlators/performance/io-cache/src/Makefile
-		xlators/performance/symlink-cache/Makefile
-		xlators/performance/symlink-cache/src/Makefile
-		xlators/performance/quick-read/Makefile
-		xlators/performance/quick-read/src/Makefile
-                xlators/performance/stat-prefetch/Makefile
-                xlators/performance/stat-prefetch/src/Makefile
-		xlators/debug/Makefile
-		xlators/debug/trace/Makefile
-		xlators/debug/trace/src/Makefile
-		xlators/debug/error-gen/Makefile
-		xlators/debug/error-gen/src/Makefile
-		xlators/debug/io-stats/Makefile
-		xlators/debug/io-stats/src/Makefile
-		xlators/protocol/Makefile
-		xlators/protocol/auth/Makefile
-		xlators/protocol/auth/addr/Makefile
-		xlators/protocol/auth/addr/src/Makefile
-		xlators/protocol/auth/login/Makefile
-		xlators/protocol/auth/login/src/Makefile
-		xlators/protocol/client/Makefile
-		xlators/protocol/client/src/Makefile
-		xlators/protocol/server/Makefile
-		xlators/protocol/server/src/Makefile
-		xlators/features/Makefile
-		xlators/features/locks/Makefile
-		xlators/features/locks/src/Makefile
-		xlators/features/trash/Makefile
-		xlators/features/trash/src/Makefile
-		xlators/features/quota/Makefile
-		xlators/features/quota/src/Makefile
-		xlators/features/read-only/Makefile
-		xlators/features/read-only/src/Makefile
-		xlators/features/mac-compat/Makefile
-		xlators/features/mac-compat/src/Makefile
-		xlators/features/quiesce/Makefile
-		xlators/features/quiesce/src/Makefile
-		xlators/encryption/Makefile
-		xlators/encryption/rot-13/Makefile
-		xlators/encryption/rot-13/src/Makefile
-                cli/Makefile
-                cli/src/Makefile
-		doc/Makefile
-		doc/examples/Makefile
-		doc/hacker-guide/Makefile
-		extras/Makefile
-                extras/glusterfs-defrag
-		extras/init.d/Makefile
-		extras/init.d/glusterd.plist
-                extras/init.d/glusterd-Debian
-                extras/init.d/glusterd-Redhat
-                extras/init.d/glusterd-SuSE
-		extras/benchmarking/Makefile
-                extras/volgen/Makefile
-                extras/volgen/glusterfs-volgen
-		contrib/fuse-util/Makefile
-                xlators/features/access-control/Makefile
-                xlators/features/access-control/src/Makefile
+                xlators/Makefile
+                xlators/meta/Makefile
+                xlators/meta/src/Makefile
+                xlators/mount/Makefile
+                xlators/mount/fuse/Makefile
+                xlators/mount/fuse/src/Makefile
+                xlators/mount/fuse/utils/mount.glusterfs
+                xlators/mount/fuse/utils/mount_glusterfs
+                xlators/mount/fuse/utils/Makefile
+                xlators/storage/Makefile
+                xlators/storage/posix/Makefile
+                xlators/storage/posix/src/Makefile
+                xlators/cluster/Makefile
+                xlators/cluster/afr/Makefile
+                xlators/cluster/afr/src/Makefile
+                xlators/cluster/dht/Makefile
+                xlators/cluster/dht/src/Makefile
+                xlators/cluster/ec/Makefile
+                xlators/cluster/ec/src/Makefile
+                xlators/performance/Makefile
+                xlators/performance/write-behind/Makefile
+                xlators/performance/write-behind/src/Makefile
+                xlators/performance/read-ahead/Makefile
+                xlators/performance/read-ahead/src/Makefile
+                xlators/performance/readdir-ahead/Makefile
+                xlators/performance/readdir-ahead/src/Makefile
+                xlators/performance/io-threads/Makefile
+                xlators/performance/io-threads/src/Makefile
+                xlators/performance/io-cache/Makefile
+                xlators/performance/io-cache/src/Makefile
+                xlators/performance/quick-read/Makefile
+                xlators/performance/quick-read/src/Makefile
+                xlators/performance/open-behind/Makefile
+                xlators/performance/open-behind/src/Makefile
+                xlators/performance/md-cache/Makefile
+                xlators/performance/md-cache/src/Makefile
+                xlators/performance/nl-cache/Makefile
+                xlators/performance/nl-cache/src/Makefile
+                xlators/debug/Makefile
+                xlators/debug/sink/Makefile
+                xlators/debug/sink/src/Makefile
+                xlators/debug/trace/Makefile
+                xlators/debug/trace/src/Makefile
+                xlators/debug/error-gen/Makefile
+                xlators/debug/error-gen/src/Makefile
+                xlators/debug/delay-gen/Makefile
+                xlators/debug/delay-gen/src/Makefile
+                xlators/debug/io-stats/Makefile
+                xlators/debug/io-stats/src/Makefile
+                xlators/protocol/Makefile
+                xlators/protocol/auth/Makefile
+                xlators/protocol/auth/addr/Makefile
+                xlators/protocol/auth/addr/src/Makefile
+                xlators/protocol/auth/login/Makefile
+                xlators/protocol/auth/login/src/Makefile
+                xlators/protocol/client/Makefile
+                xlators/protocol/client/src/Makefile
+                xlators/protocol/server/Makefile
+                xlators/protocol/server/src/Makefile
+                xlators/features/Makefile
+                xlators/features/arbiter/Makefile
+                xlators/features/arbiter/src/Makefile
+                xlators/features/thin-arbiter/Makefile
+                xlators/features/thin-arbiter/src/Makefile
+                xlators/features/changelog/Makefile
+                xlators/features/changelog/src/Makefile
+                xlators/features/changelog/lib/Makefile
+                xlators/features/changelog/lib/src/Makefile
+                xlators/features/locks/Makefile
+                xlators/features/locks/src/Makefile
+                xlators/features/quota/Makefile
+                xlators/features/quota/src/Makefile
+                xlators/features/marker/Makefile
+                xlators/features/marker/src/Makefile
+                xlators/features/selinux/Makefile
+                xlators/features/selinux/src/Makefile
+                xlators/features/sdfs/Makefile
+                xlators/features/sdfs/src/Makefile
+                xlators/features/read-only/Makefile
+                xlators/features/read-only/src/Makefile
+                xlators/features/compress/Makefile
+                xlators/features/compress/src/Makefile
+                xlators/features/namespace/Makefile
+                xlators/features/namespace/src/Makefile
+                xlators/features/quiesce/Makefile
+                xlators/features/quiesce/src/Makefile
+                xlators/features/barrier/Makefile
+                xlators/features/barrier/src/Makefile
+                xlators/features/index/Makefile
+                xlators/features/index/src/Makefile
+                xlators/features/gfid-access/Makefile
+                xlators/features/gfid-access/src/Makefile
+                xlators/features/trash/Makefile
+                xlators/features/trash/src/Makefile
+                xlators/features/snapview-server/Makefile
+                xlators/features/snapview-server/src/Makefile
+                xlators/features/snapview-client/Makefile
+                xlators/features/snapview-client/src/Makefile
+                xlators/features/upcall/Makefile
+                xlators/features/upcall/src/Makefile
+                xlators/features/shard/Makefile
+                xlators/features/shard/src/Makefile
+                xlators/features/bit-rot/Makefile
+                xlators/features/bit-rot/src/Makefile
+                xlators/features/bit-rot/src/stub/Makefile
+                xlators/features/bit-rot/src/bitd/Makefile
+                xlators/features/leases/Makefile
+                xlators/features/leases/src/Makefile
+                xlators/features/cloudsync/Makefile
+                xlators/features/cloudsync/src/Makefile
+                xlators/features/utime/Makefile
+                xlators/features/utime/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile
+                xlators/features/metadisp/Makefile
+                xlators/features/metadisp/src/Makefile
+                xlators/playground/Makefile
+                xlators/playground/template/Makefile
+                xlators/playground/template/src/Makefile
+                xlators/system/Makefile
+                xlators/system/posix-acl/Makefile
+                xlators/system/posix-acl/src/Makefile
                 xlators/nfs/Makefile
                 xlators/nfs/server/Makefile
                 xlators/nfs/server/src/Makefile
                 xlators/mgmt/Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
-		glusterfs.spec])
+                cli/Makefile
+                cli/src/Makefile
+                doc/Makefile
+                extras/Makefile
+                extras/glusterd.vol
+                extras/cliutils/Makefile
+                extras/init.d/Makefile
+                extras/init.d/glusterd.plist
+                extras/init.d/glusterd-Debian
+                extras/init.d/glusterd-Redhat
+                extras/init.d/glusterd-FreeBSD
+                extras/init.d/glusterd-SuSE
+                extras/init.d/glustereventsd-Debian
+                extras/init.d/glustereventsd-Redhat
+                extras/init.d/glustereventsd-FreeBSD
+                extras/ganesha/Makefile
+                extras/ganesha/config/Makefile
+                extras/ganesha/scripts/Makefile
+                extras/ganesha/ocf/Makefile
+                extras/systemd/Makefile
+                extras/systemd/glusterd.service
+                extras/systemd/glustereventsd.service
+                extras/systemd/glusterfssharedstorage.service
+                extras/systemd/gluster-ta-volume.service
+                extras/run-gluster.tmpfiles
+                extras/benchmarking/Makefile
+                extras/hook-scripts/Makefile
+                extras/ocf/Makefile
+                extras/ocf/glusterd
+                extras/ocf/volume
+                extras/LinuxRPM/Makefile
+                extras/geo-rep/Makefile
+                extras/geo-rep/schedule_georep.py
+                extras/firewalld/Makefile
+                extras/hook-scripts/add-brick/Makefile
+                extras/hook-scripts/add-brick/pre/Makefile
+                extras/hook-scripts/add-brick/post/Makefile
+                extras/hook-scripts/create/Makefile
+                extras/hook-scripts/create/post/Makefile
+                extras/hook-scripts/delete/Makefile
+                extras/hook-scripts/delete/pre/Makefile
+                extras/hook-scripts/start/Makefile
+                extras/hook-scripts/start/post/Makefile
+                extras/hook-scripts/set/Makefile
+                extras/hook-scripts/set/post/Makefile
+                extras/hook-scripts/stop/Makefile
+                extras/hook-scripts/stop/pre/Makefile
+                extras/hook-scripts/reset/Makefile
+                extras/hook-scripts/reset/post/Makefile
+                extras/hook-scripts/reset/pre/Makefile
+                extras/python/Makefile
+                extras/snap_scheduler/Makefile
+                events/Makefile
+                events/src/Makefile
+                events/src/eventsapiconf.py
+                events/tools/Makefile
+                contrib/fuse-util/Makefile
+                contrib/umountd/Makefile
+                glusterfs-api.pc
+                libgfchangelog.pc
+                api/Makefile
+                api/src/Makefile
+                api/examples/Makefile
+                geo-replication/Makefile
+                geo-replication/src/Makefile
+                geo-replication/syncdaemon/Makefile
+                tools/Makefile
+                tools/gfind_missing_files/Makefile
+                heal/Makefile
+                heal/src/Makefile
+                glusterfs.spec
+                tools/glusterfind/src/tool.conf
+                tools/glusterfind/glusterfind
+                tools/glusterfind/Makefile
+                tools/glusterfind/src/Makefile
+                tools/setgfid2path/Makefile
+                tools/setgfid2path/src/Makefile])
 
 AC_CANONICAL_HOST
 
 AC_PROG_CC
+AC_DISABLE_STATIC
 AC_PROG_LIBTOOL
+AC_SUBST([shrext_cmds])
+
+AC_CHECK_PROG([RPCGEN], [rpcgen], [yes], [no])
+
+if test "x$RPCGEN" = "xno"; then
+   AC_MSG_ERROR([`rpcgen` not found, glusterfs needs `rpcgen` exiting..])
+fi
+
+# Initialize CFLAGS before usage
+AC_ARG_ENABLE([debug],
+              AC_HELP_STRING([--enable-debug],
+                             [Enable debug build options.]))
+if test "x$enable_debug" = "xyes"; then
+        BUILD_DEBUG=yes
+        GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
+else
+        BUILD_DEBUG=no
+fi
+
+SANITIZER=none
+
+AC_ARG_ENABLE([asan],
+              AC_HELP_STRING([--enable-asan],
+                             [Enable Address Sanitizer support]))
+if test "x$enable_asan" = "xyes"; then
+        SANITIZER=asan
+        AC_CHECK_LIB([asan], [__asan_init], ,
+                [AC_MSG_ERROR([--enable-asan requires libasan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=address -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lasan"
+fi
+
+AC_ARG_ENABLE([tsan],
+              AC_HELP_STRING([--enable-tsan],
+                             [Enable Thread Sanitizer support]))
+if test "x$enable_tsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=tsan
+        AC_CHECK_LIB([tsan], [__tsan_init], ,
+                [AC_MSG_ERROR([--enable-tsan requires libtsan.so, exiting])])
+        if test "x$ac_cv_lib_tsan___tsan_init" = xyes; then
+                AC_MSG_CHECKING([whether tsan API can be used])
+                saved_CFLAGS=${CFLAGS}
+                CFLAGS="${CFLAGS} -fsanitize=thread"
+                AC_COMPILE_IFELSE(
+                [AC_LANG_PROGRAM([
+                   [#include <sanitizer/tsan_interface.h>]],
+                   [[__tsan_create_fiber(0)]])],
+                   [TSAN_API=yes], [TSAN_API=no])
+                AC_MSG_RESULT([$TSAN_API])
+                if test x$TSAN_API = "xyes"; then
+                   AC_DEFINE(HAVE_TSAN_API, 1, [Define if tsan API can be used.])
+                fi
+                CFLAGS=${saved_CFLAGS}
+        fi
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=thread -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -ltsan"
+fi
+
+AC_ARG_ENABLE([ubsan],
+              AC_HELP_STRING([--enable-ubsan],
+                             [Enable Undefined Behavior Sanitizer support]))
+if test "x$enable_ubsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=ubsan
+        AC_CHECK_LIB([ubsan], [__ubsan_default_options], ,
+                [AC_MSG_ERROR([--enable-ubsan requires libubsan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=undefined -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lubsan"
+fi
+
+# Initialize CFLAGS before usage
+BUILD_TCMALLOC=no
+AC_ARG_ENABLE([tcmalloc],
+              AC_HELP_STRING([--enable-tcmalloc],
+                             [Enable linking with tcmalloc library.]))
+if test "x$enable_tcmalloc" = "xyes"; then
+    BUILD_TCMALLOC=yes
+    GF_CFLAGS="${GF_CFLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"
+    AC_CHECK_LIB([tcmalloc], [malloc], [],
+                 [AC_MSG_ERROR([when --enable-tcmalloc is used, tcmalloc library needs to be present])])
+    GF_LDFLAGS="-ltcmalloc ${GF_LDFLAGS}"
+fi
+
+
+dnl When possible, prefer libtirpc over glibc rpc.
+dnl
+dnl On newer linux with only libtirpc, use libtirpc. (Specifying
+dnl --without-libtirpc is an error.)
+dnl
+dnl on older linux with glibc rpc and WITH libtirpc, use libtirpc
+dnl by default except when configured with --without-libtirpc.
+dnl
+dnl on old linux with glibc rpc and WITHOUT libtirpc, default to
+dnl use glibc rpc.
+dnl
+AC_ARG_WITH([libtirpc],
+        [AC_HELP_STRING([--without-libtirpc], [Use legacy glibc RPC.])],
+        [with_libtirpc="no"], [with_libtirpc="yes"])
+
+dnl ipv6-default is off by default
+dnl
+dnl ipv6-default requires libtirpc. (glibc rpc does not support IPv6.)
+dnl ipv6-default can only be enabled if libtipc is enabled.
+dnl
+AC_ARG_WITH([ipv6-default],
+        AC_HELP_STRING([--with-ipv6-default], [Set IPv6 as default.]),
+        [with_ipv6_default=${with_libtirpc}], [with_ipv6_default="no"])
+
+AC_CHECK_FILE([/etc/centos-release])
+if test "x$ac_cv_file__etc_centos_release" = "xyes"; then
+        if grep "release 6" /etc/centos-release; then
+                with_ipv6_default="no"
+        fi
+fi
+
+dnl On some distributions '-ldl' isn't automatically added to LIBS
+AC_CHECK_LIB([dl], [dlopen], [LIB_DL=-ldl])
+AC_SUBST(LIB_DL)
+
+AC_ARG_ENABLE([privport_tracking],
+              AC_HELP_STRING([--disable-privport_tracking],
+                             [Disable internal tracking of privileged ports.]))
+TRACK_PRIVPORTS="yes"
+if test x"$enable_privport_tracking" = x"no"; then
+        TRACK_PRIVPORTS="no"
+        AC_DEFINE(GF_DISABLE_PRIVPORT_TRACKING, 1,
+                  [Disable internal tracking of privileged ports.])
+fi
+
+case $host_os in
+  darwin*)
+    if ! test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -ge 7; then
+       AC_MSG_ERROR([You need at least OS X 10.7 (Lion) to build Glusterfs])
+    fi
+    # OSX version lesser than 9 has llvm/clang optimization issues which leads to various segfaults
+    if test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -lt 9; then
+       GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
+    fi
+    ;;
+esac
+
+# --enable-valgrind prevents calling dlclose(), this leaks memory
+AC_ARG_ENABLE([valgrind],
+              AC_HELP_STRING([--enable-valgrind@<:@=memcheck,drd@:>@],
+                             [Enable valgrind for resource leak (memcheck, which is
+                              the default) or thread synchronization (drd) debugging.]))
+case x$enable_valgrind in
+  xmemcheck|xyes)
+    AC_DEFINE(RUN_WITH_MEMCHECK, 1,
+              [Define if all processes should run under 'valgrind --tool=memcheck'.])
+    VALGRIND_TOOL=memcheck
+    ;;
+  xdrd)
+    AC_DEFINE(RUN_WITH_DRD, 1,
+              [Define if all processes should run under 'valgrind --tool=drd'.])
+    VALGRIND_TOOL=drd
+    ;;
+  x|xno)
+    VALGRIND_TOOL=no
+    ;;
+  *)
+    AC_MSG_ERROR([Please specify --enable-valgrind@<:@=memcheck,drd@:>@])
+    ;;
+esac
+
+AC_ARG_WITH([previous-options],
+        [AS_HELP_STRING([--with-previous-options],
+                        [read config.status for configure options])
+        ],
+        [ if test -r ./config.status && \
+             args=$(grep 'ac_cs_config=' config.status | \
+                    sed -e 's/.*"\(.*\)".*/\1/' -e "s/'//g" -e "s/--with-previous-options//g") ; then
+            echo "###"
+            echo "### Rerunning as '$0 $args'"
+            echo "###"
+            exec $0 $args
+          fi
+        ])
+
+AC_ARG_WITH(pkgconfigdir,
+            [  --with-pkgconfigdir=DIR      pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@],
+            [pkgconfigdir=$withval],
+            [pkgconfigdir='${libdir}/pkgconfig'])
+AC_SUBST(pkgconfigdir)
 
 AC_ARG_WITH(mountutildir,
             [  --with-mountutildir=DIR mount helper utility in DIR @<:@/sbin@:>@],
@@ -137,6 +457,13 @@ AC_ARG_WITH(mountutildir,
             [mountutildir='/sbin'])
 AC_SUBST(mountutildir)
 
+AC_ARG_WITH(systemddir,
+            [  --with-systemddir=DIR systemd service files in DIR @<:@PREFIX/lib/systemd/system@:>@],
+            [systemddir=$withval],
+            [systemddir='${prefix}/lib/systemd/system'])
+AC_SUBST(systemddir)
+AM_CONDITIONAL([USE_SYSTEMD], test [ -d '/usr/lib/systemd/system' ])
+
 AC_ARG_WITH(initdir,
             [  --with-initdir=DIR init.d scripts in DIR @<:@/etc/init.d@:>@],
             [initdir=$withval],
@@ -149,12 +476,49 @@ AC_ARG_WITH(launchddir,
             [launchddir='/Library/LaunchDaemons'])
 AC_SUBST(launchddir)
 
+AC_ARG_WITH(tmpfilesdir,
+            AC_HELP_STRING([--with-tmpfilesdir=DIR],
+                           [tmpfiles config in DIR, disabled by default]),
+            [tmpfilesdir=$withval],
+            [tmpfilesdir=''])
+AC_SUBST(tmpfilesdir)
+
+AC_ARG_WITH([ocf],
+            [AS_HELP_STRING([--without-ocf], [build OCF-compliant cluster resource agents])],
+            ,
+            [OCF_SUBDIR='ocf'],
+            )
+AC_SUBST(OCF_SUBDIR)
+
+AC_ARG_WITH([server],
+            [AS_HELP_STRING([--without-server], [do not build server components])],
+            [with_server='no'],
+            [with_server='yes'],
+            )
+AM_CONDITIONAL([WITH_SERVER], [test x$with_server = xyes])
+
 # LEX needs a check
 AC_PROG_LEX
 if test  "x${LEX}" != "xflex" -a "x${FLEX}" != "xlex"; then
    AC_MSG_ERROR([Flex or lex required to build glusterfs.])
 fi
 
+dnl
+dnl Word sizes...
+dnl
+AC_CHECK_SIZEOF(short)
+AC_CHECK_SIZEOF(int)
+AC_CHECK_SIZEOF(long)
+AC_CHECK_SIZEOF(long long)
+SIZEOF_SHORT=$ac_cv_sizeof_short
+SIZEOF_INT=$ac_cv_sizeof_int
+SIZEOF_LONG=$ac_cv_sizeof_long
+SIZEOF_LONG_LONG=$ac_cv_sizeof_long_long
+AC_SUBST(SIZEOF_SHORT)
+AC_SUBST(SIZEOF_INT)
+AC_SUBST(SIZEOF_LONG)
+AC_SUBST(SIZEOF_LONG_LONG)
+
 # YACC needs a check
 AC_PROG_YACC
 if test "x${YACC}" = "xbyacc" -o "x${YACC}" = "xyacc" -o "x${YACC}" = "x"; then
@@ -163,22 +527,116 @@ fi
 
 AC_CHECK_TOOL([LD],[ld])
 
+AC_CHECK_LIB([crypto], [MD5], , AC_MSG_ERROR([OpenSSL crypto library is required to build glusterfs]))
+
 AC_CHECK_LIB([pthread], [pthread_mutex_init], , AC_MSG_ERROR([Posix threads library is required to build glusterfs]))
-		 
+
 AC_CHECK_FUNC([dlopen], [has_dlopen=yes], AC_CHECK_LIB([dl], [dlopen], , AC_MSG_ERROR([Dynamic linking library required to build glusterfs])))
 
+AC_CHECK_LIB([readline], [rl_do_undo], [RL_UNDO="yes"], [RL_UNDO="no"])
+
+AC_CHECK_LIB([intl], [gettext])
 
 AC_CHECK_HEADERS([sys/xattr.h])
 
+AC_CHECK_HEADERS([sys/ioctl.h], AC_DEFINE(HAVE_IOCTL_IN_SYS_IOCTL_H, 1, [have sys/ioctl.h]))
+
 AC_CHECK_HEADERS([sys/extattr.h])
 
+AC_CHECK_HEADERS([openssl/dh.h])
+
+AC_CHECK_HEADERS([openssl/ecdh.h])
+
+AC_CHECK_LIB([ssl], [SSL_CTX_get0_param], [AC_DEFINE([HAVE_SSL_CTX_GET0_PARAM], [1], [define if found OpenSSL SSL_CTX_get0_param])])
+
+dnl Math library
+AC_CHECK_LIB([m], [pow], [MATH_LIB='-lm'], [MATH_LIB=''])
+AC_SUBST(MATH_LIB)
+
+dnl depend on libuuid.so
+PKG_CHECK_MODULES([UUID], [uuid],
+        [have_uuid=yes
+         AC_DEFINE(HAVE_LIBUUID, 1, [have libuuid.so])
+         PKGCONFIG_UUID=uuid],
+        [have_uuid=no])
+AM_CONDITIONAL([HAVE_LIBUUID], [test x$have_uuid = xyes])
+
+dnl older version of libuuid (from e2fsprogs) require including uuid/uuid.h
+saved_CFLAGS=${CFLAGS}
+CFLAGS="${CFLAGS} ${UUID_CFLAGS}"
+AC_CHECK_HEADER([uuid.h], [], [AC_CHECK_HEADER([uuid/uuid.h])],
+                [[#if HAVE_UUID_H
+                  #include <uuid.h>
+                  #endif
+                ]])
+CFLAGS=${saved_CFLAGS}
+if test "x$ac_cv_header_uuid_uuid_h" = "xyes"; then
+    UUID_CFLAGS="${UUID_CFLAGS} -I$(pkg-config --variable=includedir uuid)/uuid"
+    have_uuid=yes
+fi
+
+if test "x$have_uuid" != "xyes"; then
+    case $host_os in
+        *freebsd*)
+            AC_MSG_ERROR([e2fsprogs-libuuid is required to build glusterfs])
+            ;;
+        linux*)
+            AC_MSG_ERROR([libuuid is required to build glusterfs])
+            ;;
+        *)
+            AC_MSG_ERROR([a Linux compatible libuuid is required to build glusterfs])
+            ;;
+    esac
+fi
+
+dnl libglusterfs needs uuid.h, practically everything depends on it
+GF_CFLAGS="${GF_CFLAGS} ${UUID_CFLAGS}"
+dnl PKGCONFIG_UUID is used for the dependency in *.pc.in files
+AC_SUBST(PKGCONFIG_UUID)
+
+dnl NetBSD does not support POSIX ACLs :-(
 case $host_os in
-  darwin*)
-    if ! test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -ge 5; then
-      AC_MSG_ERROR([You need at least OS X 10.5 (Leopard) to build Glusterfs])
-    fi
-    ;;
+     *netbsd* | darwin*)
+        AC_MSG_WARN([platform does not support POSIX ACLs... disabling them])
+        ACL_LIBS=''
+        USE_POSIX_ACLS='0'
+        BUILD_POSIX_ACLS='no'
+        ;;
+     *)
+        AC_CHECK_HEADERS([sys/acl.h], ,
+                         AC_MSG_ERROR([Support for POSIX ACLs is required]))
+        USE_POSIX_ACLS='1'
+        BUILD_POSIX_ACLS='yes'
+        case $host_os in
+             linux*)
+                ACL_LIBS='-lacl'
+                ;;
+             solaris*)
+                ACL_LIBS='-lsec'
+                ;;
+             *freebsd*)
+                ACL_LIBS='-lc'
+                ;;
+             darwin*)
+                ACL_LIBS='-lc'
+                ;;
+        esac
+        if test "x${ACL_LIBS}" = "x-lacl"; then
+          AC_CHECK_HEADERS([acl/libacl.h], , AC_MSG_ERROR([libacl is required for building on ${host_os}]))
+        fi
+        ;;
 esac
+AC_SUBST(ACL_LIBS)
+AC_SUBST(USE_POSIX_ACLS)
+
+# libglusterfs/checksum
+AC_CHECK_HEADERS([openssl/md5.h])
+AC_CHECK_LIB([z], [adler32], [ZLIB_LIBS="-lz"], AC_MSG_ERROR([zlib is required to build glusterfs]))
+AC_SUBST(ZLIB_LIBS)
+
+AC_CHECK_HEADERS([linux/falloc.h])
+
+AC_CHECK_HEADERS([linux/oom.h], AC_DEFINE(HAVE_LINUX_OOM_H, 1, [have linux/oom.h]))
 
 dnl Mac OS X does not have spinlocks
 AC_CHECK_FUNC([pthread_spin_init], [have_spinlock=yes])
@@ -202,11 +660,68 @@ if test "x${have_setfsuid}" = "xyes" -a "x${have_setfsgid}" = "xyes"; then
   AC_DEFINE(HAVE_SET_FSID, 1, [define if found setfsuid setfsgid])
 fi
 
+dnl test umount2 function
+AC_CHECK_FUNC([umount2], [have_umount2=yes])
+
+if test "x${have_umount2}" = "xyes"; then
+   AC_DEFINE(HAVE_UMOUNT2, 1, [define if found umount2])
+fi
+
+dnl Check Python Availability
+have_python=no
+dnl if the user has not specified a python, pick one
+if test -z "${PYTHON}"; then
+  case $host_os in
+    freebsd*)
+      if test -x /usr/local/bin/python3; then
+        PYTHON=/usr/local/bin/python3
+      else
+        PYTHON=/usr/local/bin/python2
+      fi
+      ;;
+    *)
+      if test -x /usr/bin/python3; then
+        PYTHON=/usr/bin/python3
+      else
+        PYTHON=/usr/bin/python2
+      fi
+      ;;
+  esac
+fi
+AM_PATH_PYTHON([2.6],,[:])
+if test -n "${PYTHON}"; then
+   have_python=yes
+fi
+AM_CONDITIONAL(HAVE_PYTHON, test "x$have_python" = "xyes")
+
+dnl Use pkg-config to get runtime search path missing from ${PYTHON}-config
+dnl Just do "true" on failure so that configure does not bail out
+dnl Note: python 2.6's devel pkg (e.g. in CentOS/RHEL 6) does not have
+dnl pkg-config files, so this work-around instead
+if test "x${PYTHON_VERSION}" = "x2.6"; then
+  PYTHON_CFLAGS=$(python-config --includes)
+  PYTHON_LIBS=$(python-config --libs)
+else
+  PKG_CHECK_MODULES([PYTHON], "python-${PYTHON_VERSION}",,true)
+fi
+
+PYTHON_CFLAGS=$(echo ${PYTHON_CFLAGS} | sed -e 's|-I|-isystem |')
+
+BUILD_PYTHON_SITE_PACKAGES=${pythondir}
+AC_SUBST(BUILD_PYTHON_SITE_PACKAGES)
+
+# Eval two times to expand fully. First eval replaces $exec_prefix into $prefix
+# Second eval will expand $prefix
+build_python_site_packages_temp="${pythondir}"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+BUILD_PYTHON_SITE_PACKAGES_EXPANDED=${build_python_site_packages_temp}
+AC_SUBST(BUILD_PYTHON_SITE_PACKAGES_EXPANDED)
 
 # FUSE section
 AC_ARG_ENABLE([fuse-client],
-	      AC_HELP_STRING([--disable-fuse-client],
-			     [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
+              AC_HELP_STRING([--disable-fuse-client],
+                             [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
 
 BUILD_FUSE_CLIENT=no
 if test "x$enable_fuse_client" != "xno"; then
@@ -215,106 +730,339 @@ if test "x$enable_fuse_client" != "xno"; then
 fi
 
 AC_SUBST(FUSE_CLIENT_SUBDIR)
+
+AC_ARG_ENABLE([fuse-notifications],
+  AS_HELP_STRING([--disable-fuse-notifications], [Disable FUSE notifications]))
+
+AS_IF([test "x$enable_fuse_notifications" != "xno"], [
+  AC_DEFINE([HAVE_FUSE_NOTIFICATIONS], [1], [Use FUSE notifications])
+])
+
 # end FUSE section
 
 
+AC_CHECK_LIB([ssl], TLS_method, [HAVE_OPENSSL_1_1="yes"], [HAVE_OPENSSL_1_1="no"])
+if test "x$HAVE_OPENSSL_1_1" = "xyes"; then
+    AC_DEFINE([HAVE_TLS_METHOD], [1], [Using OpenSSL-1.1 TLS_method])
+else
+    AC_CHECK_LIB([ssl], TLSv1_2_method, [AC_DEFINE([HAVE_TLSV1_2_METHOD], [1], [Using OpenSSL-1.0 TLSv1_2_method])])
+fi
+
+
 # FUSERMOUNT section
 AC_ARG_ENABLE([fusermount],
-              AC_HELP_STRING([--enable-fusermount],
-                             [Build fusermount]))
-
-BUILD_FUSERMOUNT="no"
-if test "x$enable_fusermount" = "xyes"; then
-  FUSERMOUNT_SUBDIR="contrib/fuse-util"
-  BUILD_FUSERMOUNT="yes"
-  AC_DEFINE(GF_FUSERMOUNT, 1, [Use our own fusermount])
+              AC_HELP_STRING([--disable-fusermount],
+                             [Use system's fusermount]))
+
+BUILD_FUSERMOUNT="yes"
+if test "x$enable_fusermount" = "xno"; then
+   BUILD_FUSERMOUNT="no"
+else
+   AC_DEFINE(GF_FUSERMOUNT, 1, [Use our own fusermount])
+   FUSERMOUNT_SUBDIR="contrib/fuse-util"
 fi
 
 AC_SUBST(FUSERMOUNT_SUBDIR)
 #end FUSERMOUNT section
 
-
 # EPOLL section
 AC_ARG_ENABLE([epoll],
-	      AC_HELP_STRING([--disable-epoll],
-			     [Use poll instead of epoll.]))
+              AC_HELP_STRING([--disable-epoll],
+                             [Use poll instead of epoll.]))
 
 BUILD_EPOLL=no
 if test "x$enable_epoll" != "xno"; then
    AC_CHECK_HEADERS([sys/epoll.h],
                     [BUILD_EPOLL=yes],
-		    [BUILD_EPOLL=no])
+                    [BUILD_EPOLL=no])
 fi
 # end EPOLL section
 
+# SYNCDAEMON section
+AC_ARG_ENABLE([georeplication],
+              AC_HELP_STRING([--disable-georeplication],
+                             [Do not install georeplication components]))
 
-# IBVERBS section
-AC_ARG_ENABLE([ibverbs],
-	      AC_HELP_STRING([--disable-ibverbs],
-			     [Do not build the ibverbs transport]))
-
-if test "x$enable_ibverbs" != "xno"; then
-  AC_CHECK_LIB([ibverbs],
-               [ibv_get_device_list],
-	       [HAVE_LIBIBVERBS="yes"],
-	       [HAVE_LIBIBVERBS="no"])
+BUILD_SYNCDAEMON=no
+case $host_os in
+     freebsd*)
+#do nothing
+       ;;
+     linux*)
+#do nothing
+       ;;
+     netbsd*)
+#do nothing
+       ;;
+     *)
+#disabling geo replication for non-linux platforms
+        enable_georeplication=no
+        ;;
+esac
+SYNCDAEMON_COMPILE=0
+if test "x${with_server}" = "xyes" -a "x${enable_georeplication}" != "xno"; then
+  if test "x${have_python}" = "xno" ; then
+    AC_MSG_ERROR([only python 2 and 3 are supported])
+  else
+    SYNCDAEMON_SUBDIR=geo-replication
+    SYNCDAEMON_COMPILE=1
+
+    BUILD_SYNCDAEMON="yes"
+    AC_MSG_CHECKING([if python has ctypes support...])
+    if "${PYTHON}" -c 'import ctypes' 2>/dev/null; then
+      AC_MSG_RESULT("yes")
+    else
+      AC_MSG_ERROR([python does not have ctypes support])
+    fi
+  fi
 fi
+AC_SUBST(SYNCDAEMON_COMPILE)
+AC_SUBST(SYNCDAEMON_SUBDIR)
+# end SYNCDAEMON section
 
-if test "x$enable_ibverbs" = "xyes" -a "x$HAVE_LIBIBVERBS" = "xno"; then
-   echo "ibverbs requested but not found."
-   exit 1
+# only install scripts from extras/geo-rep when enabled
+if test "x${with_server}" = "xyes" -a "x$enable_georeplication" != "xno"; then
+  GEOREP_EXTRAS_SUBDIR=geo-rep
 fi
-
-BUILD_RDMA=no
-BUILD_IBVERBS=no
-if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes"; then
-  IBVERBS_SUBDIR=ib-verbs
-  BUILD_IBVERBS=yes
-  RDMA_SUBDIR=rdma
-  BUILD_RDMA=yes
+AC_SUBST(GEOREP_EXTRAS_SUBDIR)
+AM_CONDITIONAL(USE_GEOREP, test "x$enable_georeplication" != "xno")
+
+# METADISP section
+AC_ARG_ENABLE([metadisp],
+              AC_HELP_STRING([--enable-metadisp],
+                             [Enable the metadata dispersal xlator]))
+BUILD_METADISP=no
+if test "x${enable_metadisp}" = "xyes"; then
+  BUILD_METADISP=yes
 fi
+AM_CONDITIONAL([BUILD_METADISP], [test "x$BUILD_METADISP" = "xyes"])
+# end METADISP section
+
+# Events section
+AC_ARG_ENABLE([events],
+              AC_HELP_STRING([--disable-events],
+                             [Do not install Events components]))
+
+BUILD_EVENTS=no
+EVENTS_ENABLED=0
+EVENTS_SUBDIR=
+if test "x$enable_events" != "xno"; then
+  EVENTS_SUBDIR=events
+  EVENTS_ENABLED=1
+
+  BUILD_EVENTS="yes"
+
+  if test "x${have_python}" = "xno"; then
+     if test "x${enable_events}" = "xyes"; then
+        AC_MSG_ERROR([python 2 or 3 required. exiting.])
+     fi
+     AC_MSG_WARN([python not found, disabling events])
+     EVENTS_SUBDIR=
+     EVENTS_ENABLED=0
+     BUILD_EVENTS="no"
+  else
+    AC_DEFINE(USE_EVENTS, 1, [define if events enabled])
+  fi
+fi
+AC_SUBST(EVENTS_ENABLED)
+AC_SUBST(EVENTS_SUBDIR)
+AM_CONDITIONAL([BUILD_EVENTS], [test "x${BUILD_EVENTS}" = "xyes"])
+# end Events section
+
+# CDC xlator - check if libz is present if so enable HAVE_LIB_Z
+BUILD_CDC=yes
+PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0],,
+                  [AC_CHECK_LIB([z], [deflate], [ZLIB_LIBS="-lz"],
+                                [BUILD_CDC=no])])
+echo -n "features requiring zlib enabled: "
+if test "x$BUILD_CDC" = "xyes" ; then
+  echo "yes"
+  AC_DEFINE(HAVE_LIB_Z, 1, [define if zlib is present])
+else
+  echo "no"
+fi
+AC_SUBST(ZLIB_CFLAGS)
+AC_SUBST(ZLIB_LIBS)
+# end CDC xlator secion
+
+#start firewalld section
+BUILD_FIREWALLD="no"
+AC_ARG_ENABLE([firewalld],
+              AC_HELP_STRING([--enable-firewalld],
+                             [enable installation configuration for firewalld]),
+              [BUILD_FIREWALLD="${enableval}"], [BUILD_FIREWALLD="no"])
+
+if test "x${with_server}" = "xyes" -a "x${BUILD_FIREWALLD}" = "xyes"; then
+        if !(test -d /usr/lib/firewalld/services 1>/dev/null 2>&1) ; then
+                BUILD_FIREWALLD="no (firewalld not installed)"
+        fi
+fi
+AM_CONDITIONAL([USE_FIREWALLD],test ["x${BUILD_FIREWALLD}" =  "xyes"])
+
+#endof firewald section
+
+# xml-output
+AC_ARG_ENABLE([xml-output],
+              AC_HELP_STRING([--disable-xml-output],
+                             [Disable the xml output]))
+BUILD_XML_OUTPUT="yes"
+if test "x$enable_xml_output" != "xno"; then
+    PKG_CHECK_MODULES([XML], [libxml-2.0], [], [no_xml="yes"])
+    if test "x${no_xml}" = "x"; then
+        AC_DEFINE([HAVE_LIB_XML], [1], [Define to 1 if using libxml2.])
+    else
+        if test "x$enable_georeplication" != "xno"; then
+           AC_MSG_ERROR([libxml2 devel libraries not found])
+        else
+           AC_MSG_WARN([libxml2 devel libraries not found disabling XML support])
+           BUILD_XML_OUTPUT="no"
+        fi
 
-AC_SUBST(IBVERBS_SUBDIR)
-AC_SUBST(RDMA_SUBDIR)
-# end IBVERBS section
-
+    fi
+else
+    if test "x$enable_georeplication" != "xno"; then
+       AC_MSG_ERROR([geo-replication requires xml output])
+    fi
+    BUILD_XML_OUTPUT="no"
+fi
+# end of xml-output
 
-dnl FreeBSD > 5 has execinfo as a Ported library for giving a workaround
-dnl solution to GCC backtrace functionality
+dnl cloudsync section
+BUILD_CLOUDSYNC="no"
+AC_CHECK_LIB([curl], [curl_easy_setopt], [LIBCURL="-lcurl"])
+if test -n "$LIBCURL";then
+        HAVE_LIBCURL="yes"
+fi
+AC_CHECK_HEADERS([openssl/hmac.h openssl/evp.h openssl/bio.h openssl/buffer.h], [HAVE_OPENSSL="yes"])
+if test "x$HAVE_LIBCURL" = "xyes" -a "x$HAVE_OPENSSL" = "xyes";then
+   HAVE_AMAZONS3="yes"
+fi
+AM_CONDITIONAL([BUILD_AMAZONS3_PLUGIN], [test "x$HAVE_AMAZONS3" = "xyes"])
+if test "x$HAVE_AMAZONS3" = "xyes";then
+   BUILD_CLOUDSYNC="yes"
+fi
+BUILD_CVLT_PLUGIN="no"
+case $host_os in
+#enable cvlt plugin only for linux platforms
+     linux*)
+       BUILD_CVLT_PLUGIN="yes"
+       BUILD_CLOUDSYNC="yes"
+       ;;
+     *)
+       ;;
+esac
+AM_CONDITIONAL([BUILD_CVLT_PLUGIN], [test "x$BUILD_CVLT_PLUGIN" = "xyes"])
+AM_CONDITIONAL([BUILD_CLOUDSYNC], [test "x$BUILD_CLOUDSYNC" = "xyes"])
+dnl end cloudsync section
 
-AC_CHECK_HEADERS([execinfo.h], [have_backtrace=yes],
-               AC_CHECK_LIB([execinfo], [backtrace], [have_backtrace=yes]))
-dnl               AC_MSG_ERROR([libexecinfo not found libexecinfo required.])))
+dnl SELinux feature enablement
+case $host_os in
+  linux*)
+    AC_ARG_ENABLE([selinux],
+                  AC_HELP_STRING([--disable-selinux],
+                                 [Disable SELinux features]),
+                  [USE_SELINUX="${enableval}"], [USE_SELINUX="yes"])
+    ;;
+  *)
+    USE_SELINUX=no
+    ;;
+esac
+AM_CONDITIONAL(USE_SELINUX, test "x${USE_SELINUX}" = "xyes")
+dnl end of SELinux feature enablement
 
+AC_CHECK_HEADERS([execinfo.h], [have_backtrace=yes])
 if test "x${have_backtrace}" = "xyes"; then
    AC_DEFINE(HAVE_BACKTRACE, 1, [define if found backtrace])
 fi
 AC_SUBST(HAVE_BACKTRACE)
 
+dnl Old (before C11) compiler can compile (but not link) this:
+dnl
+dnl int main () {
+dnl     _Static_assert(1, "True");
+dnl     return 0;
+dnl }
+dnl
+dnl assuming that _Static_assert is an implicitly declared function. So
+dnl we're trying to link just to make sure that this is not the case.
+
+AC_MSG_CHECKING([whether $CC supports C11 _Static_assert])
+AC_TRY_LINK([], [_Static_assert(1, "True");],
+            [STATIC_ASSERT=yes], [STATIC_ASSERT=no])
+
+AC_MSG_RESULT([$STATIC_ASSERT])
+if test x$STATIC_ASSERT = "xyes"; then
+   AC_DEFINE(HAVE_STATIC_ASSERT, 1, [Define if C11 _Static_assert is supported.])
+fi
+
+if test "x${have_backtrace}" != "xyes"; then
+AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);],
+   [],
+   AC_MSG_ERROR([need math library for libexecinfo]))
+fi
+
 dnl glusterfs prints memory usage to stderr by sending it SIGUSR1
-AC_CHECK_FUNC([malloc_stats], [have_malloc_stats=yes])
-if test "x${have_malloc_stats}" = "xyes"; then
-   AC_DEFINE(HAVE_MALLOC_STATS, 1, [define if found malloc_stats])
+AC_CHECK_FUNC([mallinfo], [have_mallinfo=yes])
+if test "x${have_mallinfo}" = "xyes"; then
+   AC_DEFINE(HAVE_MALLINFO, 1, [define if found mallinfo])
 fi
-AC_SUBST(HAVE_MALLOC_STATS)
+AC_SUBST(HAVE_MALLINFO)
 
 dnl Linux, Solaris, Cygwin
 AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec])
 dnl FreeBSD, NetBSD
 AC_CHECK_MEMBERS([struct stat.st_atimespec.tv_nsec])
+case $host_os in
+        *netbsd*)
+        GF_CFLAGS="${GF_CFLAGS} -D_INCOMPLETE_XOPEN_C063 -DCONFIG_MACHINE_BSWAP_H"
+        ;;
+esac
+AC_CHECK_FUNC([linkat], [have_linkat=yes])
+if test "x${have_linkat}" = "xyes"; then
+   AC_DEFINE(HAVE_LINKAT, 1, [define if found linkat])
+fi
+AC_SUBST(HAVE_LINKAT)
+
+dnl check for Monotonic clock
+AC_CHECK_LIB([rt], [clock_gettime], ,
+             AC_MSG_WARN([System doesn't have monotonic clock using contrib]))
 
-dnl Check for argp
+dnl check for argp, FreeBSD has the header in /usr/local/include
+case $host_os in
+    *freebsd*)
+    CFLAGS="${CFLAGS} -isystem /usr/local/include"
+    ARGP_LDADD=-largp
+    ;;
+    *netbsd*)
+    ARGP_LDADD=-largp
+    ;;
+esac
+dnl argp-standalone does not provide a pkg-config file
 AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp]))
-AC_CONFIG_SUBDIRS(argp-standalone)
-BUILD_ARGP_STANDALONE=no
-if test "x${ac_cv_header_argp_h}" = "xno"; then 
-   BUILD_ARGP_STANDALONE=yes
-   ARGP_STANDALONE_CPPFLAGS='-I${top_srcdir}/argp-standalone'
-   ARGP_STANDALONE_LDADD='${top_builddir}/argp-standalone/libargp.a'
+if test "x$ac_cv_header_argp_h" != "xyes"; then
+    AC_MSG_ERROR([argp.h not found, install libargp or argp-standalone])
 fi
-
-AC_SUBST(ARGP_STANDALONE_CPPFLAGS)
-AC_SUBST(ARGP_STANDALONE_LDADD)
+AC_SUBST(ARGP_LDADD)
+
+dnl Check for atomic operation support
+AC_MSG_CHECKING([for gcc __atomic builtins])
+AC_TRY_LINK([], [int v; __atomic_load_n(&v, __ATOMIC_ACQUIRE);],
+            [have_atomic_builtins=yes], [have_atomic_builtins=no])
+if test "x${have_atomic_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, [define if __atomic_*() builtins are available])
+fi
+AC_SUBST(HAVE_ATOMIC_BUILTINS)
+AC_MSG_RESULT([$have_atomic_builtins])
+
+dnl __sync_*() will not be needed if __atomic_*() is available
+AC_MSG_CHECKING([for gcc __sync builtins])
+AC_TRY_LINK([], [__sync_synchronize();],
+            [have_sync_builtins=yes], [have_sync_builtins=no])
+if test "x${have_sync_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNC_BUILTINS, 1, [define if __sync_*() builtins are available])
+fi
+AC_SUBST(HAVE_SYNC_BUILTINS)
+AC_MSG_RESULT([$have_sync_builtins])
 
 AC_CHECK_HEADER([malloc.h], AC_DEFINE(HAVE_MALLOC_H, 1, [have malloc.h]))
 
@@ -323,12 +1071,79 @@ if test "x${have_llistxattr}" = "xyes"; then
    AC_DEFINE(HAVE_LLISTXATTR, 1, [define if llistxattr exists])
 fi
 
-AC_CHECK_FUNC([fdatasync], [have_fdatasync=yes])
+AC_CHECK_FUNC([fdatasync], [have_fdatasync=no])
 if test "x${have_fdatasync}" = "xyes"; then
    AC_DEFINE(HAVE_FDATASYNC, 1, [define if fdatasync exists])
 fi
 
-# Check the distribution where you are compiling glusterfs on 
+AC_CHECK_FUNC([fallocate], [have_fallocate=yes])
+if test "x${have_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_FALLOCATE, 1, [define if fallocate exists])
+fi
+
+AC_CHECK_FUNC([posix_fallocate], [have_posix_fallocate=yes])
+if test "x${have_posix_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_POSIX_FALLOCATE, 1, [define if posix_fallocate exists])
+fi
+
+# On fedora-29, copy_file_range syscall and the libc API both are present.
+# Whereas, on some machines such as centos-7, RHEL-7, the API is not there.
+# Only the system call is present. So, this change is to determine whether
+# the API is present or not. If not, then check whether the system call is
+# present or not. Accordingly sys_copy_file_range function will first call
+# the API if it is there. Otherwise it will call syscall(SYS_copy_file_range).
+AC_CHECK_FUNC([copy_file_range], [have_copy_file_range=yes])
+if test "x${have_copy_file_range}" = "xyes"; then
+   AC_DEFINE(HAVE_COPY_FILE_RANGE, 1, [define if copy_file_range exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_copy_file_range], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_copy_file_range}" = "xyes"; then
+      AC_DEFINE(HAVE_COPY_FILE_RANGE_SYS, 1, [define if SYS_copy_file_range is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+AC_CHECK_FUNC([syncfs], [have_syncfs=yes])
+if test "x${have_syncfs}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNCFS, 1, [define if syncfs exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_syncfs], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_syncfs}" = "xyes"; then
+      AC_DEFINE(HAVE_SYNCFS_SYS, 1, [define if SYS_syncfs is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+BUILD_NANOSECOND_TIMESTAMPS=no
+AC_CHECK_FUNC([utimensat], [have_utimensat=yes])
+if test "x${have_utimensat}" = "xyes"; then
+   BUILD_NANOSECOND_TIMESTAMPS=yes
+   AC_DEFINE(HAVE_UTIMENSAT, 1, [define if utimensat exists])
+fi
+
+OLD_CFLAGS=${CFLAGS}
+CFLAGS="-D_GNU_SOURCE"
+AC_CHECK_DECL([SEEK_HOLE], , , [#include <unistd.h>])
+if test "x${ac_cv_have_decl_SEEK_HOLE}" = "xyes"; then
+   AC_DEFINE(HAVE_SEEK_HOLE, 1, [define if SEEK_HOLE is available])
+fi
+CFLAGS=${OLD_CFLAGS}
+
+AC_CHECK_FUNC([accept4], [have_accept4=yes])
+if test "x${have_accept4}" = "xyes"; then
+   AC_DEFINE(HAVE_ACCEPT4, 1, [define if accept4 exists])
+fi
+
+AC_CHECK_FUNC([paccept], [have_paccept=yes])
+if test "x${have_paccept}" = "xyes"; then
+AC_DEFINE(HAVE_PACCEPT, 1, [define if paccept exists])
+fi
+
+# Check the distribution where you are compiling glusterfs on
 
 GF_DISTRIBUTION=
 AC_CHECK_FILE([/etc/debian_version])
@@ -348,84 +1163,555 @@ fi
 AC_SUBST(GF_DISTRIBUTION)
 
 GF_HOST_OS=""
-GF_LDFLAGS="-rdynamic"
+GF_LDFLAGS="${GF_LDFLAGS} -rdynamic"
+
+dnl see --with-libtirpc option check above, libtirpc(-devel) is required for
+dnl ipv6-default
+if test "x${with_libtirpc}" = "xyes" || test "x${with_ipv6_default}" = "xyes" ; then
+    PKG_CHECK_MODULES([TIRPC], [libtirpc],
+       [with_libtirpc="yes"; GF_CFLAGS="$GF_CFLAGS $TIRPC_CFLAGS"; GF_LDFLAGS="$GF_LDFLAGS $TIRPC_LIBS";],
+       [with_libtirpc="missing"; with_ipv6_default="no"])
+fi
+
+if test "x${with_libtirpc}" = "xmissing" ; then
+    AC_CHECK_HEADERS([rpc/rpc.h],[
+        AC_MSG_WARN([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed.
+            Disabling libtirpc and ipv6-default and falling back to legacy glibc rpc headers.
+            This is a transitional warning message. Eventually it will be an error message.
+            ---------------------------------------------------------------------------------])],[
+        AC_MSG_ERROR([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed
+            and there were no legacy glibc rpc headers and library to fall back to.
+            ---------------------------------------------------------------------------------])])
+fi
+
+if test "x$with_ipv6_default" = "xyes" ; then
+   GF_CFLAGS="$GF_CFLAGS -DIPV6_DEFAULT"
+fi
+
+dnl check for gcc -Werror=format-security
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Wformat -Werror=format-security"
+AC_MSG_CHECKING([whether $CC accepts -Werror=format-security])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_format_security=yes], [cc_werror_format_security=no])
+echo $cc_werror_format_security
+if test "x$cc_werror_format_security" = "xyes"; then
+    GF_CFLAGS="$GF_CFLAGS ${CFLAGS}"
+fi
+CFLAGS="$saved_CFLAGS"
+
+dnl check for gcc -Werror=implicit-function-declaration
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Werror=implicit-function-declaration"
+AC_MSG_CHECKING([whether $CC accepts -Werror=implicit-function-declaration])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_implicit=yes], [cc_werror_implicit=no])
+echo $cc_werror_implicit
+if test "x$cc_werror_implicit" = "xyes"; then
+    GF_CFLAGS="${GF_CFLAGS} ${CFLAGS}"
+fi
+CFLAGS="$saved_CFLAGS"
+
+dnl clang is mostly GCC-compatible, but its version is much lower,
+dnl so we have to check for it.
+AC_MSG_CHECKING([if compiling with clang])
+
+AC_COMPILE_IFELSE(
+[AC_LANG_PROGRAM([], [[
+#ifndef __clang__
+       not clang
+#endif
+]])],
+[CLANG=yes], [CLANG=no])
+
+AC_MSG_RESULT([$CLANG])
+
+if test "x$CLANG" = "xyes"; then
+   GF_CFLAGS="${GF_CFLAGS} -Wno-gnu"
+fi
+
+if test "x$ac_cv_header_execinfo_h" = "xno"; then
+   # The reason is that __builtin_frame_address(n) for n > 0 seems
+   # to just crash on most platforms when -fomit-stack-pointer is
+   # specified, which seems to be the default for many platforms on
+   # -O2. The documentation says that __builtin_frame_address()
+   # should return NULL in case it can't get the frame, but it
+   # seems to crash instead.
+
+   # execinfo.c in ./contrib/libexecinfo uses __builtin_frame_address(n)
+   # for providing cross platform backtrace*() functions.
+   if test "x$CLANG" = "xno"; then
+      GF_CFLAGS="${GF_CFLAGS} -fno-omit-frame-pointer"
+   fi
+fi
+
+old_prefix=$prefix
+if test "x$prefix" = xNONE; then
+        prefix=$ac_default_prefix
+fi
+old_exec_prefix=$exec_prefix
+if test "x$exec_prefix" = xNONE; then
+        exec_prefix="$(eval echo $prefix)"
+fi
+GLUSTERFS_LIBEXECDIR="$(eval echo $libexecdir)/glusterfs"
+prefix=$old_prefix
+exec_prefix=$old_exec_prefix
+
+### Dirty hacky stuff to make LOCALSTATEDIR work
+if test "x$prefix" = xNONE; then
+   test $localstatedir = '${prefix}/var' && localstatedir=$ac_default_prefix/var
+   localstatedir=/var
+fi
+localstatedir="$(eval echo ${localstatedir})"
+LOCALSTATEDIR=$localstatedir
+
+GLUSTERFSD_MISCDIR="$(eval echo ${localstatedir})/lib/misc/glusterfsd"
 
-GF_FUSE_LDADD="-lfuse"
+old_prefix=$prefix
+if test "x$prefix" = xNONE; then
+    prefix=$ac_default_prefix
+fi
+GLUSTERD_VOLFILE="$(eval echo ${sysconfdir})/glusterfs/glusterd.vol"
+prefix=$old_prefix
+
+
+GFAPI_EXTRA_LDFLAGS='-Wl,--version-script=$(top_srcdir)/api/src/gfapi.map'
 case $host_os in
      linux*)
-     dnl	GF_LINUX_HOST_OS=1
         GF_HOST_OS="GF_LINUX_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
-	;;
+        GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
+        GLUSTERD_WORKDIR="${LOCALSTATEDIR}/lib/glusterd"
+        ;;
      solaris*)
         GF_HOST_OS="GF_SOLARIS_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -O0"
-	GF_LDFLAGS=""
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_GLUSTERFS_LDFLAGS="-lnsl -lresolv -lsocket"
+        GF_CFLAGS="${GF_CFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -m64"
         BUILD_FUSE_CLIENT=no
         FUSE_CLIENT_SUBDIR=""
-	;;
-     *bsd*)
+        GLUSTERD_WORKDIR="${LOCALSTATEDIR}/lib/glusterd"
+        ;;
+     *netbsd*)
+        GF_HOST_OS="GF_BSD_HOST_OS"
+        GF_CFLAGS="${GF_CFLAGS} -D_INCOMPLETE_XOPEN_C063"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
+        GF_LDADD="${ARGP_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
+        fi
+        GF_FUSE_LDADD="-lperfuse"
+        BUILD_FUSE_CLIENT=yes
+        LEXLIB=""
+        BUILD_FUSERMOUNT=no
+        FUSERMOUNT_SUBDIR=""
+        GLUSTERD_WORKDIR="${LOCALSTATEDIR}/db/glusterd"
+        ;;
+     *freebsd*)
         GF_HOST_OS="GF_BSD_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-	   GF_GLUSTERFS_LDFLAGS="-lexecinfo"
-	fi				      
-	BUILD_FUSE_CLIENT=no
-	;;
+        GF_CFLAGS="${GF_CFLAGS} -DO_DSYNC=0"
+        GF_CFLAGS="${GF_CFLAGS} -Dxdr_quad_t=xdr_longlong_t"
+        GF_CFLAGS="${GF_CFLAGS} -Dxdr_u_quad_t=xdr_u_longlong_t"
+        GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
+        GF_LDADD="${ARGP_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
+        fi
+        BUILD_FUSE_CLIENT=yes
+        BUILD_FUSERMOUNT=no
+        FUSERMOUNT_SUBDIR=""
+        GLUSTERD_WORKDIR="${LOCALSTATEDIR}/db/glusterd"
+        ;;
      darwin*)
         GF_HOST_OS="GF_DARWIN_HOST_OS"
-	LIBTOOL=glibtool
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -bundle -undefined suppress -flat_namespace -D_XOPEN_SOURCE -O0"
-	GF_GLUSTERFS_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -undefined suppress -flat_namespace -O0"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
-	;;
+        LIBTOOL=glibtool
+        GF_CFLAGS="${GF_CFLAGS} -D_REENTRANT -D_XOPEN_SOURCE "
+        GF_CFLAGS="${GF_CFLAGS} -D_DARWIN_USE_64_BIT_INODE "
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_LDADD="${ARGP_LDADD}"
+        GF_LDFLAGS="${GF_LDFLAGS}"
+        GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
+        BUILD_FUSERMOUNT="no"
+        FUSERMOUNT_SUBDIR=""
+        GLUSTERD_WORKDIR="${LOCALSTATEDIR}/db/glusterd"
+        GFAPI_EXTRA_LDFLAGS='-Wl,-alias_list,$(top_srcdir)/api/src/gfapi.aliases'
+        ;;
 esac
 
+# Default value for sbindir
+prefix_temp=$prefix
+exec_prefix_temp=$exec_prefix
+
+test "${prefix}" = "NONE" && prefix="${ac_default_prefix}"
+test "${exec_prefix}" = "NONE" && exec_prefix='${prefix}'
+sbintemp="${sbindir}"
+eval sbintemp=\"${sbintemp}\"
+eval sbintemp=\"${sbintemp}\"
+SBIN_DIR=${sbintemp}
+
+sysconfdirtemp="${sysconfdir}"
+eval sysconfdirtemp=\"${sysconfdirtemp}\"
+SYSCONF_DIR=${sysconfdirtemp}
+
+prefix=$prefix_temp
+exec_prefix=$exec_prefix_temp
+
+AC_SUBST(SBIN_DIR)
+AC_SUBST(SYSCONF_DIR)
+
+# lazy umount emulation
+UMOUNTD_SUBDIR=""
+if test "x${GF_HOST_OS}" != "xGF_LINUX_HOST_OS" ; then
+        UMOUNTD_SUBDIR="contrib/umountd"
+fi
+AC_SUBST(UMOUNTD_SUBDIR)
+
+
+# enable debug section
+AC_ARG_ENABLE([debug],
+              AC_HELP_STRING([--enable-debug],
+                             [Enable debug build options.]))
+
+AC_ARG_ENABLE([mempool],
+              AC_HELP_STRING([--disable-mempool],
+                             [Disable the Gluster memory pooler.]))
+
+USE_MEMPOOL="yes"
+if test "x$enable_mempool" = "xno"; then
+        USE_MEMPOOL="no"
+        AC_DEFINE(GF_DISABLE_MEMPOOL, 1, [Disable the Gluster memory pooler.])
+fi
+
+# syslog section
+AC_ARG_ENABLE([syslog],
+              AC_HELP_STRING([--disable-syslog],
+                             [Disable syslog for logging]))
+
+USE_SYSLOG="yes"
+if test "x$enable_syslog" != "xno"; then
+  AC_DEFINE(GF_USE_SYSLOG, 1, [Use syslog for logging])
+else
+  USE_SYSLOG="no"
+fi
+AM_CONDITIONAL([ENABLE_SYSLOG], [test x$USE_SYSLOG = xyes])
+#end syslog section
+
 BUILD_READLINE=no
 AC_CHECK_LIB([readline -lcurses],[readline],[RLLIBS="-lreadline -lcurses"])
 AC_CHECK_LIB([readline -ltermcap],[readline],[RLLIBS="-lreadline -ltermcap"])
 AC_CHECK_LIB([readline -lncurses],[readline],[RLLIBS="-lreadline -lncurses"])
 
-if test "x$RLLIBS" != "x"; then
-   AC_DEFINE(HAVE_READLINE, 1, [readline enabled CLI])
-   BUILD_READLINE=yes
+if test -n "$RLLIBS"; then
+   if test "x$RL_UNDO" = "xyes"; then
+      AC_DEFINE(HAVE_READLINE, 1, [readline enabled CLI])
+      BUILD_READLINE=yes
+   else
+      BUILD_READLINE="no (present but missing undo)"
+   fi
+
+fi
+
+BUILD_LIBAIO=no
+AC_CHECK_LIB([aio],[io_setup],[LIBAIO="-laio"])
+
+if test -n "$LIBAIO"; then
+   AC_DEFINE(HAVE_LIBAIO, 1, [libaio based POSIX enabled])
+   BUILD_LIBAIO=yes
+fi
+
+dnl gnfs section
+BUILD_GNFS="no"
+RPCBIND_SERVICE=""
+AC_ARG_ENABLE([gnfs],
+              AC_HELP_STRING([--enable-gnfs],
+                             [Enable legacy gnfs server xlator.]))
+if test "x${with_server}" = "xyes" -a "x$enable_gnfs" = "xyes"; then
+    BUILD_GNFS="yes"
+    GF_CFLAGS="$GF_CFLAGS -DBUILD_GNFS"
+    RPCBIND_SERVICE="rpcbind.service"
+fi
+AM_CONDITIONAL([BUILD_GNFS], [test x$BUILD_GNFS = xyes])
+AC_SUBST(BUILD_GNFS)
+AC_SUBST(RPCBIND_SERVICE)
+dnl end gnfs section
+
+dnl Check for userspace-rcu
+PKG_CHECK_MODULES([URCU], [liburcu-bp], [],
+  [AC_CHECK_HEADERS([urcu-bp.h],
+     [URCU_LIBS='-lurcu-bp'],
+     AC_MSG_ERROR([liburcu-bp not found]))])
+PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.8], [],
+  [PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.7],
+    [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
+     USE_CONTRIB_URCU='yes'],
+    [AC_CHECK_HEADERS([urcu/cds.h],
+      [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
+       URCU_CDS_LIBS='-lurcu-cds'
+       USE_CONTRIB_URCU='yes'],
+      [AC_MSG_ERROR([liburcu-cds not found])])])])
+
+BUILD_UNITTEST="no"
+AC_ARG_ENABLE([cmocka],
+              AC_HELP_STRING([--enable-cmocka],
+                             [Enable cmocka build options.]))
+if test "x$enable_cmocka" = "xyes"; then
+    BUILD_UNITTEST="yes"
+    PKG_CHECK_MODULES([UNITTEST], [cmocka >= 1.0.1], [BUILD_UNITTEST="yes"],
+        [AC_MSG_ERROR([cmocka library is required to build glusterfs])]
+    )
+fi
+AM_CONDITIONAL([UNITTEST], [test x$BUILD_UNITTEST = xyes])
+
+dnl Define UNIT_TESTING only for building cmocka binaries.
+UNITTEST_CFLAGS="${UNITTEST_CFLAGS} -DUNIT_TESTING=1"
+
+dnl Add cmocka for unit tests
+case $host_os in
+  freebsd*)
+    dnl remove --coverage on FreeBSD due to a known llvm packaging bug
+    UNITTEST_CFLAGS="${UNITTEST_CPPFLAGS} ${UNITTEST_CFLAGS} -g -DDEBUG -O0"
+    UNITTEST_LDFLAGS="${UNITTEST_LIBS} ${UNITTEST_LDFLAGS}"
+    ;;
+  *)
+    UNITTEST_CFLAGS="${UNITTEST_CPPFLAGS} ${UNITTEST_CFLAGS} -g -DDEBUG -O0 --coverage"
+    UNITTEST_LDFLAGS="${UNITTEST_LIBS} ${UNITTEST_LDFLAGS}"
+    ;;
+esac
+
+AC_SUBST(UNITTEST_CFLAGS)
+AC_SUBST(UNITTEST_LDFLAGS)
+
+AC_SUBST(CFLAGS)
+# end enable debug section
+
+# EC dynamic code generation section
+
+EC_DYNAMIC_SUPPORT="none"
+EC_DYNAMIC_ARCH="none"
+
+AC_ARG_ENABLE([ec-dynamic],
+              AC_HELP_STRING([--disable-ec-dynamic],
+                             [Disable all dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-intel],
+              AC_HELP_STRING([--disable-ec-dynamic-intel],
+                             [Disable all INTEL dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-arm],
+              AC_HELP_STRING([--disable-ec-dynamic-arm],
+                             [Disable all ARM dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-x64],
+              AC_HELP_STRING([--disable-ec-dynamic-x64],
+                             [Disable dynamic INTEL x64 code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-sse],
+              AC_HELP_STRING([--disable-ec-dynamic-sse],
+                             [Disable dynamic INTEL SSE code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-avx],
+              AC_HELP_STRING([--disable-ec-dynamic-avx],
+                             [Disable dynamic INTEL AVX code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-neon],
+              AC_HELP_STRING([--disable-ec-dynamic-neon],
+                             [Disable dynamic ARM NEON code generation for EC module]))
+
+if test "x$enable_ec_dynamic" != "xno"; then
+  case $host in
+    x86_64*)
+      if test "x$enable_ec_dynamic_intel" != "xno"; then
+        if test "x$enable_ec_dynamic_x64" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT x64"
+          AC_DEFINE(USE_EC_DYNAMIC_X64, 1, [Defined if using dynamic INTEL x64 code])
+        fi
+        if test "x$enable_ec_dynamic_sse" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT sse"
+          AC_DEFINE(USE_EC_DYNAMIC_SSE, 1, [Defined if using dynamic INTEL SSE code])
+        fi
+        if test "x$enable_ec_dynamic_avx" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT avx"
+          AC_DEFINE(USE_EC_DYNAMIC_AVX, 1, [Defined if using dynamic INTEL AVX code])
+        fi
+
+        if test "x$EC_DYNAMIC_SUPPORT" != "xnone"; then
+          EC_DYNAMIC_ARCH="intel"
+        fi
+      fi
+      ;;
+    arm*)
+      if test "x$enable_ec_dynamic_arm" != "xno"; then
+        if test "x$enable_ec_dynamic_neon" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT neon"
+          AC_DEFINE(USE_EC_DYNAMIC_NEON, 1, [Defined if using dynamic ARM NEON code])
+        fi
+
+        if test "x$EC_DYNAMIC_SUPPORT" != "xnone"; then
+          EC_DYNAMIC_ARCH="arm"
+        fi
+      fi
+      ;;
+  esac
+
+  EC_DYNAMIC_SUPPORT="${EC_DYNAMIC_SUPPORT#none }"
 fi
 
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_INTEL], [test "x$EC_DYNAMIC_ARCH" = "xintel"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_ARM], [test "x$EC_DYNAMIC_ARCH" = "xarm"])
+
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_X64], [test "x${EC_DYNAMIC_SUPPORT##*x64*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_SSE], [test "x${EC_DYNAMIC_SUPPORT##*sse*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_AVX], [test "x${EC_DYNAMIC_SUPPORT##*avx*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_NEON], [test "x${EC_DYNAMIC_SUPPORT##*neon*}" = "x"])
+
+AC_SUBST(USE_EC_DYNAMIC_X64)
+AC_SUBST(USE_EC_DYNAMIC_SSE)
+AC_SUBST(USE_EC_DYNAMIC_AVX)
+AC_SUBST(USE_EC_DYNAMIC_NEON)
+
+# end EC dynamic code generation section
+
+dnl libglusterfs.so uses math functions
+GF_LDADD="${GF_LDADD} ${MATH_LIB}"
+
+case $host_os in
+  dnl Can't use libtool's portable "-no-undefined" as it seems to be ignored on Linux
+  linux*)
+    GF_NO_UNDEFINED='-Wl,--no-undefined'
+    ;;
+  darwin*)
+    GF_NO_UNDEFINED='-Wl,-undefined'
+    ;;
+  *)
+  dnl There's an issue on FreeBSD with reference to __progname used in some parts of code
+    GF_NO_UNDEFINED=''
+    ;;
+esac
+dnl GF_XLATOR_DEFAULT_LDFLAGS is for most xlators that expose a common set of symbols
+GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
+dnl GF_XLATOR_LDFLAGS is for xlators that expose extra symbols, e.g. dht
+GF_XLATOR_LDFLAGS='-avoid-version $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
+
 AC_SUBST(GF_HOST_OS)
-AC_SUBST(GF_GLUSTERFS_LDFLAGS)
-AC_SUBST(GF_GLUSTERFS_CFLAGS)
 AC_SUBST(GF_CFLAGS)
 AC_SUBST(GF_LDFLAGS)
 AC_SUBST(GF_LDADD)
+AC_SUBST(GF_FUSE_LDADD)
 AC_SUBST(GF_FUSE_CFLAGS)
 AC_SUBST(RLLIBS)
+AC_SUBST(LIBAIO)
+AC_SUBST(AM_MAKEFLAGS)
+AC_SUBST(AM_LIBTOOLFLAGS)
+AC_SUBST(GF_NO_UNDEFINED)
+AC_SUBST(GF_XLATOR_DEFAULT_LDFLAGS)
+AC_SUBST(GF_XLATOR_LDFLAGS)
+AC_SUBST(GF_XLATOR_MGNT_LIBADD)
+
+case $host_os in
+     *freebsd*)
+        GF_XLATOR_MGNT_LIBADD="-lutil -lprocstat"
+        ;;
+esac
 
 CONTRIBDIR='$(top_srcdir)/contrib'
 AC_SUBST(CONTRIBDIR)
 
-INCLUDES='-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/uuid'
-AC_SUBST(INCLUDES)
+GF_CPPDEFINES='-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)'
+GF_CPPINCLUDES='-include $(top_builddir)/config.h -include $(top_builddir)/site.h -I$(top_srcdir)/libglusterfs/src -I$(top_builddir)/libglusterfs/src'
+if test "x${USE_CONTRIB_URCU}" = "xyes"; then
+    GF_CPPINCLUDES="${GF_CPPINCLUDES} -I\$(CONTRIBDIR)/userspace-rcu"
+fi
+GF_CPPFLAGS="$GF_CPPFLAGS $GF_CPPDEFINES $GF_CPPINCLUDES"
+AC_SUBST([GF_CPPFLAGS])
+
+AM_CONDITIONAL([GF_LINUX_HOST_OS], test "${GF_HOST_OS}" = "GF_LINUX_HOST_OS")
+AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")
+AM_CONDITIONAL([GF_BSD_HOST_OS], test "${GF_HOST_OS}" = "GF_BSD_HOST_OS")
+if  test "${GF_HOST_OS}" = "GF_BSD_HOST_OS"; then
+    AC_DEFINE(GF_BSD_HOST_OS, 1, [This is a BSD compatible OS.])
+fi
 
-AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")	
+AC_SUBST(GLUSTERD_WORKDIR)
+AM_CONDITIONAL([GF_INSTALL_GLUSTERD_WORKDIR], test ! -d ${GLUSTERD_WORKDIR} && test -d ${sysconfdir}/glusterd )
+AC_SUBST(GLUSTERD_VOLFILE)
+AC_SUBST(GLUSTERFS_LIBEXECDIR)
+AC_SUBST(GLUSTERFSD_MISCDIR)
+
+dnl pkg-config versioning
+dnl
+dnl Once we released gluster-api.pc with version=7. Since then we undid the
+dnl library versioning and replaced it with symbol-versioning. The current
+dnl libgfapi.so has version 0, but the symbols have the version from the main
+dnl package at the time they were added.
+dnl
+dnl Because other packages (like samba) use the pkg-config version, we can not
+dnl drop it, or decrease the version easily. The simplest solution is to keep
+dnl the version=7 and add sub-digits for the actual package/symbol versions.
+GFAPI_VERSION="7."${PACKAGE_VERSION}
+LIBGFCHANGELOG_VERSION="0.0.1"
+AC_SUBST(GFAPI_VERSION)
+AC_SUBST(LIBGFCHANGELOG_VERSION)
+
+dnl libtool versioning
+LIBGFXDR_LT_VERSION="0:1:0"
+LIBGFRPC_LT_VERSION="0:1:0"
+LIBGLUSTERFS_LT_VERSION="0:1:0"
+LIBGFCHANGELOG_LT_VERSION="0:1:0"
+GFAPI_LT_VERSION="0:0:0"
+AC_SUBST(LIBGFXDR_LT_VERSION)
+AC_SUBST(LIBGFRPC_LT_VERSION)
+AC_SUBST(LIBGLUSTERFS_LT_VERSION)
+AC_SUBST(LIBGFCHANGELOG_LT_VERSION)
+AC_SUBST(GFAPI_LT_VERSION)
+AC_SUBST(GFAPI_EXTRA_LDFLAGS)
+
+GFAPI_LIBS="${ACL_LIBS}"
+AC_SUBST(GFAPI_LIBS)
+
+dnl this change necessary for run-tests.sh
+AC_CONFIG_FILES([tests/env.rc],[ln -s ${ac_abs_builddir}/env.rc ${ac_abs_srcdir}/env.rc 2>/dev/null])
 
 AC_OUTPUT
 
 echo
 echo "GlusterFS configure summary"
 echo "==========================="
-echo "FUSE client        : $BUILD_FUSE_CLIENT"
-echo "Infiniband verbs   : $BUILD_IBVERBS"
-echo "epoll IO multiplex : $BUILD_EPOLL"
-echo "argp-standalone    : $BUILD_ARGP_STANDALONE"
-echo "fusermount         : $BUILD_FUSERMOUNT"
-echo "readline           : $BUILD_READLINE"
+echo "FUSE client          : $BUILD_FUSE_CLIENT"
+echo "epoll IO multiplex   : $BUILD_EPOLL"
+echo "fusermount           : $BUILD_FUSERMOUNT"
+echo "readline             : $BUILD_READLINE"
+echo "georeplication       : $BUILD_SYNCDAEMON"
+echo "Linux-AIO            : $BUILD_LIBAIO"
+echo "Enable Debug         : $BUILD_DEBUG"
+echo "Run with Valgrind    : $VALGRIND_TOOL"
+echo "Sanitizer enabled    : $SANITIZER"
+echo "Use syslog           : $USE_SYSLOG"
+echo "XML output           : $BUILD_XML_OUTPUT"
+echo "Unit Tests           : $BUILD_UNITTEST"
+echo "Track priv ports     : $TRACK_PRIVPORTS"
+echo "POSIX ACLs           : $BUILD_POSIX_ACLS"
+echo "SELinux features     : $USE_SELINUX"
+echo "firewalld-config     : $BUILD_FIREWALLD"
+echo "Events               : $BUILD_EVENTS"
+echo "EC dynamic support   : $EC_DYNAMIC_SUPPORT"
+echo "Use memory pools     : $USE_MEMPOOL"
+echo "Nanosecond m/atimes  : $BUILD_NANOSECOND_TIMESTAMPS"
+echo "Server components    : $with_server"
+echo "Legacy gNFS server   : $BUILD_GNFS"
+echo "IPV6 default         : $with_ipv6_default"
+echo "Use TIRPC            : $with_libtirpc"
+echo "With Python          : ${PYTHON_VERSION}"
+echo "Cloudsync            : $BUILD_CLOUDSYNC"
+echo "Metadata dispersal   : $BUILD_METADISP"
+echo "Link with TCMALLOC   : $BUILD_TCMALLOC"
 echo
+
+# dnl Note: ${X^^} capitalization assumes bash >= 4.x
+if test "x$SANITIZER" != "xnone"; then
+        echo "Note: since glusterfs processes are daemon processes, use"
+        echo "'export ${SANITIZER^^}_OPTIONS=log_path=/path/to/xxx.log' to collect"
+        echo "sanitizer output. Further details and more options can be"
+        echo "found at https://github.com/google/sanitizers."
+fi
diff --git a/contrib/aclocal/mkdirp.m4 b/contrib/aclocal/mkdirp.m4
new file mode 100644
index 00000000000..d2f7edd5ccd
--- /dev/null
+++ b/contrib/aclocal/mkdirp.m4
@@ -0,0 +1,146 @@
+# Excerpt from autoconf/autoconf/programs.m4
+# This file is part of Autoconf.                       -*- Autoconf -*-
+# Checking for programs.
+
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
+# Foundation, Inc.
+
+# This file is part of Autoconf.  This program is free
+# software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# Under Section 7 of GPL version 3, you are granted additional
+# permissions described in the Autoconf Configure Script Exception,
+# version 3.0, as published by the Free Software Foundation.
+#
+# You should have received a copy of the GNU General Public License
+# and a copy of the Autoconf Configure Script Exception along with
+# this program; see the files COPYINGv3 and COPYING.EXCEPTION
+# respectively.  If not, see <http://www.gnu.org/licenses/>.
+
+# Written by David MacKenzie, with help from
+# Franc,ois Pinard, Karl Berry, Richard Pixley, Ian Lance Taylor,
+# Roland McGrath, Noah Friedman, david d zuhn, and many others.
+
+# AC_PROG_MKDIR_P
+# ---------------
+# Check whether `mkdir -p' is known to be thread-safe, and fall back to
+# install-sh -d otherwise.
+#
+# Automake 1.8 used `mkdir -m 0755 -p --' to ensure that directories
+# created by `make install' are always world readable, even if the
+# installer happens to have an overly restrictive umask (e.g. 077).
+# This was a mistake.  There are at least two reasons why we must not
+# use `-m 0755':
+#   - it causes special bits like SGID to be ignored,
+#   - it may be too restrictive (some setups expect 775 directories).
+#
+# Do not use -m 0755 and let people choose whatever they expect by
+# setting umask.
+#
+# We cannot accept any implementation of `mkdir' that recognizes `-p'.
+# Some implementations (such as Solaris 8's) are vulnerable to race conditions:
+# if a parallel make tries to run `mkdir -p a/b' and `mkdir -p a/c'
+# concurrently, both version can detect that a/ is missing, but only
+# one can create it and the other will error out.  Consequently we
+# restrict ourselves to known race-free implementations.
+#
+# Automake used to define mkdir_p as `mkdir -p .', in order to
+# allow $(mkdir_p) to be used without argument.  As in
+#   $(mkdir_p) $(somedir)
+# where $(somedir) is conditionally defined.  However we don't do
+# that for MKDIR_P.
+#  1. before we restricted the check to GNU mkdir, `mkdir -p .' was
+#     reported to fail in read-only directories.  The system where this
+#     happened has been forgotten.
+#  2. in practice we call $(MKDIR_P) on directories such as
+#       $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     and we don't want to create $(DESTDIR) if $(somedir) is empty.
+#     To support the latter case, we have to write
+#       test -z "$(somedir)" || $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     so $(MKDIR_P) always has an argument.
+#     We will have better chances of detecting a missing test if
+#     $(MKDIR_P) complains about missing arguments.
+#  3. $(MKDIR_P) is named after `mkdir -p' and we don't expect this
+#     to accept no argument.
+#  4. having something like `mkdir .' in the output is unsightly.
+#
+# On NextStep and OpenStep, the `mkdir' command does not
+# recognize any option.  It will interpret all options as
+# directories to create.
+AN_MAKEVAR([MKDIR_P], [AC_PROG_MKDIR_P])
+AC_DEFUN_ONCE([AC_PROG_MKDIR_P],
+[AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl
+
+AC_MSG_CHECKING([for a thread-safe mkdir -p])
+if test -z "$MKDIR_P"; then
+  AC_CACHE_VAL([ac_cv_path_mkdir],
+    [_AS_PATH_WALK([$PATH$PATH_SEPARATOR/opt/sfw/bin],
+      [for ac_prog in mkdir gmkdir; do
+	 for ac_exec_ext in '' $ac_executable_extensions; do
+	   AS_EXECUTABLE_P(["$as_dir/$ac_prog$ac_exec_ext"]) || continue
+	   case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
+	     'mkdir (GNU coreutils) '* | \
+	     'mkdir (coreutils) '* | \
+	     'mkdir (fileutils) '4.1*)
+	       ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
+	       break 3;;
+	   esac
+	 done
+       done])])
+  test -d ./--version && rmdir ./--version
+  if test "${ac_cv_path_mkdir+set}" = set; then
+    MKDIR_P="$ac_cv_path_mkdir -p"
+  else
+    # As a last resort, use the slow shell script.  Don't cache a
+    # value for MKDIR_P within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the value is a relative name.
+    MKDIR_P="$ac_install_sh -d"
+  fi
+fi
+dnl status.m4 does special magic for MKDIR_P instead of AC_SUBST,
+dnl to get relative names right.  However, also AC_SUBST here so
+dnl that Automake versions before 1.10 will pick it up (they do not
+dnl trace AC_SUBST_TRACE).
+dnl FIXME: Remove this once we drop support for Automake < 1.10.
+AC_SUBST([MKDIR_P])dnl
+AC_MSG_RESULT([$MKDIR_P])
+])# AC_PROG_MKDIR_P
+
+
+# From automake/m4/mkdirp.m4
+##                                                          -*- Autoconf -*-
+# Copyright (C) 2003, 2004, 2005, 2006  Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_MKDIR_P
+# ---------------
+# Check for `mkdir -p'.
+AC_DEFUN([AM_PROG_MKDIR_P],
+[
+AC_REQUIRE([AC_PROG_MKDIR_P])dnl
+dnl Automake 1.8 to 1.9.6 used to define mkdir_p.  We now use MKDIR_P,
+dnl while keeping a definition of mkdir_p for backward compatibility.
+dnl @MKDIR_P@ is magic: AC_OUTPUT adjusts its value for each Makefile.
+dnl However we cannot define mkdir_p as $(MKDIR_P) for the sake of
+dnl Makefile.ins that do not define MKDIR_P, so we do our own
+dnl adjustment using top_builddir (which is defined more often than
+dnl MKDIR_P).
+AC_SUBST([mkdir_p], ["$MKDIR_P"])dnl
+case $mkdir_p in
+  [[\\/$]]* | ?:[[\\/]]*) ;;
+  */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
+esac
+])
diff --git a/contrib/aclocal/python.m4 b/contrib/aclocal/python.m4
new file mode 100644
index 00000000000..a39a9009046
--- /dev/null
+++ b/contrib/aclocal/python.m4
@@ -0,0 +1,209 @@
+## ------------------------                                 -*- Autoconf -*-
+## Python file handling
+## From Andrew Dalke
+## Updated by James Henstridge
+## ------------------------
+# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008, 2009
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PATH_PYTHON([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+# ---------------------------------------------------------------------------
+# Adds support for distributing Python modules and packages.  To
+# install modules, copy them to $(pythondir), using the python_PYTHON
+# automake variable.  To install a package with the same name as the
+# automake package, install to $(pkgpythondir), or use the
+# pkgpython_PYTHON automake variable.
+#
+# The variables $(pyexecdir) and $(pkgpyexecdir) are provided as
+# locations to install python extension modules (shared libraries).
+# Another macro is required to find the appropriate flags to compile
+# extension modules.
+#
+# If your package is configured with a different prefix to python,
+# users will have to add the install directory to the PYTHONPATH
+# environment variable, or create a .pth file (see the python
+# documentation for details).
+#
+# If the MINIMUM-VERSION argument is passed, AM_PATH_PYTHON will
+# cause an error if the version of python installed on the system
+# doesn't meet the requirement.  MINIMUM-VERSION should consist of
+# numbers and dots only.
+AC_DEFUN([AM_PATH_PYTHON],
+ [
+  dnl Find a Python interpreter.  Python versions prior to 2.0 are not
+  dnl supported. (2.0 was released on October 16, 2000).
+  m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+                    [python python2 python3 python3.2 python3.1 python3.0 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 dnl
+python2.1 python2.0])
+
+  m4_if([$1],[],[
+    dnl No version check is needed.
+    # Find any Python interpreter.
+    if test -z "$PYTHON"; then
+      AC_PATH_PROGS([PYTHON], _AM_PYTHON_INTERPRETER_LIST, :)
+    fi
+    am_display_PYTHON=python
+  ], [
+    dnl A version check is needed.
+    if test -n "$PYTHON"; then
+      # If the user set $PYTHON, use it and don't search something else.
+      AC_MSG_CHECKING([whether $PYTHON version >= $1])
+      AM_PYTHON_CHECK_VERSION([$PYTHON], [$1],
+			      [AC_MSG_RESULT(yes)],
+			      [AC_MSG_ERROR(too old)])
+      am_display_PYTHON=$PYTHON
+    else
+      # Otherwise, try each interpreter until we find one that satisfies
+      # VERSION.
+      AC_CACHE_CHECK([for a Python interpreter with version >= $1],
+	[am_cv_pathless_PYTHON],[
+	for am_cv_pathless_PYTHON in _AM_PYTHON_INTERPRETER_LIST none; do
+	  test "$am_cv_pathless_PYTHON" = none && break
+	  AM_PYTHON_CHECK_VERSION([$am_cv_pathless_PYTHON], [$1], [break])
+	done])
+      # Set $PYTHON to the absolute path of $am_cv_pathless_PYTHON.
+      if test "$am_cv_pathless_PYTHON" = none; then
+	PYTHON=:
+      else
+        AC_PATH_PROG([PYTHON], [$am_cv_pathless_PYTHON])
+      fi
+      am_display_PYTHON=$am_cv_pathless_PYTHON
+    fi
+  ])
+
+  if test "$PYTHON" = :; then
+  dnl Run any user-specified action, or abort.
+    m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])])
+  else
+
+  dnl Query Python for its version number.  Getting [:3] seems to be
+  dnl the best way to do this; it's what "site.py" does in the standard
+  dnl library.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version],
+    [am_cv_python_version=`$PYTHON -c "import sys; sys.stdout.write(sys.version[[:3]])"`])
+  AC_SUBST([PYTHON_VERSION], [$am_cv_python_version])
+
+  dnl Use the values of $prefix and $exec_prefix for the corresponding
+  dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX.  These are made
+  dnl distinct variables so they can be overridden if need be.  However,
+  dnl general consensus is that you shouldn't need this ability.
+
+  AC_SUBST([PYTHON_PREFIX], ['${prefix}'])
+  AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}'])
+
+  dnl At times (like when building shared libraries) you may want
+  dnl to know which OS platform Python thinks this is.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform],
+    [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`])
+  AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform])
+
+
+  dnl Set up 4 directories:
+
+  dnl pythondir -- where to install python scripts.  This is the
+  dnl   site-packages directory, not the python standard library
+  dnl   directory like in previous automake betas.  This behavior
+  dnl   is more consistent with lispdir.m4 for example.
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON script directory],
+    [am_cv_python_pythondir],
+    [if test "x$prefix" = xNONE
+     then
+       am_py_prefix=$ac_default_prefix
+     else
+       am_py_prefix=$prefix
+     fi
+     am_cv_python_pythondir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(0,0,prefix='$am_py_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pythondir in
+     $am_py_prefix*)
+       am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
+       am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"`
+       ;;
+     *)
+       case $am_py_prefix in
+         /usr|/System*) ;;
+         *)
+	  am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	  ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pythondir], [$am_cv_python_pythondir])
+
+  dnl pkgpythondir -- $PACKAGE directory under pythondir.  Was
+  dnl   PYTHON_SITE_PACKAGE in previous betas, but this naming is
+  dnl   more consistent with the rest of automake.
+
+  AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE])
+
+  dnl pyexecdir -- directory for installing python extension modules
+  dnl   (shared libraries)
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory],
+    [am_cv_python_pyexecdir],
+    [if test "x$exec_prefix" = xNONE
+     then
+       am_py_exec_prefix=$am_py_prefix
+     else
+       am_py_exec_prefix=$exec_prefix
+     fi
+     am_cv_python_pyexecdir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(1,0,prefix='$am_py_exec_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pyexecdir in
+     $am_py_exec_prefix*)
+       am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
+       am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"`
+       ;;
+     *)
+       case $am_py_exec_prefix in
+         /usr|/System*) ;;
+         *)
+	   am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	   ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir])
+
+  dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE)
+
+  AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE])
+
+  dnl Run any user-specified action.
+  $2
+  fi
+
+])
+
+
+# AM_PYTHON_CHECK_VERSION(PROG, VERSION, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
+# ---------------------------------------------------------------------------
+# Run ACTION-IF-TRUE if the Python interpreter PROG has version >= VERSION.
+# Run ACTION-IF-FALSE otherwise.
+# This test uses sys.hexversion instead of the string equivalent (first
+# word of sys.version), in order to cope with versions such as 2.2c1.
+# This supports Python 2.0 or higher. (2.0 was released on October 16, 2000).
+AC_DEFUN([AM_PYTHON_CHECK_VERSION],
+ [prog="import sys
+# split strings by '.' and convert to numeric.  Append some zeros
+# because we need at least 4 digits for the hex conversion.
+# map returns an iterator in Python 3.0 and a list in 2.x
+minver = list(map(int, '$2'.split('.'))) + [[0, 0, 0]]
+minverhex = 0
+# xrange is not present in Python 3.0 and range returns an iterator
+for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]]
+sys.exit(sys.hexversion < minverhex)"
+  AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])])
diff --git a/contrib/apple/daemon.c b/contrib/apple/daemon.c
deleted file mode 100644
index 9389201a1af..00000000000
--- a/contrib/apple/daemon.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-
-   Based on http://www.opensource.apple.com/source/Libc/Libc-583/gen/FreeBSD/daemon.c
- */
-
-/*-
- * Copyright (c) 1990, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <errno.h>
-#include <fcntl.h>
-#include <paths.h>
-#include <signal.h>
-#include <unistd.h>
-
-int
-os_daemon(nochdir, noclose)
-	int nochdir, noclose;
-{
-	struct sigaction osa, sa;
-	int fd;
-	pid_t newgrp;
-	int oerrno;
-	int osa_ok;
-
-	/* A SIGHUP may be thrown when the parent exits below. */
-	sigemptyset(&sa.sa_mask);
-	sa.sa_handler = SIG_IGN;
-	sa.sa_flags = 0;
-	osa_ok = sigaction(SIGHUP, &sa, &osa);
-
-	switch (fork()) {
-	case -1:
-		return (-1);
-	case 0:
-		break;
-	default:
-		_exit(0);
-	}
-
-	newgrp = setsid();
-	oerrno = errno;
-	if (osa_ok != -1)
-		sigaction(SIGHUP, &osa, NULL);
-
-	if (newgrp == -1) {
-		errno = oerrno;
-		return (-1);
-	}
-
-	if (!nochdir)
-		(void)chdir("/");
-
-	if (!noclose && (fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) {
-		(void)dup2(fd, STDIN_FILENO);
-		(void)dup2(fd, STDOUT_FILENO);
-		(void)dup2(fd, STDERR_FILENO);
-		if (fd > 2)
-			(void)close(fd);
-	}
-	return (0);
-}
diff --git a/contrib/apple/daemon.h b/contrib/apple/daemon.h
deleted file mode 100644
index 7a2824b6a47..00000000000
--- a/contrib/apple/daemon.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-int os_daemon(int nochdir, int noclose);
diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index ca571ce5e30..7d28462de47 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -8,4 +8,6 @@
 */
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
-int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param);
+int gf_fuse_unmount_daemon (const char *mountpoint, int fd);
+int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
+                   pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-include/fuse_kernel.h b/contrib/fuse-include/fuse_kernel.h
index 9ae25d6f9c0..1e41e237e6f 100644
--- a/contrib/fuse-include/fuse_kernel.h
+++ b/contrib/fuse-include/fuse_kernel.h
@@ -60,23 +60,87 @@
  * 7.13
  *  - make max number of background requests and congestion threshold
  *    tunables
+ *
+ * 7.14
+ *  - add splice support to fuse device
+ *
+ * 7.15
+ *  - add store notify
+ *  - add retrieve notify
+ *
+ * 7.16
+ *  - add BATCH_FORGET request
+ *  - FUSE_IOCTL_UNRESTRICTED shall now return with array of 'struct
+ *    fuse_ioctl_iovec' instead of ambiguous 'struct iovec'
+ *  - add FUSE_IOCTL_32BIT flag
+ *
+ * 7.17
+ *  - add FUSE_FLOCK_LOCKS and FUSE_RELEASE_FLOCK_UNLOCK
+ *
+ * 7.18
+ *  - add FUSE_IOCTL_DIR flag
+ *  - add FUSE_NOTIFY_DELETE
+ *
+ * 7.19
+ *  - add FUSE_FALLOCATE
+ *
+ * 7.20
+ *  - add FUSE_AUTO_INVAL_DATA
+ *
+ * 7.21
+ *  - add FUSE_READDIRPLUS
+ *  - send the requested events in POLL request
+ *
+ * 7.22
+ *  - add FUSE_ASYNC_DIO
+ *
+ * 7.23
+ *  - add FUSE_WRITEBACK_CACHE
+ *  - add time_gran to fuse_init_out
+ *  - add reserved space to fuse_init_out
+ *  - add FATTR_CTIME
+ *  - add ctime and ctimensec to fuse_setattr_in
+ *  - add FUSE_RENAME2 request
+ *  - add FUSE_NO_OPEN_SUPPORT flag
+ *
+ *  7.24
+ *  - add FUSE_LSEEK for SEEK_HOLE and SEEK_DATA support
  */
 
 #ifndef _LINUX_FUSE_H
 #define _LINUX_FUSE_H
 
-#include <sys/types.h>
-#define __u64 uint64_t
-#define __s64 int64_t
-#define __u32 uint32_t
-#define __s32 int32_t
-#define __u16 uint16_t
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#endif
+
+/*
+ * Version negotiation:
+ *
+ * Both the kernel and userspace send the version they support in the
+ * INIT request and reply respectively.
+ *
+ * If the major versions match then both shall use the smallest
+ * of the two minor versions for communication.
+ *
+ * If the kernel supports a larger major version, then userspace shall
+ * reply with the major version it supports, ignore the rest of the
+ * INIT message and expect a new INIT message from the kernel with a
+ * matching major version.
+ *
+ * If the library supports a larger major version, then it shall fall
+ * back to the major protocol version sent by the kernel for
+ * communication and reply with that major version (and an arbitrary
+ * supported minor version).
+ */
 
 /** Version number of this interface */
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 13
+#define FUSE_KERNEL_MINOR_VERSION 24
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -85,42 +149,42 @@
    userspace works under 64bit kernels */
 
 struct fuse_attr {
-	__u64	ino;
-	__u64	size;
-	__u64	blocks;
-	__u64	atime;
-	__u64	mtime;
-	__u64	ctime;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	ctimensec;
-	__u32	mode;
-	__u32	nlink;
-	__u32	uid;
-	__u32	gid;
-	__u32	rdev;
-	__u32	blksize;
-	__u32	padding;
+	uint64_t	ino;
+	uint64_t	size;
+	uint64_t	blocks;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	ctime;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	ctimensec;
+	uint32_t	mode;
+	uint32_t	nlink;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	rdev;
+	uint32_t	blksize;
+	uint32_t	padding;
 };
 
 struct fuse_kstatfs {
-	__u64	blocks;
-	__u64	bfree;
-	__u64	bavail;
-	__u64	files;
-	__u64	ffree;
-	__u32	bsize;
-	__u32	namelen;
-	__u32	frsize;
-	__u32	padding;
-	__u32	spare[6];
+	uint64_t	blocks;
+	uint64_t	bfree;
+	uint64_t	bavail;
+	uint64_t	files;
+	uint64_t	ffree;
+	uint32_t	bsize;
+	uint32_t	namelen;
+	uint32_t	frsize;
+	uint32_t	padding;
+	uint32_t	spare[6];
 };
 
 struct fuse_file_lock {
-	__u64	start;
-	__u64	end;
-	__u32	type;
-	__u32	pid; /* tgid */
+	uint64_t	start;
+	uint64_t	end;
+	uint32_t	type;
+	uint32_t	pid; /* tgid */
 };
 
 /**
@@ -136,6 +200,7 @@ struct fuse_file_lock {
 #define FATTR_ATIME_NOW	(1 << 7)
 #define FATTR_MTIME_NOW	(1 << 8)
 #define FATTR_LOCKOWNER	(1 << 9)
+#define FATTR_CTIME	(1 << 10)
 
 /**
  * Flags returned by the OPEN request
@@ -151,8 +216,24 @@ struct fuse_file_lock {
 /**
  * INIT request/reply flags
  *
+ * FUSE_ASYNC_READ: asynchronous read requests
+ * FUSE_POSIX_LOCKS: remote locking for POSIX file locks
+ * FUSE_FILE_OPS: kernel sends file handle for fstat, etc... (not yet supported)
+ * FUSE_ATOMIC_O_TRUNC: handles the O_TRUNC open flag in the filesystem
  * FUSE_EXPORT_SUPPORT: filesystem handles lookups of "." and ".."
+ * FUSE_BIG_WRITES: filesystem can handle write size larger than 4kB
  * FUSE_DONT_MASK: don't apply umask to file mode on create operations
+ * FUSE_SPLICE_WRITE: kernel supports splice write on the device
+ * FUSE_SPLICE_MOVE: kernel supports splice move on the device
+ * FUSE_SPLICE_READ: kernel supports splice read on the device
+ * FUSE_FLOCK_LOCKS: remote locking for BSD style file locks
+ * FUSE_HAS_IOCTL_DIR: kernel supports ioctl on directories
+ * FUSE_AUTO_INVAL_DATA: automatically invalidate cached pages
+ * FUSE_DO_READDIRPLUS: do READDIRPLUS (READDIR+LOOKUP in one)
+ * FUSE_READDIRPLUS_AUTO: adaptive readdirplus
+ * FUSE_ASYNC_DIO: asynchronous direct I/O submission
+ * FUSE_WRITEBACK_CACHE: use writeback cache for buffered writes
+ * FUSE_NO_OPEN_SUPPORT: kernel supports zero-message opens
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -161,6 +242,17 @@ struct fuse_file_lock {
 #define FUSE_EXPORT_SUPPORT	(1 << 4)
 #define FUSE_BIG_WRITES		(1 << 5)
 #define FUSE_DONT_MASK		(1 << 6)
+#define FUSE_SPLICE_WRITE	(1 << 7)
+#define FUSE_SPLICE_MOVE	(1 << 8)
+#define FUSE_SPLICE_READ	(1 << 9)
+#define FUSE_FLOCK_LOCKS	(1 << 10)
+#define FUSE_HAS_IOCTL_DIR	(1 << 11)
+#define FUSE_AUTO_INVAL_DATA	(1 << 12)
+#define FUSE_DO_READDIRPLUS	(1 << 13)
+#define FUSE_READDIRPLUS_AUTO	(1 << 14)
+#define FUSE_ASYNC_DIO		(1 << 15)
+#define FUSE_WRITEBACK_CACHE	(1 << 16)
+#define FUSE_NO_OPEN_SUPPORT	(1 << 17)
 
 /**
  * CUSE INIT request/reply flags
@@ -173,6 +265,7 @@ struct fuse_file_lock {
  * Release flags
  */
 #define FUSE_RELEASE_FLUSH	(1 << 0)
+#define FUSE_RELEASE_FLOCK_UNLOCK	(1 << 1)
 
 /**
  * Getattr flags
@@ -204,12 +297,16 @@ struct fuse_file_lock {
  * FUSE_IOCTL_COMPAT: 32bit compat ioctl on 64bit machine
  * FUSE_IOCTL_UNRESTRICTED: not restricted to well-formed ioctls, retry allowed
  * FUSE_IOCTL_RETRY: retry with new iovecs
+ * FUSE_IOCTL_32BIT: 32bit ioctl
+ * FUSE_IOCTL_DIR: is a directory
  *
  * FUSE_IOCTL_MAX_IOV: maximum of in_iovecs + out_iovecs
  */
 #define FUSE_IOCTL_COMPAT	(1 << 0)
 #define FUSE_IOCTL_UNRESTRICTED	(1 << 1)
 #define FUSE_IOCTL_RETRY	(1 << 2)
+#define FUSE_IOCTL_32BIT	(1 << 3)
+#define FUSE_IOCTL_DIR		(1 << 4)
 
 #define FUSE_IOCTL_MAX_IOV	256
 
@@ -259,6 +356,12 @@ enum fuse_opcode {
 	FUSE_DESTROY       = 38,
 	FUSE_IOCTL         = 39,
 	FUSE_POLL          = 40,
+	FUSE_NOTIFY_REPLY  = 41,
+	FUSE_BATCH_FORGET  = 42,
+	FUSE_FALLOCATE     = 43,
+	FUSE_READDIRPLUS   = 44,
+	FUSE_RENAME2       = 45,
+	FUSE_LSEEK         = 46,
 
 	/* CUSE specific operations */
 	CUSE_INIT          = 4096,
@@ -268,6 +371,9 @@ enum fuse_notify_code {
 	FUSE_NOTIFY_POLL   = 1,
 	FUSE_NOTIFY_INVAL_INODE = 2,
 	FUSE_NOTIFY_INVAL_ENTRY = 3,
+	FUSE_NOTIFY_STORE = 4,
+	FUSE_NOTIFY_RETRIEVE = 5,
+	FUSE_NOTIFY_DELETE = 6,
 	FUSE_NOTIFY_CODE_MAX,
 };
 
@@ -277,133 +383,149 @@ enum fuse_notify_code {
 #define FUSE_COMPAT_ENTRY_OUT_SIZE 120
 
 struct fuse_entry_out {
-	__u64	nodeid;		/* Inode ID */
-	__u64	generation;	/* Inode generation: nodeid:gen must
-				   be unique for the fs's lifetime */
-	__u64	entry_valid;	/* Cache timeout for the name */
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	entry_valid_nsec;
-	__u32	attr_valid_nsec;
+	uint64_t	nodeid;		/* Inode ID */
+	uint64_t	generation;	/* Inode generation: nodeid:gen must
+					   be unique for the fs's lifetime */
+	uint64_t	entry_valid;	/* Cache timeout for the name */
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	entry_valid_nsec;
+	uint32_t	attr_valid_nsec;
 	struct fuse_attr attr;
 };
 
 struct fuse_forget_in {
-	__u64	nlookup;
+	uint64_t	nlookup;
+};
+
+struct fuse_forget_one {
+	uint64_t	nodeid;
+	uint64_t	nlookup;
+};
+
+struct fuse_batch_forget_in {
+	uint32_t	count;
+	uint32_t	dummy;
 };
 
 struct fuse_getattr_in {
-	__u32	getattr_flags;
-	__u32	dummy;
-	__u64	fh;
+	uint32_t	getattr_flags;
+	uint32_t	dummy;
+	uint64_t	fh;
 };
 
 #define FUSE_COMPAT_ATTR_OUT_SIZE 96
 
 struct fuse_attr_out {
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	attr_valid_nsec;
-	__u32	dummy;
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	attr_valid_nsec;
+	uint32_t	dummy;
 	struct fuse_attr attr;
 };
 
 #define FUSE_COMPAT_MKNOD_IN_SIZE 8
 
 struct fuse_mknod_in {
-	__u32	mode;
-	__u32	rdev;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	mode;
+	uint32_t	rdev;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_mkdir_in {
-	__u32	mode;
-	__u32	umask;
+	uint32_t	mode;
+	uint32_t	umask;
 };
 
 struct fuse_rename_in {
-	__u64	newdir;
+	uint64_t	newdir;
+};
+
+struct fuse_rename2_in {
+	uint64_t	newdir;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 struct fuse_link_in {
-	__u64	oldnodeid;
+	uint64_t	oldnodeid;
 };
 
 struct fuse_setattr_in {
-	__u32	valid;
-	__u32	padding;
-	__u64	fh;
-	__u64	size;
-	__u64	lock_owner;
-	__u64	atime;
-	__u64	mtime;
-	__u64	unused2;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	unused3;
-	__u32	mode;
-	__u32	unused4;
-	__u32	uid;
-	__u32	gid;
-	__u32	unused5;
+	uint32_t	valid;
+	uint32_t	padding;
+	uint64_t	fh;
+	uint64_t	size;
+	uint64_t	lock_owner;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	ctime;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	ctimensec;
+	uint32_t	mode;
+	uint32_t	unused4;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	unused5;
 };
 
 struct fuse_open_in {
-	__u32	flags;
-	__u32	unused;
+	uint32_t	flags;
+	uint32_t	unused;
 };
 
 struct fuse_create_in {
-	__u32	flags;
-	__u32	mode;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	flags;
+	uint32_t	mode;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_open_out {
-	__u64	fh;
-	__u32	open_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	open_flags;
+	uint32_t	padding;
 };
 
 struct fuse_release_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	release_flags;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	release_flags;
+	uint64_t	lock_owner;
 };
 
 struct fuse_flush_in {
-	__u64	fh;
-	__u32	unused;
-	__u32	padding;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	unused;
+	uint32_t	padding;
+	uint64_t	lock_owner;
 };
 
 struct fuse_read_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	read_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	read_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_WRITE_IN_SIZE 24
 
 struct fuse_write_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	write_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	write_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 struct fuse_write_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_STATFS_SIZE 48
@@ -413,32 +535,32 @@ struct fuse_statfs_out {
 };
 
 struct fuse_fsync_in {
-	__u64	fh;
-	__u32	fsync_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	fsync_flags;
+	uint32_t	padding;
 };
 
 struct fuse_setxattr_in {
-	__u32	size;
-	__u32	flags;
+	uint32_t	size;
+	uint32_t	flags;
 };
 
 struct fuse_getxattr_in {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_getxattr_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_lk_in {
-	__u64	fh;
-	__u64	owner;
+	uint64_t	fh;
+	uint64_t	owner;
 	struct fuse_file_lock lk;
-	__u32	lk_flags;
-	__u32	padding;
+	uint32_t	lk_flags;
+	uint32_t	padding;
 };
 
 struct fuse_lk_out {
@@ -446,134 +568,206 @@ struct fuse_lk_out {
 };
 
 struct fuse_access_in {
-	__u32	mask;
-	__u32	padding;
+	uint32_t	mask;
+	uint32_t	padding;
 };
 
 struct fuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
 };
 
+#define FUSE_COMPAT_INIT_OUT_SIZE 8
+#define FUSE_COMPAT_22_INIT_OUT_SIZE 24
+
 struct fuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
-	__u16   max_background;
-	__u16   congestion_threshold;
-	__u32	max_write;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
+	uint16_t	max_background;
+	uint16_t	congestion_threshold;
+	uint32_t	max_write;
+	uint32_t	time_gran;
+	uint32_t	unused[9];
 };
 
 #define CUSE_INIT_INFO_MAX 4096
 
 struct cuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
 };
 
 struct cuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
-	__u32	max_read;
-	__u32	max_write;
-	__u32	dev_major;		/* chardev major */
-	__u32	dev_minor;		/* chardev minor */
-	__u32	spare[10];
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
+	uint32_t	max_read;
+	uint32_t	max_write;
+	uint32_t	dev_major;		/* chardev major */
+	uint32_t	dev_minor;		/* chardev minor */
+	uint32_t	spare[10];
 };
 
 struct fuse_interrupt_in {
-	__u64	unique;
+	uint64_t	unique;
 };
 
 struct fuse_bmap_in {
-	__u64	block;
-	__u32	blocksize;
-	__u32	padding;
+	uint64_t	block;
+	uint32_t	blocksize;
+	uint32_t	padding;
 };
 
 struct fuse_bmap_out {
-	__u64	block;
+	uint64_t	block;
 };
 
 struct fuse_ioctl_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	cmd;
-	__u64	arg;
-	__u32	in_size;
-	__u32	out_size;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	cmd;
+	uint64_t	arg;
+	uint32_t	in_size;
+	uint32_t	out_size;
+};
+
+struct fuse_ioctl_iovec {
+	uint64_t	base;
+	uint64_t	len;
 };
 
 struct fuse_ioctl_out {
-	__s32	result;
-	__u32	flags;
-	__u32	in_iovs;
-	__u32	out_iovs;
+	int32_t		result;
+	uint32_t	flags;
+	uint32_t	in_iovs;
+	uint32_t	out_iovs;
 };
 
 struct fuse_poll_in {
-	__u64	fh;
-	__u64	kh;
-	__u32	flags;
-	__u32   padding;
+	uint64_t	fh;
+	uint64_t	kh;
+	uint32_t	flags;
+	uint32_t	events;
 };
 
 struct fuse_poll_out {
-	__u32	revents;
-	__u32	padding;
+	uint32_t	revents;
+	uint32_t	padding;
 };
 
 struct fuse_notify_poll_wakeup_out {
-	__u64	kh;
+	uint64_t	kh;
+};
+
+struct fuse_fallocate_in {
+	uint64_t	fh;
+	uint64_t	offset;
+	uint64_t	length;
+	uint32_t	mode;
+	uint32_t	padding;
 };
 
 struct fuse_in_header {
-	__u32	len;
-	__u32	opcode;
-	__u64	unique;
-	__u64	nodeid;
-	__u32	uid;
-	__u32	gid;
-	__u32	pid;
-	__u32	padding;
+	uint32_t	len;
+	uint32_t	opcode;
+	uint64_t	unique;
+	uint64_t	nodeid;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	pid;
+	uint32_t	padding;
 };
 
 struct fuse_out_header {
-	__u32	len;
-	__s32	error;
-	__u64	unique;
+	uint32_t	len;
+	int32_t		error;
+	uint64_t	unique;
 };
 
 struct fuse_dirent {
-	__u64	ino;
-	__u64	off;
-	__u32	namelen;
-	__u32	type;
-	char name[0];
+	uint64_t	ino;
+	uint64_t	off;
+	uint32_t	namelen;
+	uint32_t	type;
+	char name[];
 };
 
 #define FUSE_NAME_OFFSET offsetof(struct fuse_dirent, name)
-#define FUSE_DIRENT_ALIGN(x) (((x) + sizeof(__u64) - 1) & ~(sizeof(__u64) - 1))
+#define FUSE_DIRENT_ALIGN(x) \
+	(((x) + sizeof(uint64_t) - 1) & ~(sizeof(uint64_t) - 1))
 #define FUSE_DIRENT_SIZE(d) \
 	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
 
+struct fuse_direntplus {
+	struct fuse_entry_out entry_out;
+	struct fuse_dirent dirent;
+};
+
+#define FUSE_NAME_OFFSET_DIRENTPLUS \
+	offsetof(struct fuse_direntplus, dirent.name)
+#define FUSE_DIRENTPLUS_SIZE(d) \
+	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET_DIRENTPLUS + (d)->dirent.namelen)
+
 struct fuse_notify_inval_inode_out {
-	__u64	ino;
-	__s64	off;
-	__s64	len;
+	uint64_t	ino;
+	int64_t		off;
+	int64_t		len;
 };
 
 struct fuse_notify_inval_entry_out {
-	__u64	parent;
-	__u32	namelen;
-	__u32	padding;
+	uint64_t	parent;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_delete_out {
+	uint64_t	parent;
+	uint64_t	child;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_store_out {
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+struct fuse_notify_retrieve_out {
+	uint64_t	notify_unique;
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+/* Matches the size of fuse_write_in */
+struct fuse_notify_retrieve_in {
+	uint64_t	dummy1;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	dummy2;
+	uint64_t	dummy3;
+	uint64_t	dummy4;
+};
+
+struct fuse_lseek_in {
+	uint64_t	fh;
+	uint64_t	offset;
+	int32_t		whence;
+	uint32_t	padding;
+};
+
+struct fuse_lseek_out {
+	uint64_t	offset;
 };
 
 #endif /* _LINUX_FUSE_H */
diff --git a/contrib/fuse-include/fuse_kernel_macfuse.h b/contrib/fuse-include/fuse_kernel_macfuse.h
index 3fbf24f70aa..31bc495a552 100644
--- a/contrib/fuse-include/fuse_kernel_macfuse.h
+++ b/contrib/fuse-include/fuse_kernel_macfuse.h
@@ -61,67 +61,59 @@
    userspace works under 64bit kernels */
 
 struct fuse_attr {
-	__u64	ino;
-	__u64	size;
-	__u64	blocks;
-	__u64	atime;
-	__u64	mtime;
-	__u64	ctime;
-#if (__FreeBSD__ >= 10)
-	__u64	crtime;
-#endif /* __FreeBSD__ >= 10 */
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	ctimensec;
-#if (__FreeBSD__ >= 10)
-	__u32	crtimensec;
-#endif /* __FreeBSD__ >= 10 */
-	__u32	mode;
-	__u32	nlink;
-	__u32	uid;
-	__u32	gid;
-	__u32	rdev;
-#if (__FreeBSD__ >= 10)
-	__u32	flags; /* file flags; see chflags(2) */
-#endif /* __FreeBSD__ >= 10 */
+        __u64  ino;
+        __u64  size;
+        __u64  blocks;
+        __u64  atime;
+        __u64  mtime;
+        __u64  ctime;
+        __u64  crtime;
+        __u32  atimensec;
+        __u32  mtimensec;
+        __u32  ctimensec;
+        __u32  crtimensec;
+        __u32  mode;
+        __u32  nlink;
+        __u32  uid;
+        __u32  gid;
+        __u32  rdev;
+        __u32  flags; /* file flags; see chflags(2) */
 };
 
 struct fuse_kstatfs {
-	__u64	blocks;
-	__u64	bfree;
-	__u64	bavail;
-	__u64	files;
-	__u64	ffree;
-	__u32	bsize;
-	__u32	namelen;
-	__u32	frsize;
-	__u32	padding;
-	__u32	spare[6];
+        __u64   blocks;
+        __u64   bfree;
+        __u64   bavail;
+        __u64   files;
+        __u64   ffree;
+        __u32   bsize;
+        __u32   namelen;
+        __u32   frsize;
+        __u32   padding;
+        __u32   spare[6];
 };
 
 struct fuse_file_lock {
-	__u64	start;
-	__u64	end;
-	__u32	type;
-	__u32	pid; /* tgid */
+        __u64   start;
+        __u64   end;
+        __u32   type;
+        __u32   pid; /* tgid */
 };
 
 /**
  * Bitmasks for fuse_setattr_in.valid
  */
-#define FATTR_MODE	(1 << 0)
-#define FATTR_UID	(1 << 1)
-#define FATTR_GID	(1 << 2)
-#define FATTR_SIZE	(1 << 3)
-#define FATTR_ATIME	(1 << 4)
-#define FATTR_MTIME	(1 << 5)
-#define FATTR_FH	(1 << 6)
-#if (__FreeBSD__ >= 10)
-#define FATTR_CRTIME	(1 << 28)
-#define FATTR_CHGTIME	(1 << 29)
-#define FATTR_BKUPTIME	(1 << 30)
-#define FATTR_FLAGS	(1 << 31)
-#endif /* __FreeBSD__ >= 10 */
+#define FATTR_MODE      (1 << 0)
+#define FATTR_UID       (1 << 1)
+#define FATTR_GID       (1 << 2)
+#define FATTR_SIZE      (1 << 3)
+#define FATTR_ATIME     (1 << 4)
+#define FATTR_MTIME     (1 << 5)
+#define FATTR_FH        (1 << 6)
+#define FATTR_CRTIME    (1 << 28)
+#define FATTR_CHGTIME   (1 << 29)
+#define FATTR_BKUPTIME  (1 << 30)
+#define FATTR_FLAGS     (1 << 31)
 
 /**
  * Flags returned by the OPEN request
@@ -129,311 +121,304 @@ struct fuse_file_lock {
  * FOPEN_DIRECT_IO: bypass page cache for this open file
  * FOPEN_KEEP_CACHE: don't invalidate the data cache on open
  */
-#define FOPEN_DIRECT_IO		(1 << 0)
-#define FOPEN_KEEP_CACHE	(1 << 1)
-#if (__FreeBSD__ >= 10)
-#define FOPEN_PURGE_ATTR	(1 << 30)
-#define FOPEN_PURGE_UBC		(1 << 31)
-#endif
+#define FOPEN_DIRECT_IO         (1 << 0)
+#define FOPEN_KEEP_CACHE        (1 << 1)
+#define FOPEN_PURGE_ATTR        (1 << 30)
+#define FOPEN_PURGE_UBC         (1 << 31)
 
 /**
  * INIT request/reply flags
  */
-#define FUSE_ASYNC_READ		(1 << 0)
-#define FUSE_POSIX_LOCKS	(1 << 1)
-#if (__FreeBSD__ >= 10)
-#define FUSE_CASE_INSENSITIVE	(1 << 29)
-#define FUSE_VOL_RENAME		(1 << 30)
-#define FUSE_XTIMES		(1 << 31)
-#endif /* __FreeBSD__ >= 10 */
+#define FUSE_ASYNC_READ         (1 << 0)
+#define FUSE_POSIX_LOCKS        (1 << 1)
+#define FUSE_CASE_INSENSITIVE   (1 << 29)
+#define FUSE_VOL_RENAME         (1 << 30)
+#define FUSE_XTIMES             (1 << 31)
 
 /**
  * Release flags
  */
-#define FUSE_RELEASE_FLUSH	(1 << 0)
+#define FUSE_RELEASE_FLUSH      (1 << 0)
 
 enum fuse_opcode {
-	FUSE_LOOKUP	   = 1,
-	FUSE_FORGET	   = 2,  /* no reply */
-	FUSE_GETATTR	   = 3,
-	FUSE_SETATTR	   = 4,
-	FUSE_READLINK	   = 5,
-	FUSE_SYMLINK	   = 6,
-	FUSE_MKNOD	   = 8,
-	FUSE_MKDIR	   = 9,
-	FUSE_UNLINK	   = 10,
-	FUSE_RMDIR	   = 11,
-	FUSE_RENAME	   = 12,
-	FUSE_LINK	   = 13,
-	FUSE_OPEN	   = 14,
-	FUSE_READ	   = 15,
-	FUSE_WRITE	   = 16,
-	FUSE_STATFS	   = 17,
-	FUSE_RELEASE       = 18,
-	FUSE_FSYNC         = 20,
-	FUSE_SETXATTR      = 21,
-	FUSE_GETXATTR      = 22,
-	FUSE_LISTXATTR     = 23,
-	FUSE_REMOVEXATTR   = 24,
-	FUSE_FLUSH         = 25,
-	FUSE_INIT          = 26,
-	FUSE_OPENDIR       = 27,
-	FUSE_READDIR       = 28,
-	FUSE_RELEASEDIR    = 29,
-	FUSE_FSYNCDIR      = 30,
-	FUSE_GETLK         = 31,
-	FUSE_SETLK         = 32,
-	FUSE_SETLKW        = 33,
-	FUSE_ACCESS        = 34,
-	FUSE_CREATE        = 35,
-	FUSE_INTERRUPT     = 36,
-	FUSE_BMAP          = 37,
-	FUSE_DESTROY       = 38,
-#if (__FreeBSD__ >= 10)
+        FUSE_LOOKUP        = 1,
+        FUSE_FORGET        = 2,  /* no reply */
+        FUSE_GETATTR       = 3,
+        FUSE_SETATTR       = 4,
+        FUSE_READLINK      = 5,
+        FUSE_SYMLINK       = 6,
+        FUSE_MKNOD         = 8,
+        FUSE_MKDIR         = 9,
+        FUSE_UNLINK        = 10,
+        FUSE_RMDIR         = 11,
+        FUSE_RENAME        = 12,
+        FUSE_LINK          = 13,
+        FUSE_OPEN          = 14,
+        FUSE_READ          = 15,
+        FUSE_WRITE         = 16,
+        FUSE_STATFS        = 17,
+        FUSE_RELEASE       = 18,
+        FUSE_FSYNC         = 20,
+        FUSE_SETXATTR      = 21,
+        FUSE_GETXATTR      = 22,
+        FUSE_LISTXATTR     = 23,
+        FUSE_REMOVEXATTR   = 24,
+        FUSE_FLUSH         = 25,
+        FUSE_INIT          = 26,
+        FUSE_OPENDIR       = 27,
+        FUSE_READDIR       = 28,
+        FUSE_RELEASEDIR    = 29,
+        FUSE_FSYNCDIR      = 30,
+        FUSE_GETLK         = 31,
+        FUSE_SETLK         = 32,
+        FUSE_SETLKW        = 33,
+        FUSE_ACCESS        = 34,
+        FUSE_CREATE        = 35,
+        FUSE_INTERRUPT     = 36,
+        FUSE_BMAP          = 37,
+        FUSE_DESTROY       = 38,
+        /*
+        FUSE_IOCTL         = 39,
+        FUSE_POLL          = 40,
+        FUSE_NOTIFY_REPLY  = 41,
+        FUSE_BATCH_FORGET  = 42,
+        FUSE_FALLOCATE     = 43,
+        FUSE_READDIRPLUS   = 44,
+        */
+
         FUSE_SETVOLNAME    = 61,
-	FUSE_GETXTIMES     = 62,
-	FUSE_EXCHANGE      = 63,
-#endif /* __FreeBSD__ >= 10 */
+        FUSE_GETXTIMES     = 62,
+        FUSE_EXCHANGE      = 63,
 };
 
 /* The read buffer is required to be at least 8k, but may be much larger */
 #define FUSE_MIN_READ_BUFFER 8192
 
 struct fuse_entry_out {
-	__u64	nodeid;		/* Inode ID */
-	__u64	generation;	/* Inode generation: nodeid:gen must
-				   be unique for the fs's lifetime */
-	__u64	entry_valid;	/* Cache timeout for the name */
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	entry_valid_nsec;
-	__u32	attr_valid_nsec;
-	struct fuse_attr attr;
+        __u64   nodeid;         /* Inode ID */
+        __u64   generation;     /* Inode generation: nodeid:gen must
+                                   be unique for the fs's lifetime */
+        __u64   entry_valid;    /* Cache timeout for the name */
+        __u64   attr_valid;     /* Cache timeout for the attributes */
+        __u32   entry_valid_nsec;
+        __u32   attr_valid_nsec;
+        struct fuse_attr attr;
 };
 
 struct fuse_forget_in {
-	__u64	nlookup;
+        __u64   nlookup;
 };
 
 struct fuse_attr_out {
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	attr_valid_nsec;
-	__u32	dummy;
-	struct fuse_attr attr;
+        __u64   attr_valid;     /* Cache timeout for the attributes */
+        __u32   attr_valid_nsec;
+        __u32   dummy;
+        struct fuse_attr attr;
 };
 
-#if (__FreeBSD__ >= 10)
 struct fuse_getxtimes_out {
-	__u64	bkuptime;
-	__u64	crtime;
-	__u32	bkuptimensec;
-	__u32	crtimensec;
+        __u64   bkuptime;
+        __u64   crtime;
+        __u32   bkuptimensec;
+        __u32   crtimensec;
 };
-#endif /* __FreeBSD__ >= 10 */
 
 struct fuse_mknod_in {
-	__u32	mode;
-	__u32	rdev;
+        __u32   mode;
+        __u32   rdev;
 };
 
 struct fuse_mkdir_in {
-	__u32	mode;
-	__u32	padding;
+        __u32   mode;
+        __u32   padding;
 };
 
 struct fuse_rename_in {
-	__u64	newdir;
+        __u64   newdir;
 };
 
-#if (__FreeBSD__ >= 10)
 struct fuse_exchange_in {
-	__u64	olddir;
-	__u64	newdir;
-	__u64	options;
+        __u64   olddir;
+        __u64   newdir;
+        __u64   options;
 };
-#endif /* __FreeBSD__ >= 10 */
 
 struct fuse_link_in {
-	__u64	oldnodeid;
+        __u64   oldnodeid;
 };
 
 struct fuse_setattr_in {
-	__u32	valid;
-	__u32	padding;
-	__u64	fh;
-	__u64	size;
-	__u64	unused1;
-	__u64	atime;
-	__u64	mtime;
-	__u64	unused2;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	unused3;
-	__u32	mode;
-	__u32	unused4;
-	__u32	uid;
-	__u32	gid;
-	__u32	unused5;
-#if (__FreeBSD__ >= 10)
-	__u64	bkuptime;
-	__u64	chgtime;
-	__u64	crtime;
-	__u32	bkuptimensec;
-	__u32	chgtimensec;
-	__u32	crtimensec;
-	__u32	flags; /* file flags; see chflags(2) */
-#endif /* __FreeBSD__ >= 10 */
+        __u32   valid;
+        __u32   padding;
+        __u64   fh;
+        __u64   size;
+        __u64   unused1;
+        __u64   atime;
+        __u64   mtime;
+        __u64   unused2;
+        __u32   atimensec;
+        __u32   mtimensec;
+        __u32   unused3;
+        __u32   mode;
+        __u32   unused4;
+        __u32   uid;
+        __u32   gid;
+        __u32   unused5;
+        __u64   bkuptime;
+        __u64   chgtime;
+        __u64   crtime;
+        __u32   bkuptimensec;
+        __u32   chgtimensec;
+        __u32   crtimensec;
+        __u32   flags; /* file flags; see chflags(2) */
 };
 
 struct fuse_open_in {
-	__u32	flags;
-	__u32	mode;
+        __u32   flags;
+        __u32   mode;
 };
 
 struct fuse_open_out {
-	__u64	fh;
-	__u32	open_flags;
-	__u32	padding;
+        __u64   fh;
+        __u32   open_flags;
+        __u32   padding;
 };
 
 struct fuse_release_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	release_flags;
-	__u64	lock_owner;
+        __u64   fh;
+        __u32   flags;
+        __u32   release_flags;
+        __u64   lock_owner;
 };
 
 struct fuse_flush_in {
-	__u64	fh;
-	__u32	unused;
-	__u32	padding;
-	__u64	lock_owner;
+        __u64   fh;
+        __u32   unused;
+        __u32   padding;
+        __u64   lock_owner;
 };
 
 struct fuse_read_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	padding;
+        __u64   fh;
+        __u64   offset;
+        __u32   size;
+        __u32   padding;
 };
 
 struct fuse_write_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	write_flags;
+        __u64   fh;
+        __u64   offset;
+        __u32   size;
+        __u32   write_flags;
 };
 
 struct fuse_write_out {
-	__u32	size;
-	__u32	padding;
+        __u32   size;
+        __u32   padding;
 };
 
 #define FUSE_COMPAT_STATFS_SIZE 48
 
 struct fuse_statfs_out {
-	struct fuse_kstatfs st;
+        struct fuse_kstatfs st;
 };
 
 struct fuse_fsync_in {
-	__u64	fh;
-	__u32	fsync_flags;
-	__u32	padding;
+        __u64   fh;
+        __u32   fsync_flags;
+        __u32   padding;
 };
 
 struct fuse_setxattr_in {
-	__u32	size;
-	__u32	flags;
-#if (__FreeBSD__ >= 10)
-	__u32	position;
-	__u32	padding;
-#endif /* __FreeBSD__ >= 10 */
+        __u32   size;
+        __u32   flags;
+        __u32   position;
+        __u32   padding;
 };
 
 struct fuse_getxattr_in {
-	__u32	size;
-	__u32	padding;
-#if (__FreeBSD__ >= 10)
-	__u32	position;
-	__u32	padding2;
-#endif /* __FreeBSD__ >= 10 */
+        __u32   size;
+        __u32   padding;
+        __u32   position;
+        __u32   padding2;
 };
 
 struct fuse_getxattr_out {
-	__u32	size;
-	__u32	padding;
+        __u32   size;
+        __u32   padding;
 };
 
 struct fuse_lk_in {
-	__u64	fh;
-	__u64	owner;
-	struct fuse_file_lock lk;
+        __u64   fh;
+        __u64   owner;
+        struct fuse_file_lock lk;
 };
 
 struct fuse_lk_out {
-	struct fuse_file_lock lk;
+        struct fuse_file_lock lk;
 };
 
 struct fuse_access_in {
-	__u32	mask;
-	__u32	padding;
+        __u32   mask;
+        __u32   padding;
 };
 
 struct fuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
+        __u32   major;
+        __u32   minor;
+        __u32   max_readahead;
+        __u32   flags;
 };
 
 struct fuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
-	__u32	unused;
-	__u32	max_write;
+        __u32   major;
+        __u32   minor;
+        __u32   max_readahead;
+        __u32   flags;
+        __u32   unused;
+        __u32   max_write;
 };
 
 struct fuse_interrupt_in {
-	__u64	unique;
+        __u64   unique;
 };
 
 struct fuse_bmap_in {
-	__u64	block;
-	__u32	blocksize;
-	__u32	padding;
+        __u64   block;
+        __u32   blocksize;
+        __u32   padding;
 };
 
 struct fuse_bmap_out {
-	__u64	block;
+        __u64   block;
 };
 
 struct fuse_in_header {
-	__u32	len;
-	__u32	opcode;
-	__u64	unique;
-	__u64	nodeid;
-	__u32	uid;
-	__u32	gid;
-	__u32	pid;
-	__u32	padding;
+        __u32   len;
+        __u32   opcode;
+        __u64   unique;
+        __u64   nodeid;
+        __u32   uid;
+        __u32   gid;
+        __u32   pid;
+        __u32   padding;
 };
 
 struct fuse_out_header {
-	__u32	len;
-	__s32	error;
-	__u64	unique;
+        __u32   len;
+        __s32   error;
+        __u64   unique;
 };
 
 struct fuse_dirent {
-	__u64	ino;
-	__u64	off;
-	__u32	namelen;
-	__u32	type;
-	char name[0];
+        __u64   ino;
+        __u64   off;
+        __u32   namelen;
+        __u32   type;
+        char name[0];
 };
 
 #define FUSE_NAME_OFFSET offsetof(struct fuse_dirent, name)
 #define FUSE_DIRENT_ALIGN(x) (((x) + sizeof(__u64) - 1) & ~(sizeof(__u64) - 1))
 #define FUSE_DIRENT_SIZE(d) \
-	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
+        FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
diff --git a/contrib/fuse-util/mount_util.h b/contrib/fuse-include/mount_util.h
index f392f99f17a..f392f99f17a 100644
--- a/contrib/fuse-util/mount_util.h
+++ b/contrib/fuse-include/mount_util.h
diff --git a/contrib/fuse-lib/misc.c b/contrib/fuse-lib/misc.c
index 28a9284bf96..1a9b418e511 100644
--- a/contrib/fuse-lib/misc.c
+++ b/contrib/fuse-lib/misc.c
@@ -10,7 +10,7 @@
 #include <string.h>
 #include <limits.h>
 #include <fcntl.h>
-#include "glusterfs.h"
+#include "glusterfs/glusterfs.h"
 #include "fuse_kernel.h"
 #include "fuse-misc.h"
 
@@ -41,7 +41,6 @@ void
 convert_fuse_file_lock (struct fuse_file_lock *fl, struct gf_flock *flock,
                         uint64_t lk_owner)
 {
-        memset (flock, 0, sizeof (struct flock));
         flock->l_type = fl->type;
         flock->l_whence = SEEK_SET;
         flock->l_start = fl->start;
@@ -50,5 +49,5 @@ convert_fuse_file_lock (struct fuse_file_lock *fl, struct gf_flock *flock,
         else
                 flock->l_len = fl->end - fl->start + 1;
         flock->l_pid = fl->pid;
-        flock->l_owner = lk_owner;
+        set_lk_owner_from_uint64 (&flock->l_owner, lk_owner);
 }
diff --git a/contrib/fuse-lib/mount-common.c b/contrib/fuse-lib/mount-common.c
new file mode 100644
index 00000000000..cffd4c01ed5
--- /dev/null
+++ b/contrib/fuse-lib/mount-common.c
@@ -0,0 +1,282 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include "mount-gluster-compat.h"
+
+/*
+ * These functions (and gf_fuse_umount() in mount.c)
+ * were originally taken from libfuse as of commit 7960e99e
+ * (http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=commit;h=7960e99e)
+ * almost verbatim. What has been changed upon adoption:
+ * - style adopted to that of glusterfs
+ * - s/fprintf/gf_log/
+ * - s/free/FREE/, s/malloc/MALLOC/
+ * - there are some other minor things
+ *
+ * For changes that were made later and syncs with upstream,
+ * see the commit log and per-function comments.
+ */
+
+#ifdef GF_LINUX_HOST_OS
+/* FUSE: cherry-picked bd99f9cf */
+static int
+mtab_needs_update (const char *mnt)
+{
+        int res;
+        struct stat stbuf;
+
+        /* If mtab is within new mount, don't touch it */
+        if (strncmp (mnt, _PATH_MOUNTED, sizeof (_PATH_MOUNTED) - 1) == 0 &&
+            _PATH_MOUNTED[strlen (mnt)] == '/')
+                return 0;
+
+        /*
+         * Skip mtab update if /etc/mtab:
+         *
+         *  - doesn't exist,
+         *  - is a symlink,
+         *  - is on a read-only filesystem.
+         */
+        res = lstat (_PATH_MOUNTED, &stbuf);
+        if (res == -1) {
+                if (errno == ENOENT)
+                        return 0;
+        } else {
+                uid_t ruid;
+                int err;
+
+                if (S_ISLNK (stbuf.st_mode))
+                        return 0;
+
+                ruid = getuid ();
+                if (ruid != 0)
+                        setreuid (0, -1);
+
+                res = access (_PATH_MOUNTED, W_OK);
+                err = (res == -1) ? errno : 0;
+                if (ruid != 0)
+                        setreuid (ruid, -1);
+
+                if (err == EROFS)
+                        return 0;
+        }
+
+        return 1;
+}
+#else /* GF_LINUX_HOST_OS */
+#define mtab_needs_update(x) 1
+#endif /* GF_LINUX_HOST_OS */
+
+/* FUSE: called add_mount_legacy(); R.I.P. as of cbd3a2a8 */
+int
+fuse_mnt_add_mount (const char *progname, const char *fsname,
+                    const char *mnt, const char *type, const char *opts)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (mnt))
+                return 0;
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s",
+                               progname, strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                char templ[] = "/tmp/fusermountXXXXXX";
+                char *tmp;
+
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                res = setuid (geteuid ());
+                if (res != 0) {
+                        GFFUSE_LOGERR ("%s: setuid: %s", progname, strerror (errno));
+                        exit (1);
+                }
+
+                /*
+                 * hide in a directory, where mount isn't able to resolve
+                 * fsname as a valid path
+                 */
+                tmp = mkdtemp (templ);
+                if (!tmp) {
+                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
+                                       progname);
+                        exit (1);
+                }
+                if (chdir (tmp)) {
+                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
+                                       progname, tmp, strerror (errno));
+                        exit (1);
+                }
+                rmdir (tmp);
+                execl (_PATH_MOUNT, _PATH_MOUNT, "-i", "-f", "-t", type,
+                       "-o", opts, fsname, mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute %s: %s",
+                               progname, _PATH_MOUNT, strerror (errno));
+                exit (1);
+        }
+
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+        res = (res != -1 && status == 0) ? 0 : -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
+
+char *
+fuse_mnt_resolve_path (const char *progname, const char *orig)
+{
+        char buf[PATH_MAX];
+        char *copy;
+        char *dst;
+        char *end;
+        char *lastcomp;
+        const char *toresolv;
+
+        if (!orig[0]) {
+                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
+                return NULL;
+        }
+
+        copy = strdup (orig);
+        if (copy == NULL) {
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+                return NULL;
+        }
+
+        toresolv = copy;
+        lastcomp = NULL;
+        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
+        if (end[0] != '/') {
+                char *tmp;
+                end[1] = '\0';
+                tmp = strrchr (copy, '/');
+                if (tmp == NULL) {
+                        lastcomp = copy;
+                        toresolv = ".";
+                } else {
+                        lastcomp = tmp + 1;
+                        if (tmp == copy)
+                                toresolv = "/";
+                }
+                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
+                        lastcomp = NULL;
+                        toresolv = copy;
+                }
+                else if (tmp)
+                        tmp[0] = '\0';
+        }
+        if (realpath (toresolv, buf) == NULL) {
+                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
+                               strerror (errno));
+                FREE (copy);
+                return NULL;
+        }
+        if (lastcomp == NULL)
+                dst = strdup (buf);
+        else {
+                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
+                if (dst) {
+                        unsigned buflen = strlen (buf);
+                        if (buflen && buf[buflen-1] == '/')
+                                sprintf (dst, "%s%s", buf, lastcomp);
+                        else
+                                sprintf (dst, "%s/%s", buf, lastcomp);
+                }
+        }
+        FREE (copy);
+        if (dst == NULL)
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+        return dst;
+}
+
+/* FUSE: to support some changes that were reverted since
+ * then, it was split in two (fuse_mnt_umount() and
+ * exec_umount()); however the actual code is same as here
+ * since 0197ce40
+ */
+int
+fuse_mnt_umount (const char *progname, const char *abs_mnt,
+                 const char *rel_mnt, int lazy)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (abs_mnt)) {
+                res = umount2 (rel_mnt, lazy ? 2 : 0);
+                if (res == -1)
+                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
+                                       progname, abs_mnt, strerror (errno));
+                return res;
+        }
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
+                               strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                res = setuid (geteuid ());
+                if (res != 0) {
+                        GFFUSE_LOGERR ("%s: setuid: %s", progname, strerror (errno));
+                        exit (1);
+                }
+#ifdef GF_LINUX_HOST_OS
+                execl ("umount", "umount", "-i", rel_mnt,
+                       lazy ? "-l" : NULL, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
+                               progname, strerror (errno));
+#elif __NetBSD__
+                /* exitting the filesystem causes the umount */
+                exit (0);
+#else
+                execl ("umount", "umount", "-f", rel_mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
+                               progname, strerror (errno));
+#endif /* GF_LINUX_HOST_OS */
+                exit (1);
+        }
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+
+        if (status != 0)
+                res = -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
diff --git a/contrib/fuse-lib/mount-gluster-compat.h b/contrib/fuse-lib/mount-gluster-compat.h
new file mode 100644
index 00000000000..d3646d08d8e
--- /dev/null
+++ b/contrib/fuse-lib/mount-gluster-compat.h
@@ -0,0 +1,105 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stddef.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <dirent.h>
+#include <signal.h>
+#if defined(GF_LINUX_HOST_OS)
+#include <mntent.h>
+#endif /* GF_LINUX_HOST_OS */
+#include <sys/stat.h>
+#include <sys/poll.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <sys/mount.h>
+
+#ifdef GF_LINUX_HOST_OS
+typedef unsigned long mount_flag_t;
+#endif
+
+#if defined(__NetBSD__)
+#include <perfuse.h>
+#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+#define MS_NODEV  MNT_NODEV
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+typedef int mount_flag_t;
+#endif
+
+#if defined(GF_DARWIN_HOST_OS) || defined(__FreeBSD__)
+#include <sys/param.h>
+#include <sys/mount.h>
+#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#endif
+
+#if defined(__FreeBSD__)
+#define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+/* "nodev"/MNT_NODEV was removed from FreBSD, as it became unneeded because "As
+ * of FreeBSD 6.0 device nodes may be created in regular file systems but such
+ * nodes cannot be used to access devices." (See
+ * https://freebsd.org/cgi/man.cgi?query=mknod&sektion=8 .
+ * Also see:
+ * - https://github.com/freebsd/freebsd/commit/266790a
+ * - https://github.com/freebsd/freebsd/commit/a5e716d
+ * - 700008 in
+ *   https://www.freebsd.org/doc/en/books/porters-handbook/versions-7.html .)
+ */
+#if __FreeBSD_version < 700008
+#define MS_NODEV  MNT_NODEV
+#else
+#define MS_NODEV  0
+#endif
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+#if __FreeBSD_version < 1000715
+typedef int  mount_flag_t;
+#else
+/* __FreeBSD_version was not bumped for this type change. Anyway, see
+ * https://github.com/freebsd/freebsd/commit/e8d76f8
+ * and respective __FreeBSD_version:
+ * https://github.com/freebsd/freebsd/blob/e8d76f8/sys/sys/param.h#L61 .
+ * We use the subsequent value, 1000715, to switch. (Also see:
+ * https://www.freebsd.org/doc/en/books/porters-handbook/versions-10.html .)
+ */
+typedef long long mount_flag_t;
+#endif
+#endif
+
+#ifdef GF_LINUX_HOST_OS
+#define _PATH_MOUNT "/bin/mount"
+#else /* FreeBSD, NetBSD, MacOS X */
+#define _PATH_MOUNT "/sbin/mount"
+#endif
+
+#ifdef FUSE_UTIL
+#define MALLOC(size) malloc (size)
+#define FREE(ptr) free (ptr)
+#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
+#else /* FUSE_UTIL */
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
+
+#define GFFUSE_LOGERR(...) \
+        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
+#endif /* !FUSE_UTIL */
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 83b4d4c3e45..06ff191f542 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -7,311 +7,197 @@
   See the file COPYING.LIB.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stddef.h>
-#include <limits.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <dirent.h>
-#include <mntent.h>
-#include <sys/stat.h>
-#include <sys/poll.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/wait.h>
-#include <sys/mount.h>
-
-#ifdef FUSE_UTIL
-#define MALLOC(size) malloc (size)
-#define FREE(ptr) free (ptr)
-#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
-#else /* FUSE_UTIL */
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "mount_util.h"
+#include "mount-gluster-compat.h"
 
 #ifdef GF_FUSERMOUNT
 #define FUSERMOUNT_PROG FUSERMOUNT_DIR "/fusermount-glusterfs"
 #else
 #define FUSERMOUNT_PROG "fusermount"
 #endif
-#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
+#define FUSE_DEVFD_ENV "_FUSE_DEVFD"
 
-#define GFFUSE_LOGERR(...) \
-        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
-#endif /* !FUSE_UTIL */
-
-/*
- * Functions below, until following note, were taken from libfuse
- * (http://git.gluster.com/?p=users/csaba/fuse.git;a=commit;h=b988bbf9)
- * almost verbatim. What has been changed:
- * - style adopted to that of glusterfs
- * - s/fprintf/gf_log/
- * - s/free/FREE/, s/malloc/MALLOC/
- * - there are some other minor things
- */
+#ifdef __FreeBSD__
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#endif /* __FreeBSD__ */
 
-static int
-mtab_needs_update (const char *mnt)
+/* FUSE: function is called fuse_kern_unmount() */
+void
+gf_fuse_unmount (const char *mountpoint, int fd)
 {
         int res;
-        struct stat stbuf;
-
-        /* If mtab is within new mount, don't touch it */
-        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
-            _PATH_MOUNTED[strlen (mnt)] == '/')
-                return 0;
-
-        /*
-         * Skip mtab update if /etc/mtab:
-         *
-         *  - doesn't exist,
-         *  - is a symlink,
-         *  - is on a read-only filesystem.
-         */
-        res = lstat (_PATH_MOUNTED, &stbuf);
-        if (res == -1) {
-                if (errno == ENOENT)
-                        return 0;
-        } else {
-                if (S_ISLNK (stbuf.st_mode))
-                        return 0;
+        int pid;
+
+        if (!mountpoint)
+                return;
+
+        if (fd != -1) {
+                struct pollfd pfd;
 
-                res = access (_PATH_MOUNTED, W_OK);
-                if (res == -1 && errno == EROFS)
-                        return 0;
+                pfd.fd = fd;
+                pfd.events = 0;
+                res = poll (&pfd, 1, 0);
+                /* If file poll returns POLLERR on the device file descriptor,
+                   then the filesystem is already unmounted */
+                if (res == 1 && (pfd.revents & POLLERR))
+                        return;
+
+                /* Need to close file descriptor, otherwise synchronous umount
+                   would recurse into filesystem, and deadlock */
+                close (fd);
         }
 
-        return 1;
+        if (geteuid () == 0) {
+                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
+                return;
+        } else {
+            GFFUSE_LOGERR ("fuse: Effective-uid: %d", geteuid());
+        }
+
+        res = umount2 (mountpoint, 2);
+        if (res == 0)
+                return;
+
+        GFFUSE_LOGERR ("fuse: failed to unmount %s: %s",
+                       mountpoint, strerror (errno));
+        pid = fork ();
+        if (pid == -1)
+                return;
+
+        if (pid == 0) {
+                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
+                                       "--", mountpoint, NULL };
+
+                execvp (FUSERMOUNT_PROG, (char **)argv);
+                GFFUSE_LOGERR ("fuse: failed to execute fuserumount: %s",
+                               strerror (errno));
+                _exit (1);
+        }
+        waitpid (pid, NULL, 0);
 }
 
-#ifndef FUSE_UTIL
-static
-#endif
+
+/* gluster-specific routines */
+
+/* Unmounting in a daemon that lurks 'till main process exits */
 int
-fuse_mnt_add_mount (const char *progname, const char *fsname,
-                    const char *mnt, const char *type, const char *opts)
+gf_fuse_unmount_daemon (const char *mountpoint, int fd)
 {
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (mnt))
-                return 0;
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s",
-                               progname, strerror (errno));
+        int   ret = -1;
+        pid_t pid = -1;
+
+        if (fd == -1)
                 return -1;
-        }
 
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                char templ[] = "/tmp/fusermountXXXXXX";
-                char *tmp;
-
-                /* mtab update done async, just log if fails */
-                res = fork ();
-                if (res)
-                        exit (res == -1 ? 1 : 0);
-                res = fork ();
-                if (res) {
-                        if (res != -1)
-                                res = waitpid (res, &status, 0);
-                        if (res == -1)
-                                GFFUSE_LOGERR ("%s: /etc/mtab update failed",
-                                               progname);
-
-                        exit (0);
-                }
+        int ump[2] = {0,};
 
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-
-                /*
-                 * hide in a directory, where mount isn't able to resolve
-                 * fsname as a valid path
-                 */
-                tmp = mkdtemp (templ);
-                if (!tmp) {
-                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
-                                       progname);
-                        exit (1);
-                }
-                if (chdir (tmp)) {
-                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
-                                       progname, tmp, strerror (errno));
-                        exit (1);
-                }
-                rmdir (tmp);
-                execl ("/bin/mount", "/bin/mount", "-i", "-f", "-t", type,
-                       "-o", opts, fsname, mnt, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/mount: %s",
-                               progname, strerror (errno));
-                exit (1);
+        ret = pipe(ump);
+        if (ret == -1) {
+                close (fd);
+                return -1;
         }
-        res = waitpid (res, &status, 0);
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
 
-        if (status != 0)
-                res = -1;
+        pid = fork ();
+        switch (pid) {
+        case 0:
+        {
+                char c = 0;
+                sigset_t sigset;
+
+                close_fds_except (ump, 1);
+
+                setsid();
+                (void)chdir("/");
+                sigfillset(&sigset);
+                sigprocmask(SIG_BLOCK, &sigset, NULL);
+
+                read (ump[0], &c, 1);
 
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
+                gf_fuse_unmount (mountpoint, fd);
+                exit (0);
+        }
+        case -1:
+                close (fd);
+                fd = -1;
+                ret = -1;
+                close (ump[1]);
+        }
+        close (ump[0]);
+
+        return ret;
 }
 
-#ifndef FUSE_UTIL
-static
-#endif
-char
-*fuse_mnt_resolve_path (const char *progname, const char *orig)
+static char *
+escape (char *s)
 {
-        char buf[PATH_MAX];
-        char *copy;
-        char *dst;
-        char *end;
-        char *lastcomp;
-        const char *toresolv;
-
-        if (!orig[0]) {
-                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
-                return NULL;
-        }
+        size_t len = 0;
+        char *p = NULL;
+        char *q = NULL;
+        char *e = NULL;
 
-        copy = strdup (orig);
-        if (copy == NULL) {
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-                return NULL;
+        for (p = s; *p; p++) {
+                if (*p == ',')
+                       len++;
+                len++;
         }
 
-        toresolv = copy;
-        lastcomp = NULL;
-        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
-        if (end[0] != '/') {
-                char *tmp;
-                end[1] = '\0';
-                tmp = strrchr (copy, '/');
-                if (tmp == NULL) {
-                        lastcomp = copy;
-                        toresolv = ".";
-                } else {
-                        lastcomp = tmp + 1;
-                        if (tmp == copy)
-                                toresolv = "/";
-                }
-                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
-                        lastcomp = NULL;
-                        toresolv = copy;
-                }
-                else if (tmp)
-                        tmp[0] = '\0';
-        }
-        if (realpath (toresolv, buf) == NULL) {
-                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
-                               strerror (errno));
-                FREE (copy);
+        e = CALLOC (1, len + 1);
+        if (!e)
                 return NULL;
-        }
-        if (lastcomp == NULL)
-                dst = strdup (buf);
-        else {
-                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
-                if (dst) {
-                        unsigned buflen = strlen (buf);
-                        if (buflen && buf[buflen-1] == '/')
-                                sprintf (dst, "%s%s", buf, lastcomp);
-                        else
-                                sprintf (dst, "%s/%s", buf, lastcomp);
+
+        for (p = s, q = e; *p; p++, q++) {
+                if (*p == ',') {
+                        *q = '\\';
+                        q++;
                 }
+                *q = *p;
         }
-        FREE (copy);
-        if (dst == NULL)
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-        return dst;
+
+        return e;
 }
 
-#ifndef FUSE_UTIL
-/* return value:
- * >= 0         => fd
- * -1         => error
- */
 static int
-receive_fd (int fd)
+fuse_mount_fusermount (const char *mountpoint, char *fsname,
+                       char *mnt_param, int fd)
 {
-        struct msghdr msg;
-        struct iovec iov;
-        char buf[1];
-        int rv;
-        size_t ccmsg[CMSG_SPACE (sizeof (int)) / sizeof (size_t)];
-        struct cmsghdr *cmsg;
-
-        iov.iov_base = buf;
-        iov.iov_len = 1;
-
-        msg.msg_name = 0;
-        msg.msg_namelen = 0;
-        msg.msg_iov = &iov;
-        msg.msg_iovlen = 1;
-        /* old BSD implementations should use msg_accrights instead of
-         * msg_control; the interface is different. */
-        msg.msg_control = ccmsg;
-        msg.msg_controllen = sizeof (ccmsg);
-
-        while (((rv = recvmsg (fd, &msg, 0)) == -1) && errno == EINTR);
-        if (rv == -1) {
-                GFFUSE_LOGERR ("recvmsg failed: %s", strerror (errno));
-                return -1;
-        }
-        if (!rv) {
-                /* EOF */
-                return -1;
-        }
+        int  pid = -1;
+        int  res = 0;
+        int  ret = -1;
+        char *fm_mnt_params = NULL;
+        char *efsname = NULL;
 
-        cmsg = CMSG_FIRSTHDR (&msg);
-        if (!cmsg->cmsg_type == SCM_RIGHTS) {
-                GFFUSE_LOGERR ("got control message of unknown type %d",
-                               cmsg->cmsg_type);
-                return -1;
-        }
-        return *(int*)CMSG_DATA (cmsg);
-}
+#ifndef GF_FUSERMOUNT
+        GFFUSE_LOGERR ("Mounting via helper utility "
+                       "(unprivileged mounting) is supported "
+                       "only if glusterfs is compiled with "
+                       "--enable-fusermount");
+        return -1;
+#endif
 
-static int
-fuse_mount_fusermount (const char *mountpoint, const char *opts)
-{
-        int fds[2], pid;
-        int res;
-        int rv;
+        efsname = escape (fsname);
+        if (!efsname) {
+                GFFUSE_LOGERR ("Out of memory");
 
-        res = socketpair (PF_UNIX, SOCK_STREAM, 0, fds);
-        if (res == -1) {
-                GFFUSE_LOGERR ("socketpair() failed: %s", strerror (errno));
                 return -1;
         }
+        ret = asprintf (&fm_mnt_params,
+                        "%s,fsname=%s,nonempty,subtype=glusterfs",
+                        mnt_param, efsname);
+        FREE (efsname);
+        if (ret == -1) {
+                GFFUSE_LOGERR ("Out of memory");
+
+                goto out;
+        }
 
+        /* fork to exec fusermount */
         pid = fork ();
         if (pid == -1) {
                 GFFUSE_LOGERR ("fork() failed: %s", strerror (errno));
-                close (fds[0]);
-                close (fds[1]);
-                return -1;
+                ret = -1;
+                goto out;
         }
 
         if (pid == 0) {
@@ -320,224 +206,197 @@ fuse_mount_fusermount (const char *mountpoint, const char *opts)
                 int a = 0;
 
                 argv[a++] = FUSERMOUNT_PROG;
-                if (opts) {
-                        argv[a++] = "-o";
-                        argv[a++] = opts;
-                }
+                argv[a++] = "-o";
+                argv[a++] = fm_mnt_params;
                 argv[a++] = "--";
                 argv[a++] = mountpoint;
                 argv[a++] = NULL;
 
-                close (fds[1]);
-                fcntl (fds[0], F_SETFD, 0);
-                snprintf (env, sizeof (env), "%i", fds[0]);
-                setenv (FUSE_COMMFD_ENV, env, 1);
+                snprintf (env, sizeof (env), "%i", fd);
+                setenv (FUSE_DEVFD_ENV, env, 1);
                 execvp (FUSERMOUNT_PROG, (char **)argv);
                 GFFUSE_LOGERR ("failed to exec fusermount: %s",
                                strerror (errno));
                 _exit (1);
         }
 
-        close (fds[0]);
-        rv = receive_fd (fds[1]);
-        close (fds[1]);
-        waitpid (pid, NULL, 0); /* bury zombie */
-
-        return rv;
+        ret = waitpid (pid, &res, 0);
+        ret = (ret == pid && res == 0) ? 0 : -1;
+ out:
+        FREE (fm_mnt_params);
+        return ret;
 }
-#endif
 
-#ifndef FUSE_UTIL
-static
-#endif
-int
-fuse_mnt_umount (const char *progname, const char *abs_mnt,
-                 const char *rel_mnt, int lazy)
+#if defined(__FreeBSD__)
+void
+build_iovec(struct iovec **iov, int *iovlen, const char *name, void *val,
+            size_t len)
 {
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (abs_mnt)) {
-                res = umount2 (rel_mnt, lazy ? 2 : 0);
-                if (res == -1)
-                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
-                                       progname, abs_mnt, strerror (errno));
-                return res;
-        }
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
-                               strerror (errno));
-                return -1;
-        }
+        int i;
+        if (*iovlen < 0)
+                return;
+        i = *iovlen;
 
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
-                      lazy ? "-l" : NULL, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
-                               progname, strerror (errno));
-                exit (1);
+        *iov = realloc(*iov, sizeof **iov * (i + 2));
+        if (*iov == NULL) {
+                *iovlen = -1;
+                return;
         }
-        res = waitpid (res, &status, 0);
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
 
-        if (status != 0)
-                res = -1;
-
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
-}
+        (*iov)[i].iov_base = strdup(name);
+        (*iov)[i].iov_len = strlen(name) + 1;
 
-#ifdef FUSE_UTIL
-int
-fuse_mnt_check_empty (const char *progname, const char *mnt,
-                      mode_t rootmode, off_t rootsize)
-{
-        int isempty = 1;
-
-        if (S_ISDIR (rootmode)) {
-                struct dirent *ent;
-                DIR *dp = opendir (mnt);
-                if (dp == NULL) {
-                        fprintf (stderr,
-                                 "%s: failed to open mountpoint for reading: %s\n",
-                                 progname, strerror (errno));
-                        return -1;
-                }
-                while ((ent = readdir (dp)) != NULL) {
-                        if (strcmp (ent->d_name, ".") != 0 &&
-                            strcmp (ent->d_name, "..") != 0) {
-                                isempty = 0;
-                                break;
-                        }
-                }
-                closedir (dp);
-        } else if (rootsize)
-                isempty = 0;
-
-        if (!isempty) {
-                fprintf (stderr, "%s: mountpoint is not empty\n", progname);
-                fprintf (stderr, "%s: if you are sure this is safe, "
-                         "use the 'nonempty' mount option\n", progname);
-                return -1;
+        i++;
+        (*iov)[i].iov_base = val;
+        if (len == (size_t) -1) {
+                if (val != NULL)
+                        len = strlen(val) + 1;
+                else
+                        len = 0;
         }
-        return 0;
+        (*iov)[i].iov_len = (int)len;
+        *iovlen = ++i;
 }
 
-int
-fuse_mnt_check_fuseblk (void)
+/*
+ * This function is needed for compatibility with parameters
+ * which used to use the mount_argf() command for the old mount() syscall.
+ */
+void
+build_iovec_argf(struct iovec **iov, int *iovlen, const char *name,
+                 const char *fmt, ...)
 {
-        char buf[256];
-        FILE *f = fopen ("/proc/filesystems", "r");
-        if (!f)
-                return 1;
-
-        while (fgets (buf, sizeof (buf), f))
-                if (strstr (buf, "fuseblk\n")) {
-                        fclose (f);
-                        return 1;
-                }
+        va_list ap;
+        char val[255] = { 0 };
 
-        fclose (f);
-        return 0;
+        va_start(ap, fmt);
+        vsnprintf(val, sizeof(val), fmt, ap);
+        va_end(ap);
+        build_iovec(iov, iovlen, name, strdup(val), (size_t)-1);
 }
+#endif /* __FreeBSD__ */
+
+struct mount_flags {
+        const char *opt;
+        mount_flag_t flag;
+        int on;
+} mount_flags[] = {
+        /* We provide best effort cross platform support for mount flags by
+         * defining the ones which are commonly used in Unix-like OS-es.
+         */
+        {"ro",      MS_RDONLY,      1},
+        {"nosuid",  MS_NOSUID,      1},
+        {"nodev",   MS_NODEV,       1},
+        {"noatime", MS_NOATIME,     1},
+        {"noexec",  MS_NOEXEC,      1},
+#ifdef GF_LINUX_HOST_OS
+        {"rw",      MS_RDONLY,      0},
+        {"suid",    MS_NOSUID,      0},
+        {"dev",     MS_NODEV,       0},
+        {"exec",    MS_NOEXEC,      0},
+        {"async",   MS_SYNCHRONOUS, 0},
+        {"sync",    MS_SYNCHRONOUS, 1},
+        {"atime",   MS_NOATIME,     0},
+        {"dirsync", MS_DIRSYNC,     1},
 #endif
+        {NULL,      0,              0}
+};
 
-#ifndef FUSE_UTIL
-void
-gf_fuse_unmount (const char *mountpoint, int fd)
+static int
+mount_param_to_flag (char *mnt_param, mount_flag_t *mntflags,
+                     char **mnt_param_new)
 {
-        int res;
-        int pid;
-
-        if (!mountpoint)
-                return;
-
-        if (fd != -1) {
-                struct pollfd pfd;
-
-                pfd.fd = fd;
-                pfd.events = 0;
-                res = poll (&pfd, 1, 0);
-                /* If file poll returns POLLERR on the device file descriptor,
-                   then the filesystem is already unmounted */
-                if (res == 1 && (pfd.revents & POLLERR))
-                        return;
+        gf_boolean_t found = _gf_false;
+        struct mount_flags *flag = NULL;
+        char *param_tok = NULL;
+        token_iter_t tit = {0,};
+        gf_boolean_t iter_end = _gf_false;
+
+        /* Allocate a buffer that will hold the mount parameters remaining
+         * after the ones corresponding to mount flags are processed and
+         * removed.The length of the original params are a good upper bound
+         * of the size needed.
+         */
+        *mnt_param_new = strdup (mnt_param);
+        if (!*mnt_param_new)
+                return -1;
 
-                /* Need to close file descriptor, otherwise synchronous umount
-                   would recurse into filesystem, and deadlock */
-                close (fd);
-        }
+        for (param_tok = token_iter_init (*mnt_param_new, ',', &tit) ;;) {
+                iter_end = next_token (&param_tok, &tit);
+
+                found = _gf_false;
+                for (flag = mount_flags; flag->opt; flag++) {
+                        /* Compare the mount flag name to the param
+                         * name at hand.
+                         */
+                        if (strcmp (flag->opt, param_tok) == 0) {
+                                /* If there is a match, adjust mntflags
+                                 * accordingly and break.
+                                 */
+                                if (flag->on) {
+                                        *mntflags |= flag->flag;
+                                } else {
+                                        *mntflags &= ~flag->flag;
+                                }
+                                found = _gf_true;
+                                break;
+                        }
+                }
+                /* Exclude flag names from new parameter list. */
+                if (found)
+                        drop_token (param_tok, &tit);
 
-        if (geteuid () == 0) {
-                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
-                return;
+                if (iter_end)
+                        break;
         }
 
-        res = umount2 (mountpoint, 2);
-        if (res == 0)
-                return;
-
-        pid = fork ();
-        if (pid == -1)
-                return;
-
-        if (pid == 0) {
-                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
-                                       "--", mountpoint, NULL };
-
-                execvp (FUSERMOUNT_PROG, (char **)argv);
-                _exit (1);
-        }
-        waitpid (pid, NULL, 0);
+        return 0;
 }
-#endif
-
-/*
- * Functions below are loosely modelled after similar functions of libfuse
- */
 
-#ifndef FUSE_UTIL
 static int
-fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param)
+fuse_mount_sys (const char *mountpoint, char *fsname,
+                char *mnt_param, int fd)
 {
-        int fd = -1, ret = -1;
+        int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
-
-        fd = open ("/dev/fuse", O_RDWR);
-        if (fd == -1) {
-                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)", strerror (errno));
-
-                return -1;
-        }
-
-        ret = asprintf (&mnt_param_mnt,
-                        "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
-                        mnt_param, fd, S_IFDIR, getuid (), getgid ());
+        mount_flag_t mountflags = 0;
+        char *mnt_param_new = NULL;
+
+        ret = mount_param_to_flag (mnt_param, &mountflags, &mnt_param_new);
+        if (ret == 0)
+                ret = asprintf (&mnt_param_mnt,
+                                "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
+                                mnt_param_new, fd, S_IFDIR, getuid (),
+                                getgid ());
         if (ret == -1) {
                 GFFUSE_LOGERR ("Out of memory");
 
                 goto out;
         }
-        ret = mount (source, mountpoint, fstype, 0,
+
+#ifdef __FreeBSD__
+        struct iovec *iov = NULL;
+        int iovlen = 0;
+        char fdstr[15];
+        sprintf (fdstr, "%d", fd);
+
+        build_iovec (&iov, &iovlen, "fstype", "fusefs", -1);
+        build_iovec (&iov, &iovlen, "subtype", "glusterfs", -1);
+        build_iovec (&iov, &iovlen, "fspath", __DECONST(void *, mountpoint),
+                     -1);
+        build_iovec (&iov, &iovlen, "from", "/dev/fuse", -1);
+        build_iovec (&iov, &iovlen, "volname", source, -1);
+        build_iovec (&iov, &iovlen, "fd", fdstr, -1);
+        build_iovec (&iov, &iovlen, "allow_other", NULL, -1);
+        ret = nmount (iov, iovlen, mountflags);
+#else
+        ret = mount (source, mountpoint, fstype, mountflags,
                      mnt_param_mnt);
+#endif /* __FreeBSD__ */
+#ifdef GF_LINUX_HOST_OS
         if (ret == -1 && errno == ENODEV) {
                 /* fs subtype support was added by 79c0b2df aka
                    v2.6.21-3159-g79c0b2d. Probably we have an
@@ -549,16 +408,19 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param)
 
                         goto out;
                 }
-                ret = mount (source, mountpoint, fstype, 0,
+                ret = mount (source, mountpoint, fstype, mountflags,
                              mnt_param_mnt);
         }
+#endif /* GF_LINUX_HOST_OS */
         if (ret == -1)
                 goto out;
         else
                 mounted = 1;
 
+#ifdef GF_LINUX_HOST_OS
         if (geteuid () == 0) {
                 char *newmnt = fuse_mnt_resolve_path ("fuse", mountpoint);
+                char *mnt_param_mtab = NULL;
 
                 if (!newmnt) {
                         ret = -1;
@@ -566,8 +428,17 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param)
                         goto out;
                 }
 
-                ret = fuse_mnt_add_mount ("fuse", source, newmnt, fstype,
-                                          mnt_param);
+                ret = asprintf (&mnt_param_mtab, "%s%s",
+                                mountflags & MS_RDONLY ? "ro," : "",
+                                mnt_param_new);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("Out of memory");
+                else {
+                        ret = fuse_mnt_add_mount ("fuse", source, newmnt,
+                                                  fstype, mnt_param_mtab);
+                        FREE (mnt_param_mtab);
+                }
+
                 FREE (newmnt);
                 if (ret == -1) {
                         GFFUSE_LOGERR ("failed to add mtab entry");
@@ -575,94 +446,81 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param)
                         goto out;
                 }
         }
+#endif /* GF_LINUX_HOST_OS */
 
- out:
+out:
         if (ret == -1) {
+                GFFUSE_LOGERR("ret = -1\n");
                 if (mounted)
                         umount2 (mountpoint, 2); /* lazy umount */
-                close (fd);
-                fd = -1;
         }
         FREE (mnt_param_mnt);
+        FREE (mnt_param_new);
         if (source != fsname)
                 FREE (source);
-        return fd;
-}
-
-static char *
-escape (char *s)
-{
-        size_t len = 0;
-        char *p = NULL;
-        char *q = NULL;
-        char *e = NULL;
 
-        for (p = s; *p; p++) {
-                if (*p == ',')
-                       len++;
-                len++;
-        }
-
-        e = CALLOC (1, len + 1);
-        if (!e)
-                return NULL;
-
-        for (p = s, q = e; *p; p++, q++) {
-                if (*p == ',') {
-                        *q = '\\';
-                        q++;
-                }
-                *q = *p;
-        }
-
-        return e;
+        return ret;
 }
 
 int
-gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param)
+gf_fuse_mount (const char *mountpoint, char *fsname,
+               char *mnt_param, pid_t *mnt_pid, int status_fd)
 {
-        int fd = -1, rv = -1;
-        char *fm_mnt_params = NULL, *p = NULL;
-        char *efsname = NULL;
+        int   fd  = -1;
+        pid_t pid = -1;
+        int   ret = -1;
 
-        fd = fuse_mount_sys (mountpoint, fsname, mnt_param);
+        fd = open ("/dev/fuse", O_RDWR);
         if (fd == -1) {
-                gf_log ("glusterfs-fuse", GF_LOG_NORMAL,
-                        "direct mount failed (%s), "
-                        "retry to mount via fusermount",
-                        strerror (errno));
-
-                efsname = escape (fsname);
-                if (!efsname) {
-                        GFFUSE_LOGERR ("Out of memory");
+                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
+                                strerror (errno));
+                return -1;
+        }
 
-                        return -1;
+        /* start mount agent */
+        pid = fork();
+        switch (pid) {
+        case 0:
+                /* hello it's mount agent */
+                if (!mnt_pid) {
+                        /* daemonize mount agent, caller is
+                         * not interested in waiting for it
+                         */
+                        pid = fork ();
+                        if (pid)
+                                exit (pid == -1 ? 1 : 0);
                 }
-                rv = asprintf (&fm_mnt_params,
-                               "%s,fsname=%s,nonempty,subtype=glusterfs",
-                               mnt_param, efsname);
-                FREE (efsname);
-                if (rv == -1) {
-                        GFFUSE_LOGERR ("Out of memory");
 
-                        return -1;
-                }
+                ret = fuse_mount_sys (mountpoint, fsname, mnt_param, fd);
+                if (ret == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "direct mount failed (%s) errno %d",
+                                strerror (errno), errno);
 
-                fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
-                if (fd == -1) {
-                        p = fm_mnt_params + strlen (fm_mnt_params);
-                        while (*--p != ',');
-                        *p = '\0';
+                        if (errno == EPERM) {
+                                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                        "retry to mount via fusermount");
 
-                        fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
+                                ret = fuse_mount_fusermount (mountpoint, fsname,
+                                                             mnt_param, fd);
+                        }
                 }
 
-                FREE (fm_mnt_params);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("mount of %s to %s (%s) failed",
+                                       fsname, mountpoint, mnt_param);
 
-                if (fd == -1)
-                       GFFUSE_LOGERR ("mount failed");
+                if (status_fd >= 0)
+                        (void)write (status_fd, &ret, sizeof (ret));
+                exit (!!ret);
+                /* bye mount agent */
+        case -1:
+                close (fd);
+                fd = -1;
         }
 
+        if (mnt_pid)
+               *mnt_pid = pid;
+
         return fd;
 }
-#endif
diff --git a/contrib/fuse-util/Makefile.am b/contrib/fuse-util/Makefile.am
index 42609a6883d..abbc10eb6d9 100644
--- a/contrib/fuse-util/Makefile.am
+++ b/contrib/fuse-util/Makefile.am
@@ -1,9 +1,11 @@
 bin_PROGRAMS = fusermount-glusterfs
 
-fusermount_glusterfs_SOURCES = fusermount.c $(CONTRIBDIR)/fuse-lib/mount.c
-noinst_HEADERS = mount_util.h
+fusermount_glusterfs_SOURCES = fusermount.c mount_util.c $(CONTRIBDIR)/fuse-lib/mount-common.c
+noinst_HEADERS = $(CONTRIBDIR)/fuse-include/mount_util.h
 
-AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DFUSE_UTIL $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -DFUSE_UTIL -I$(CONTRIBDIR)/fuse-include -I$(CONTRIBDIR)/fuse-lib
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 install-exec-hook:
 	-chown root $(DESTDIR)$(bindir)/fusermount-glusterfs
diff --git a/contrib/fuse-util/fusermount.c b/contrib/fuse-util/fusermount.c
index 39da9b6a0b6..ff743f75a21 100644
--- a/contrib/fuse-util/fusermount.c
+++ b/contrib/fuse-util/fusermount.c
@@ -10,6 +10,11 @@
 #include <config.h>
 
 #include "mount_util.h"
+
+#ifndef HAVE_UMOUNT2
+#include "mount-gluster-compat.h"
+#endif
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -19,15 +24,24 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <pwd.h>
+#include <limits.h>
+#if !defined(__NetBSD__) && !defined(GF_DARWIN_HOST_OS)
 #include <mntent.h>
+#endif /* __NetBSD__ */
 #include <sys/wait.h>
 #include <sys/stat.h>
-#include <sys/mount.h>
+#ifdef HAVE_SET_FSID
 #include <sys/fsuid.h>
+#endif
+#ifdef GF_DARWIN_HOST_OS
+#include <sys/param.h>
+#endif
+#include <sys/mount.h>
 #include <sys/socket.h>
 #include <sys/utsname.h>
 #include <sched.h>
 
+#define FUSE_DEVFD_ENV		"_FUSE_DEVFD"
 #define FUSE_COMMFD_ENV		"_FUSE_COMMFD"
 
 #define FUSE_DEV_OLD "/proc/fs/fuse/dev"
@@ -61,22 +75,32 @@ static const char *get_user_name(void)
 	}
 }
 
+#ifdef HAVE_SET_FSID
 static uid_t oldfsuid;
 static gid_t oldfsgid;
+#endif
 
 static void drop_privs(void)
 {
 	if (getuid() != 0) {
+#ifdef HAVE_SET_FSID
 		oldfsuid = setfsuid(getuid());
 		oldfsgid = setfsgid(getgid());
+#else
+		fprintf(stderr, "%s: Implement alternative setfsuid/gid \n", progname);
+#endif
 	}
 }
 
 static void restore_privs(void)
 {
 	if (getuid() != 0) {
+#ifdef HAVE_SET_FSID
 		setfsuid(oldfsuid);
 		setfsgid(oldfsgid);
+#else
+		fprintf(stderr, "%s: Implement alternative setfsuid/gid \n", progname);
+#endif
 	}
 }
 
@@ -114,8 +138,16 @@ static int lock_umount(void)
 
 static void unlock_umount(int mtablock)
 {
-	lockf(mtablock, F_ULOCK, 0);
-	close(mtablock);
+	if (mtablock >= 0) {
+		int res;
+
+		res = lockf(mtablock, F_ULOCK, 0);
+		if (res < 0) {
+			fprintf(stderr, "%s: error releasing lock: %s\n",
+				progname, strerror(errno));
+		}
+		close(mtablock);
+	}
 }
 
 static int add_mount(const char *source, const char *mnt, const char *type,
@@ -238,7 +270,7 @@ static int check_is_mount_child(void *p)
 	}
 
 	count = 0;
-	while ((entp = getmntent(fp)) != NULL)
+	while (getmntent(fp) != NULL)
 		count++;
 	endmntent(fp);
 
@@ -325,7 +357,7 @@ static int check_is_mount(const char *last, const char *mnt)
 	return 0;
 }
 
-static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
+static int chdir_to_parent(char *copy, const char **lastp)
 {
 	char *tmp;
 	const char *parent;
@@ -350,14 +382,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 		parent = "/";
 	}
 
-	*currdir_fd = open(".", O_RDONLY);
-	if (*currdir_fd == -1) {
-		fprintf(stderr,
-			"%s: failed to open current directory: %s\n",
-			progname, strerror(errno));
-		return -1;
-	}
-
 	res = chdir(parent);
 	if (res == -1) {
 		fprintf(stderr, "%s: failed to chdir to %s: %s\n",
@@ -382,7 +406,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 
 static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 {
-	int currdir_fd = -1;
 	char *copy;
 	const char *last;
 	int res;
@@ -399,7 +422,7 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 		return -1;
 	}
 
-	res = chdir_to_parent(copy, &last, &currdir_fd);
+	res = chdir_to_parent(copy, &last);
 	if (res == -1)
 		goto out;
 
@@ -411,10 +434,6 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 
 out:
 	free(copy);
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 
 	return res;
 }
@@ -501,20 +520,22 @@ static void parse_line(char *line, int linenum)
 
 static void read_conf(void)
 {
+	int  len;
 	FILE *fp = fopen(FUSE_CONF, "r");
 	if (fp != NULL) {
 		int linenum = 1;
 		char line[256];
 		int isnewline = 1;
 		while (fgets(line, sizeof(line), fp) != NULL) {
+			len = strlen (line);
 			if (isnewline) {
-				if (line[strlen(line)-1] == '\n') {
+				if (len && line[len-1] == '\n') {
 					strip_line(line);
 					parse_line(line, linenum);
 				} else {
 					isnewline = 0;
 				}
-			} else if(line[strlen(line)-1] == '\n') {
+			} else if (len && line[len-1] == '\n') {
 				fprintf(stderr, "%s: reading %s: line %i too long\n", progname, FUSE_CONF, linenum);
 
 				isnewline = 1;
@@ -609,7 +630,7 @@ static int add_option(char **optsp, const char *opt, unsigned expand)
 static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 {
 	int i;
-	int l;
+	size_t l;
 
 	if (!(flags & MS_RDONLY) && add_option(mnt_optsp, "rw", 0) == -1)
 		return -1;
@@ -624,7 +645,7 @@ static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 		return -1;
 	/* remove comma from end of opts*/
 	l = strlen(*mnt_optsp);
-	if ((*mnt_optsp)[l-1] == ',')
+	if (l && (*mnt_optsp)[l-1] == ',')
 		(*mnt_optsp)[l-1] = '\0';
 	if (getuid() != 0) {
 		const char *user = get_user_name();
@@ -653,8 +674,7 @@ static int get_string_opt(const char *s, unsigned len, const char *opt,
 	unsigned opt_len = strlen(opt);
 	char *d;
 
-	if (*val)
-		free(*val);
+	free(*val);
 	*val = (char *) malloc(len - opt_len + 1);
 	if (!*val) {
 		fprintf(stderr, "%s: failed to allocate memory\n", progname);
@@ -823,15 +843,14 @@ static int do_mount(const char *mnt, char **typep, mode_t rootmode,
 			fprintf(stderr, "%s: mount failed: %s\n", progname,
 				strerror(errno_save));
 		goto err;
-	} else {
-		*sourcep = source;
-		*typep = type;
-		*mnt_optsp = mnt_opts;
 	}
+	*sourcep = source;
+	*typep = type;
+	*mnt_optsp = mnt_opts;
 	free(fsname);
 	free(optbuf);
 
-	return res;
+	return 0;
 
 err:
 	free(fsname);
@@ -874,8 +893,7 @@ static int check_version(const char *dev)
 	return 0;
 }
 
-static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
-		      int *mountpoint_fd)
+static int check_perm(const char **mntp, struct stat *stbuf, int *mountpoint_fd)
 {
 	int res;
 	const char *mnt = *mntp;
@@ -893,13 +911,6 @@ static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
 		return 0;
 
 	if (S_ISDIR(stbuf->st_mode)) {
-		*currdir_fd = open(".", O_RDONLY);
-		if (*currdir_fd == -1) {
-			fprintf(stderr,
-				"%s: failed to open current directory: %s\n",
-				progname, strerror(errno));
-			return -1;
-		}
 		res = chdir(mnt);
 		if (res == -1) {
 			fprintf(stderr,
@@ -1016,8 +1027,36 @@ static int open_fuse_device(char **devp)
 	return -1;
 }
 
+static int check_fuse_device(char *devfd, char **devp)
+{
+	int res;
+	char *devlink;
 
-static int mount_fuse(const char *mnt, const char *opts)
+	res = asprintf(&devlink, "/proc/self/fd/%s", devfd);
+	if (res == -1) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		return -1;
+	}
+
+	*devp = (char *) calloc(1, PATH_MAX + 1);
+	if (!*devp) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		free(devlink);
+		return -1;
+	}
+
+	res = readlink (devlink, *devp, PATH_MAX);
+	free (devlink);
+	if (res == -1) {
+		fprintf(stderr, "%s: specified fuse fd is invalid\n",
+			progname);
+		return -1;
+	}
+
+	return atoi(devfd);
+}
+
+static int mount_fuse(const char *mnt, const char *opts, char *devfd)
 {
 	int res;
 	int fd;
@@ -1027,10 +1066,9 @@ static int mount_fuse(const char *mnt, const char *opts)
 	char *source = NULL;
 	char *mnt_opts = NULL;
 	const char *real_mnt = mnt;
-	int currdir_fd = -1;
 	int mountpoint_fd = -1;
 
-	fd = open_fuse_device(&dev);
+	fd = devfd ? check_fuse_device(devfd, &dev) : open_fuse_device(&dev);
 	if (fd == -1)
 		return -1;
 
@@ -1041,15 +1079,13 @@ static int mount_fuse(const char *mnt, const char *opts)
 		int mount_count = count_fuse_fs();
 		if (mount_count >= mount_max) {
 			fprintf(stderr, "%s: too many FUSE filesystems mounted; mount_max=N can be set in /etc/fuse.conf\n", progname);
-			close(fd);
-			return -1;
+			goto fail_close_fd;
 		}
 	}
 
 	res = check_version(dev);
 	if (res != -1) {
-		res = check_perm(&real_mnt, &stbuf, &currdir_fd,
-				 &mountpoint_fd);
+		res = check_perm(&real_mnt, &stbuf, &mountpoint_fd);
 		restore_privs();
 		if (res != -1)
 			res = do_mount(real_mnt, &type, stbuf.st_mode & S_IFMT,
@@ -1058,33 +1094,38 @@ static int mount_fuse(const char *mnt, const char *opts)
 	} else
 		restore_privs();
 
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 	if (mountpoint_fd != -1)
 		close(mountpoint_fd);
 
+	if (res == -1)
+		goto fail_close_fd;
+
+	res = chdir("/");
 	if (res == -1) {
-		close(fd);
-		return -1;
+		fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+		goto fail_close_fd;
 	}
 
 	if (geteuid() == 0) {
 		res = add_mount(source, mnt, type, mnt_opts);
 		if (res == -1) {
-			umount2(mnt, 2); /* lazy umount */
-			close(fd);
-			return -1;
+			/* Can't clean up mount in a non-racy way */
+			goto fail_close_fd;
 		}
 	}
 
+out_free:
 	free(source);
 	free(type);
 	free(mnt_opts);
 	free(dev);
 
 	return fd;
+
+fail_close_fd:
+	close(fd);
+	fd = -1;
+	goto out_free;
 }
 
 static int send_fd(int sock_fd, int fd)
@@ -1154,6 +1195,7 @@ int main(int argc, char *argv[])
 	static int unmount = 0;
 	static int lazy = 0;
 	static int quiet = 0;
+	char *devfd;
 	char *commfd;
 	int cfd;
 	const char *opts = "";
@@ -1222,6 +1264,13 @@ int main(int argc, char *argv[])
 
 	drop_privs();
 	mnt = fuse_mnt_resolve_path(progname, origmnt);
+	if (mnt != NULL) {
+		res = chdir("/");
+		if (res == -1) {
+			fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+			exit(1);
+		}
+	}
 	restore_privs();
 	if (mnt == NULL)
 		exit(1);
@@ -1242,21 +1291,26 @@ int main(int argc, char *argv[])
 		return 0;
 	}
 
-	commfd = getenv(FUSE_COMMFD_ENV);
-	if (commfd == NULL) {
-		fprintf(stderr, "%s: old style mounting not supported\n",
-			progname);
-		exit(1);
+	devfd = getenv(FUSE_DEVFD_ENV);
+	if (devfd == NULL) {
+		commfd = getenv(FUSE_COMMFD_ENV);
+		if (commfd == NULL) {
+			fprintf(stderr, "%s: old style mounting not supported\n",
+				progname);
+			exit(1);
+		}
 	}
 
-	fd = mount_fuse(mnt, opts);
+	fd = mount_fuse(mnt, opts, devfd);
 	if (fd == -1)
 		exit(1);
 
-	cfd = atoi(commfd);
-	res = send_fd(cfd, fd);
-	if (res == -1)
-		exit(1);
+	if (devfd == NULL) {
+		cfd = atoi(commfd);
+		res = send_fd(cfd, fd);
+		if (res == -1)
+			exit(1);
+	}
 
 	return 0;
 }
diff --git a/contrib/fuse-util/mount_util.c b/contrib/fuse-util/mount_util.c
new file mode 100644
index 00000000000..911b84445e9
--- /dev/null
+++ b/contrib/fuse-util/mount_util.c
@@ -0,0 +1,64 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include <dirent.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+int fuse_mnt_check_empty(const char *progname, const char *mnt,
+			 mode_t rootmode, off_t rootsize)
+{
+	int isempty = 1;
+
+	if (S_ISDIR(rootmode)) {
+		struct dirent *ent;
+		DIR *dp = opendir(mnt);
+		if (dp == NULL) {
+			fprintf(stderr,
+				"%s: failed to open mountpoint for reading: %s\n",
+				progname, strerror(errno));
+			return -1;
+		}
+		while ((ent = readdir(dp)) != NULL) {
+			if (strcmp(ent->d_name, ".") != 0 &&
+			    strcmp(ent->d_name, "..") != 0) {
+				isempty = 0;
+				break;
+			}
+		}
+		closedir(dp);
+	} else if (rootsize)
+		isempty = 0;
+
+	if (!isempty) {
+		fprintf(stderr, "%s: mountpoint is not empty\n", progname);
+		fprintf(stderr, "%s: if you are sure this is safe, use the 'nonempty' mount option\n", progname);
+		return -1;
+	}
+	return 0;
+}
+
+int fuse_mnt_check_fuseblk(void)
+{
+	char buf[256];
+	FILE *f = fopen("/proc/filesystems", "r");
+	if (!f)
+		return 1;
+
+	while (fgets(buf, sizeof(buf), f))
+		if (strstr(buf, "fuseblk\n")) {
+			fclose(f);
+			return 1;
+		}
+
+	fclose(f);
+	return 0;
+}
diff --git a/contrib/libexecinfo/execinfo.c b/contrib/libexecinfo/execinfo.c
new file mode 100644
index 00000000000..03500257788
--- /dev/null
+++ b/contrib/libexecinfo/execinfo.c
@@ -0,0 +1,442 @@
+/*
+ * Copyright (c) 2003 Maxim Sobolev <sobomax@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: execinfo.c,v 1.3 2004/07/19 05:21:09 sobomax Exp $
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifndef HAVE_BACKTRACE
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <dlfcn.h>
+#include <math.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <stddef.h>
+
+#include "execinfo_compat.h"
+
+#define D10(x) ceil(log10(((x) == 0) ? 2 : ((x) + 1)))
+
+static void *
+getreturnaddr(int level)
+{
+        switch (level) {
+        case 0: return __builtin_return_address(1);
+        case 1: return __builtin_return_address(2);
+        case 2: return __builtin_return_address(3);
+        case 3: return __builtin_return_address(4);
+        case 4: return __builtin_return_address(5);
+        case 5: return __builtin_return_address(6);
+        case 6: return __builtin_return_address(7);
+        case 7: return __builtin_return_address(8);
+        case 8: return __builtin_return_address(9);
+        case 9: return __builtin_return_address(10);
+        case 10: return __builtin_return_address(11);
+        case 11: return __builtin_return_address(12);
+        case 12: return __builtin_return_address(13);
+        case 13: return __builtin_return_address(14);
+        case 14: return __builtin_return_address(15);
+        case 15: return __builtin_return_address(16);
+        case 16: return __builtin_return_address(17);
+        case 17: return __builtin_return_address(18);
+        case 18: return __builtin_return_address(19);
+        case 19: return __builtin_return_address(20);
+        case 20: return __builtin_return_address(21);
+        case 21: return __builtin_return_address(22);
+        case 22: return __builtin_return_address(23);
+        case 23: return __builtin_return_address(24);
+        case 24: return __builtin_return_address(25);
+        case 25: return __builtin_return_address(26);
+        case 26: return __builtin_return_address(27);
+        case 27: return __builtin_return_address(28);
+        case 28: return __builtin_return_address(29);
+        case 29: return __builtin_return_address(30);
+        case 30: return __builtin_return_address(31);
+        case 31: return __builtin_return_address(32);
+        case 32: return __builtin_return_address(33);
+        case 33: return __builtin_return_address(34);
+        case 34: return __builtin_return_address(35);
+        case 35: return __builtin_return_address(36);
+        case 36: return __builtin_return_address(37);
+        case 37: return __builtin_return_address(38);
+        case 38: return __builtin_return_address(39);
+        case 39: return __builtin_return_address(40);
+        case 40: return __builtin_return_address(41);
+        case 41: return __builtin_return_address(42);
+        case 42: return __builtin_return_address(43);
+        case 43: return __builtin_return_address(44);
+        case 44: return __builtin_return_address(45);
+        case 45: return __builtin_return_address(46);
+        case 46: return __builtin_return_address(47);
+        case 47: return __builtin_return_address(48);
+        case 48: return __builtin_return_address(49);
+        case 49: return __builtin_return_address(50);
+        case 50: return __builtin_return_address(51);
+        case 51: return __builtin_return_address(52);
+        case 52: return __builtin_return_address(53);
+        case 53: return __builtin_return_address(54);
+        case 54: return __builtin_return_address(55);
+        case 55: return __builtin_return_address(56);
+        case 56: return __builtin_return_address(57);
+        case 57: return __builtin_return_address(58);
+        case 58: return __builtin_return_address(59);
+        case 59: return __builtin_return_address(60);
+        case 60: return __builtin_return_address(61);
+        case 61: return __builtin_return_address(62);
+        case 62: return __builtin_return_address(63);
+        case 63: return __builtin_return_address(64);
+        case 64: return __builtin_return_address(65);
+        case 65: return __builtin_return_address(66);
+        case 66: return __builtin_return_address(67);
+        case 67: return __builtin_return_address(68);
+        case 68: return __builtin_return_address(69);
+        case 69: return __builtin_return_address(70);
+        case 70: return __builtin_return_address(71);
+        case 71: return __builtin_return_address(72);
+        case 72: return __builtin_return_address(73);
+        case 73: return __builtin_return_address(74);
+        case 74: return __builtin_return_address(75);
+        case 75: return __builtin_return_address(76);
+        case 76: return __builtin_return_address(77);
+        case 77: return __builtin_return_address(78);
+        case 78: return __builtin_return_address(79);
+        case 79: return __builtin_return_address(80);
+        case 80: return __builtin_return_address(81);
+        case 81: return __builtin_return_address(82);
+        case 82: return __builtin_return_address(83);
+        case 83: return __builtin_return_address(84);
+        case 84: return __builtin_return_address(85);
+        case 85: return __builtin_return_address(86);
+        case 86: return __builtin_return_address(87);
+        case 87: return __builtin_return_address(88);
+        case 88: return __builtin_return_address(89);
+        case 89: return __builtin_return_address(90);
+        case 90: return __builtin_return_address(91);
+        case 91: return __builtin_return_address(92);
+        case 92: return __builtin_return_address(93);
+        case 93: return __builtin_return_address(94);
+        case 94: return __builtin_return_address(95);
+        case 95: return __builtin_return_address(96);
+        case 96: return __builtin_return_address(97);
+        case 97: return __builtin_return_address(98);
+        case 98: return __builtin_return_address(99);
+        case 99: return __builtin_return_address(100);
+        case 100: return __builtin_return_address(101);
+        case 101: return __builtin_return_address(102);
+        case 102: return __builtin_return_address(103);
+        case 103: return __builtin_return_address(104);
+        case 104: return __builtin_return_address(105);
+        case 105: return __builtin_return_address(106);
+        case 106: return __builtin_return_address(107);
+        case 107: return __builtin_return_address(108);
+        case 108: return __builtin_return_address(109);
+        case 109: return __builtin_return_address(110);
+        case 110: return __builtin_return_address(111);
+        case 111: return __builtin_return_address(112);
+        case 112: return __builtin_return_address(113);
+        case 113: return __builtin_return_address(114);
+        case 114: return __builtin_return_address(115);
+        case 115: return __builtin_return_address(116);
+        case 116: return __builtin_return_address(117);
+        case 117: return __builtin_return_address(118);
+        case 118: return __builtin_return_address(119);
+        case 119: return __builtin_return_address(120);
+        case 120: return __builtin_return_address(121);
+        case 121: return __builtin_return_address(122);
+        case 122: return __builtin_return_address(123);
+        case 123: return __builtin_return_address(124);
+        case 124: return __builtin_return_address(125);
+        case 125: return __builtin_return_address(126);
+        case 126: return __builtin_return_address(127);
+        case 127: return __builtin_return_address(128);
+        default: return NULL;
+        }
+}
+
+static void *
+getframeaddr(int level)
+{
+
+        switch (level) {
+        case 0: return __builtin_frame_address(1);
+        case 1: return __builtin_frame_address(2);
+        case 2: return __builtin_frame_address(3);
+        case 3: return __builtin_frame_address(4);
+        case 4: return __builtin_frame_address(5);
+        case 5: return __builtin_frame_address(6);
+        case 6: return __builtin_frame_address(7);
+        case 7: return __builtin_frame_address(8);
+        case 8: return __builtin_frame_address(9);
+        case 9: return __builtin_frame_address(10);
+        case 10: return __builtin_frame_address(11);
+        case 11: return __builtin_frame_address(12);
+        case 12: return __builtin_frame_address(13);
+        case 13: return __builtin_frame_address(14);
+        case 14: return __builtin_frame_address(15);
+        case 15: return __builtin_frame_address(16);
+        case 16: return __builtin_frame_address(17);
+        case 17: return __builtin_frame_address(18);
+        case 18: return __builtin_frame_address(19);
+        case 19: return __builtin_frame_address(20);
+        case 20: return __builtin_frame_address(21);
+        case 21: return __builtin_frame_address(22);
+        case 22: return __builtin_frame_address(23);
+        case 23: return __builtin_frame_address(24);
+        case 24: return __builtin_frame_address(25);
+        case 25: return __builtin_frame_address(26);
+        case 26: return __builtin_frame_address(27);
+        case 27: return __builtin_frame_address(28);
+        case 28: return __builtin_frame_address(29);
+        case 29: return __builtin_frame_address(30);
+        case 30: return __builtin_frame_address(31);
+        case 31: return __builtin_frame_address(32);
+        case 32: return __builtin_frame_address(33);
+        case 33: return __builtin_frame_address(34);
+        case 34: return __builtin_frame_address(35);
+        case 35: return __builtin_frame_address(36);
+        case 36: return __builtin_frame_address(37);
+        case 37: return __builtin_frame_address(38);
+        case 38: return __builtin_frame_address(39);
+        case 39: return __builtin_frame_address(40);
+        case 40: return __builtin_frame_address(41);
+        case 41: return __builtin_frame_address(42);
+        case 42: return __builtin_frame_address(43);
+        case 43: return __builtin_frame_address(44);
+        case 44: return __builtin_frame_address(45);
+        case 45: return __builtin_frame_address(46);
+        case 46: return __builtin_frame_address(47);
+        case 47: return __builtin_frame_address(48);
+        case 48: return __builtin_frame_address(49);
+        case 49: return __builtin_frame_address(50);
+        case 50: return __builtin_frame_address(51);
+        case 51: return __builtin_frame_address(52);
+        case 52: return __builtin_frame_address(53);
+        case 53: return __builtin_frame_address(54);
+        case 54: return __builtin_frame_address(55);
+        case 55: return __builtin_frame_address(56);
+        case 56: return __builtin_frame_address(57);
+        case 57: return __builtin_frame_address(58);
+        case 58: return __builtin_frame_address(59);
+        case 59: return __builtin_frame_address(60);
+        case 60: return __builtin_frame_address(61);
+        case 61: return __builtin_frame_address(62);
+        case 62: return __builtin_frame_address(63);
+        case 63: return __builtin_frame_address(64);
+        case 64: return __builtin_frame_address(65);
+        case 65: return __builtin_frame_address(66);
+        case 66: return __builtin_frame_address(67);
+        case 67: return __builtin_frame_address(68);
+        case 68: return __builtin_frame_address(69);
+        case 69: return __builtin_frame_address(70);
+        case 70: return __builtin_frame_address(71);
+        case 71: return __builtin_frame_address(72);
+        case 72: return __builtin_frame_address(73);
+        case 73: return __builtin_frame_address(74);
+        case 74: return __builtin_frame_address(75);
+        case 75: return __builtin_frame_address(76);
+        case 76: return __builtin_frame_address(77);
+        case 77: return __builtin_frame_address(78);
+        case 78: return __builtin_frame_address(79);
+        case 79: return __builtin_frame_address(80);
+        case 80: return __builtin_frame_address(81);
+        case 81: return __builtin_frame_address(82);
+        case 82: return __builtin_frame_address(83);
+        case 83: return __builtin_frame_address(84);
+        case 84: return __builtin_frame_address(85);
+        case 85: return __builtin_frame_address(86);
+        case 86: return __builtin_frame_address(87);
+        case 87: return __builtin_frame_address(88);
+        case 88: return __builtin_frame_address(89);
+        case 89: return __builtin_frame_address(90);
+        case 90: return __builtin_frame_address(91);
+        case 91: return __builtin_frame_address(92);
+        case 92: return __builtin_frame_address(93);
+        case 93: return __builtin_frame_address(94);
+        case 94: return __builtin_frame_address(95);
+        case 95: return __builtin_frame_address(96);
+        case 96: return __builtin_frame_address(97);
+        case 97: return __builtin_frame_address(98);
+        case 98: return __builtin_frame_address(99);
+        case 99: return __builtin_frame_address(100);
+        case 100: return __builtin_frame_address(101);
+        case 101: return __builtin_frame_address(102);
+        case 102: return __builtin_frame_address(103);
+        case 103: return __builtin_frame_address(104);
+        case 104: return __builtin_frame_address(105);
+        case 105: return __builtin_frame_address(106);
+        case 106: return __builtin_frame_address(107);
+        case 107: return __builtin_frame_address(108);
+        case 108: return __builtin_frame_address(109);
+        case 109: return __builtin_frame_address(110);
+        case 110: return __builtin_frame_address(111);
+        case 111: return __builtin_frame_address(112);
+        case 112: return __builtin_frame_address(113);
+        case 113: return __builtin_frame_address(114);
+        case 114: return __builtin_frame_address(115);
+        case 115: return __builtin_frame_address(116);
+        case 116: return __builtin_frame_address(117);
+        case 117: return __builtin_frame_address(118);
+        case 118: return __builtin_frame_address(119);
+        case 119: return __builtin_frame_address(120);
+        case 120: return __builtin_frame_address(121);
+        case 121: return __builtin_frame_address(122);
+        case 122: return __builtin_frame_address(123);
+        case 123: return __builtin_frame_address(124);
+        case 124: return __builtin_frame_address(125);
+        case 125: return __builtin_frame_address(126);
+        case 126: return __builtin_frame_address(127);
+        case 127: return __builtin_frame_address(128);
+        default: return NULL;
+        }
+}
+
+static inline void *
+realloc_safe(void *ptr, size_t size)
+{
+        void *nptr;
+
+        nptr = realloc (ptr, size);
+        if (nptr == NULL)
+                free (ptr);
+        return nptr;
+}
+
+int
+backtrace(void **buffer, int size)
+{
+        int i;
+
+        for (i = 1; getframeaddr(i + 1) != NULL && i != size + 1; i++) {
+                buffer[i - 1] = getreturnaddr(i);
+                if (buffer[i - 1] == NULL)
+                        break;
+        }
+        return i - 1;
+}
+
+char **
+backtrace_symbols(void *const *buffer, int size)
+{
+        size_t clen, alen;
+        int i, offset;
+        char **rval;
+        Dl_info info;
+
+        clen = size * sizeof(char *);
+        rval = malloc(clen);
+        if (rval == NULL)
+                return NULL;
+        for (i = 0; i < size; i++) {
+                if (dladdr(buffer[i], &info) != 0) {
+                        if (info.dli_sname == NULL)
+                                info.dli_sname = "???";
+                        if (info.dli_saddr == NULL)
+                                info.dli_saddr = buffer[i];
+                        offset = buffer[i] - info.dli_saddr;
+                        /* "0x01234567 <function+offset> at filename" */
+                        alen = 2 +                      /* "0x" */
+                                (sizeof(void *) * 2) +   /* "01234567" */
+                                2 +                      /* " <" */
+                                strlen(info.dli_sname) + /* "function" */
+                                1 +                      /* "+" */
+                                10 +                     /* "offset */
+                                5 +                      /* "> at " */
+                                strlen(info.dli_fname) + /* "filename" */
+                                1;                       /* "\0" */
+                        rval = realloc_safe(rval, clen + alen);
+                        if (rval == NULL)
+                                return NULL;
+                        snprintf((char *) rval + clen, alen, "%p <%s+%d> at %s",
+                                 buffer[i], info.dli_sname, offset, info.dli_fname);
+                } else {
+                        alen = 2 +                      /* "0x" */
+                                (sizeof(void *) * 2) +   /* "01234567" */
+                                1;                       /* "\0" */
+                        rval = realloc_safe(rval, clen + alen);
+                        if (rval == NULL)
+                                return NULL;
+                        snprintf((char *) rval + clen, alen, "%p", buffer[i]);
+                }
+                rval[i] = (char *) clen;
+                clen += alen;
+        }
+
+        for (i = 0; i < size; i++)
+                rval[i] += (long) rval;
+
+        return rval;
+}
+
+void
+backtrace_symbols_fd(void *const *buffer, int size, int fd)
+{
+        int i, len, offset;
+        char *buf;
+        Dl_info info;
+
+        for (i = 0; i < size; i++) {
+                if (dladdr(buffer[i], &info) != 0) {
+                        if (info.dli_sname == NULL)
+                                info.dli_sname = "???";
+                        if (info.dli_saddr == NULL)
+                                info.dli_saddr = buffer[i];
+                        offset = buffer[i] - info.dli_saddr;
+                        /* "0x01234567 <function+offset> at filename" */
+                        len = 2 +                      /* "0x" */
+                                (sizeof(void *) * 2) +   /* "01234567" */
+                                2 +                      /* " <" */
+                                strlen(info.dli_sname) + /* "function" */
+                                1 +                      /* "+" */
+                                D10(offset) +            /* "offset */
+                                5 +                      /* "> at " */
+                                strlen(info.dli_fname) + /* "filename" */
+                                2;                       /* "\n\0" */
+                        buf = alloca(len);
+                        if (buf == NULL)
+                                return;
+                        snprintf(buf, len, "%p <%s+%d> at %s\n",
+                                 buffer[i], info.dli_sname, offset, info.dli_fname);
+                } else {
+                        len = 2 +                      /* "0x" */
+                                (sizeof(void *) * 2) +   /* "01234567" */
+                                2;                       /* "\n\0" */
+                        buf = alloca(len);
+                        if (buf == NULL)
+                                return;
+                        snprintf(buf, len, "%p\n", buffer[i]);
+                }
+                if (write(fd, buf, strlen(buf)) == -1)
+                        return;
+        }
+}
+#endif
diff --git a/contrib/libexecinfo/execinfo_compat.h b/contrib/libexecinfo/execinfo_compat.h
new file mode 100644
index 00000000000..ae84cfb1f35
--- /dev/null
+++ b/contrib/libexecinfo/execinfo_compat.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2003 Maxim Sobolev <sobomax@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: execinfo.h,v 1.2 2004/07/19 05:20:29 sobomax Exp $
+ */
+
+#ifndef _EXECINFO_H_
+#define _EXECINFO_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifndef HAVE_BACKTRACE
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int   backtrace(void **, int);
+extern char  **backtrace_symbols(void *const *, int);
+extern void  backtrace_symbols_fd(void *const *, int, int);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+
+#endif /* _EXECINFO_H_ */
diff --git a/contrib/libgen/basename_r.c b/contrib/libgen/basename_r.c
new file mode 100644
index 00000000000..2c3a87afe1c
--- /dev/null
+++ b/contrib/libgen/basename_r.c
@@ -0,0 +1,40 @@
+/*
+ * borrowed from glibc-2.12.1/string/basename.c
+ * Modified to return "." for NULL or "", as required for SUSv2.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_BASENAME
+
+/* Return the name-within-directory of a file name.
+   Copyright (C) 1996,97,98,2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+basename_r (filename)
+     const char *filename;
+{
+  char *p;
+
+  if ((filename == NULL) || (*filename == '\0'))
+    return ".";
+
+  p = strrchr (filename, '/');
+  return p ? p + 1 : (char *) filename;
+}
+#endif /* THREAD_UNSAFE_BASENAME */
diff --git a/contrib/libgen/dirname_r.c b/contrib/libgen/dirname_r.c
new file mode 100644
index 00000000000..131cbcf2a96
--- /dev/null
+++ b/contrib/libgen/dirname_r.c
@@ -0,0 +1,243 @@
+/*
+ * Borrowed from glibc-2.12.1/string/memrchr.c
+ * Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+ * Removed code for long bigger than 32 bytes, renamed __ptr_t as void *
+ * changed reg_char type to char.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_DIRNAME
+
+/* memrchr -- find the last occurrence of a byte in a memory block
+   Copyright (C) 1991, 93, 96, 97, 99, 2000 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+   with help from Dan Sahlin (dan@sics.se) and
+   commentary by Jim Blandy (jimb@ai.mit.edu);
+   adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu),
+   and implemented by Roland McGrath (roland@ai.mit.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+void *
+__memrchr (s, c_in, n)
+     const void * s;
+     int c_in;
+     size_t n;
+{
+  const unsigned char *char_ptr;
+  const unsigned long int *longword_ptr;
+  unsigned long int longword, magic_bits, charmask;
+  unsigned char c;
+
+  c = (unsigned char) c_in;
+
+  /* Handle the last few characters by reading one character at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s + n;
+       n > 0 && ((unsigned long int) char_ptr
+		 & (sizeof (longword) - 1)) != 0;
+       --n)
+    if (*--char_ptr == c)
+      return (void *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to 8-byte longwords.  */
+
+  longword_ptr = (const unsigned long int *) char_ptr;
+
+  /* Bits 31, 24, 16, and 8 of this number are zero.  Call these bits
+     the "holes."  Note that there is a hole just to the left of
+     each byte, with an extra at the end:
+
+     bits:  01111110 11111110 11111110 11111111
+     bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD
+
+     The 1-bits make sure that carries propagate to the next 0-bit.
+     The 0-bits provide holes for carries to fall into.  */
+
+  if (sizeof (longword) != 4 && sizeof (longword) != 8)
+    abort ();
+
+  magic_bits = 0x7efefeff;
+
+  /* Set up a longword, each of whose bytes is C.  */
+  charmask = c | (c << 8);
+  charmask |= charmask << 16;
+
+  /* Instead of the traditional loop which tests each character,
+     we will test a longword at a time.  The tricky part is testing
+     if *any of the four* bytes in the longword in question are zero.  */
+  while (n >= sizeof (longword))
+    {
+      /* We tentatively exit the loop if adding MAGIC_BITS to
+	 LONGWORD fails to change any of the hole bits of LONGWORD.
+
+	 1) Is this safe?  Will it catch all the zero bytes?
+	 Suppose there is a byte with all zeros.  Any carry bits
+	 propagating from its left will fall into the hole at its
+	 least significant bit and stop.  Since there will be no
+	 carry from its most significant bit, the LSB of the
+	 byte to the left will be unchanged, and the zero will be
+	 detected.
+
+	 2) Is this worthwhile?  Will it ignore everything except
+	 zero bytes?  Suppose every byte of LONGWORD has a bit set
+	 somewhere.  There will be a carry into bit 8.  If bit 8
+	 is set, this will carry into bit 16.  If bit 8 is clear,
+	 one of bits 9-15 must be set, so there will be a carry
+	 into bit 16.  Similarly, there will be a carry into bit
+	 24.  If one of bits 24-30 is set, there will be a carry
+	 into bit 31, so all of the hole bits will be changed.
+
+	 The one misfire occurs when bits 24-30 are clear and bit
+	 31 is set; in this case, the hole at bit 31 is not
+	 changed.  If we had access to the processor carry flag,
+	 we could close this loophole by putting the fourth hole
+	 at bit 32!
+
+	 So it ignores everything except 128's, when they're aligned
+	 properly.
+
+	 3) But wait!  Aren't we looking for C, not zero?
+	 Good point.  So what we do is XOR LONGWORD with a longword,
+	 each of whose bytes is C.  This turns each byte that is C
+	 into a zero.  */
+
+      longword = *--longword_ptr ^ charmask;
+
+      /* Add MAGIC_BITS to LONGWORD.  */
+      if ((((longword + magic_bits)
+
+	    /* Set those bits that were unchanged by the addition.  */
+	    ^ ~longword)
+
+	   /* Look at only the hole bits.  If any of the hole bits
+	      are unchanged, most likely one of the bytes was a
+	      zero.  */
+	   & ~magic_bits) != 0)
+	{
+	  /* Which of the bytes was C?  If none of them were, it was
+	     a misfire; continue the search.  */
+
+	  const unsigned char *cp = (const unsigned char *) longword_ptr;
+
+	  if (cp[3] == c)
+	    return (void *) &cp[3];
+	  if (cp[2] == c)
+	    return (void *) &cp[2];
+	  if (cp[1] == c)
+	    return (void *) &cp[1];
+	  if (cp[0] == c)
+	    return (void *) cp;
+	}
+
+      n -= sizeof (longword);
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  while (n-- > 0)
+    {
+      if (*--char_ptr == c)
+	return (void *) char_ptr;
+    }
+
+  return 0;
+}
+
+/*
+ * Borrowed from glibc-2.12.1/misc/dirname.c
+ */
+
+/* dirname - return directory part of PATH.
+   Copyright (C) 1996, 2000, 2001, 2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@cygnus.com>, 1996.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+dirname_r (char *path)
+{
+  static const char dot[] = ".";
+  char *last_slash;
+
+  /* Find last '/'.  */
+  last_slash = path != NULL ? strrchr (path, '/') : NULL;
+
+  if (last_slash != NULL && last_slash != path && last_slash[1] == '\0')
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* The '/' is the last character, we have to look further.  */
+      if (runp != path)
+        last_slash = __memrchr (path, '/', runp - path);
+    }
+
+  if (last_slash != NULL)
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* Terminate the path.  */
+      if (runp == path)
+        {
+          /* The last slash is the first character in the string.  We have to
+             return "/".  As a special case we have to return "//" if there
+             are exactly two slashes at the beginning of the string.  See
+             XBD 4.10 Path Name Resolution for more information.  */
+          if (last_slash == path + 1)
+            ++last_slash;
+          else
+            last_slash = path + 1;
+        }
+      else
+        last_slash = runp;
+
+      last_slash[0] = '\0';
+    }
+  else
+    /* This assignment is ill-designed but the XPG specs require to
+       return a string containing "." in any case no directory part is
+       found and so a static and constant string is required.  */
+    path = (char *) dot;
+
+  return path;
+}
+#endif /* THREAD_UNSAFE_DIRNAME */
diff --git a/contrib/macfuse/fuse_param.h b/contrib/macfuse/fuse_param.h
index 81d753c6cd7..347db9464bc 100644
--- a/contrib/macfuse/fuse_param.h
+++ b/contrib/macfuse/fuse_param.h
@@ -1,4 +1,9 @@
 /*
+ * 'rebel' branch modifications:
+ *     Copyright (C) 2010 Tuxera. All Rights Reserved.
+ */
+
+/*
  * Copyright (C) 2006-2008 Google. All Rights Reserved.
  * Amit Singh <singh@>
  */
@@ -6,69 +11,81 @@
 #ifndef _FUSE_PARAM_H_
 #define _FUSE_PARAM_H_
 
-/* Compile-time tunables (M_MACFUSE*) */
-
-#define M_MACFUSE_ENABLE_FIFOFS            0
-#define M_MACFUSE_ENABLE_INTERRUPT         1
-#define M_MACFUSE_ENABLE_SPECFS            0
-#define M_MACFUSE_ENABLE_TSLOCKING         0
-#define M_MACFUSE_ENABLE_UNSUPPORTED       1
-#define M_MACFUSE_ENABLE_XATTR             1
-
-#if M_MACFUSE_ENABLE_UNSUPPORTED
-  #define M_MACFUSE_ENABLE_DSELECT         0
-  #define M_MACFUSE_ENABLE_EXCHANGE        1
-  #define M_MACFUSE_ENABLE_KQUEUE          1
-  #define M_MACFUSE_ENABLE_KUNC            0
-#if __LP64__
-    #define M_MACFUSE_ENABLE_INTERIM_FSNODE_LOCK 1
-#endif /* __LP64__ */
-#endif /* M_MACFUSE_ENABLE_UNSUPPORTED */
-
-#if M_MACFUSE_ENABLE_INTERIM_FSNODE_LOCK
-#define FUSE_VNOP_EXPORT __private_extern__
+#include <AvailabilityMacros.h>
+
+/* Compile-time tunables (M_OSXFUSE*) */
+
+#define M_OSXFUSE_ENABLE_FIFOFS                0
+#define M_OSXFUSE_ENABLE_INTERRUPT             1
+#define M_OSXFUSE_ENABLE_SPECFS                0
+#define M_OSXFUSE_ENABLE_TSLOCKING             1
+#define M_OSXFUSE_ENABLE_UNSUPPORTED           1
+#define M_OSXFUSE_ENABLE_XATTR                 1
+#define M_OSXFUSE_ENABLE_DSELECT               1
+
+#if M_OSXFUSE_ENABLE_UNSUPPORTED
+#  define M_OSXFUSE_ENABLE_EXCHANGE            1
+#  define M_OSXFUSE_ENABLE_KUNC                0
+#  define M_OSXFUSE_ENABLE_INTERIM_FSNODE_LOCK 1
+#endif /* M_OSXFUSE_ENABLE_UNSUPPORTED */
+
+#if MAC_OS_X_VERSION_MIN_REQUIRED < 1060
+#  if M_OSXFUSE_ENABLE_UNSUPPORTED
+     /*
+      * In Mac OS X 10.5 the file system implementation is responsible for
+      * posting kqueue events. Starting with Mac OS X 10.6 VFS took over that
+      * job.
+      */
+#    define M_OSXFUSE_ENABLE_KQUEUE            1
+#  endif
+#endif /* MAC_OS_X_VERSION_MIN_REQUIRED < 1060 */
+
+#if M_OSXFUSE_ENABLE_INTERIM_FSNODE_LOCK
+#  define M_OSXFUSE_ENABLE_HUGE_LOCK           0
+#  define M_OSXFUSE_ENABLE_LOCK_LOGGING        0
+#  define FUSE_VNOP_EXPORT __private_extern__
 #else
-#define FUSE_VNOP_EXPORT static
-#endif /* M_MACFUSE_ENABLE_INTERIM_FSNODE_LOCK */
+#  define FUSE_VNOP_EXPORT static
+#endif /* M_OSXFUSE_ENABLE_INTERIM_FSNODE_LOCK */
 
 /* User Control */
 
-#define MACFUSE_POSTUNMOUNT_SIGNAL         SIGKILL
+#define OSXFUSE_POSTUNMOUNT_SIGNAL         SIGKILL
 
 #define MACOSX_ADMIN_GROUP_NAME            "admin"
 
-#define SYSCTL_MACFUSE_TUNABLES_ADMIN      "macfuse.tunables.admin_group"
-#define SYSCTL_MACFUSE_VERSION_NUMBER      "macfuse.version.number"
+#define SYSCTL_OSXFUSE_TUNABLES_ADMIN      "osxfuse.tunables.admin_group"
+#define SYSCTL_OSXFUSE_VERSION_NUMBER      "osxfuse.version.number"
 
 /* Paths */
 
-#define MACFUSE_BUNDLE_PATH "/Library/Filesystems/fusefs.fs"
-#define MACFUSE_KEXT        MACFUSE_BUNDLE_PATH "/Support/fusefs.kext"
-#define MACFUSE_LOAD_PROG   MACFUSE_BUNDLE_PATH "/Support/load_fusefs"
-#define MACFUSE_MOUNT_PROG  MACFUSE_BUNDLE_PATH "/Support/mount_fusefs"
+#define OSXFUSE_BUNDLE_PATH "/Library/Filesystems/osxfusefs.fs"
+#define OSXFUSE_KEXT        OSXFUSE_BUNDLE_PATH "/Support/osxfusefs.kext"
+#define OSXFUSE_LOAD_PROG   OSXFUSE_BUNDLE_PATH "/Support/load_osxfusefs"
+#define OSXFUSE_MOUNT_PROG  OSXFUSE_BUNDLE_PATH "/Support/mount_osxfusefs"
 #define SYSTEM_KEXTLOAD     "/sbin/kextload"
 #define SYSTEM_KEXTUNLOAD   "/sbin/kextunload"
 
 /* Compatible API version */
 
-#define MACFUSE_MIN_USER_VERSION_MAJOR     7
-#define MACFUSE_MIN_USER_VERSION_MINOR     5
+#define OSXFUSE_MIN_USER_VERSION_MAJOR     7
+#define OSXFUSE_MIN_USER_VERSION_MINOR     5
 
 /* Device Interface */
 
 /*
- * This is the prefix ("fuse" by default) of the name of a FUSE device node
- * in devfs. The suffix is the device number. "/dev/fuse0" is the first FUSE
+ * This is the prefix ("osxfuse" by default) of the name of a FUSE device node
+ * in devfs. The suffix is the device number. "/dev/osxfuse0" is the first FUSE
  * device by default. If you change the prefix from the default to something
  * else, the user-space FUSE library will need to know about it too.
  */
-#define MACFUSE_DEVICE_BASENAME            "fuse"
+#define OSXFUSE_DEVICE_BASENAME            "osxfuse"
 
 /*
- * This is the number of /dev/fuse<n> nodes we will create. <n> goes from
- * 0 to (FUSE_NDEVICES - 1).
+ * This is the number of /dev/osxfuse<n> nodes we will create. <n> goes from
+ * 0 to (OSXFUSE_NDEVICES - 1).
  */
-#define MACFUSE_NDEVICES                   24
+#define OSXFUSE_NDEVICES                   24
 
 /*
  * This is the default block size of the virtual storage devices that are
@@ -131,13 +148,13 @@
 /* User-Kernel IPC Buffer */
 
 #define FUSE_MIN_USERKERNEL_BUFSIZE        (128  * 1024)
-#define FUSE_MAX_USERKERNEL_BUFSIZE        (4096 * 1024)
+#define FUSE_MAX_USERKERNEL_BUFSIZE        (16   * 1024 * 1024)
 
 #define FUSE_REASONABLE_XATTRSIZE          FUSE_MIN_USERKERNEL_BUFSIZE
 
 #endif /* KERNEL */
 
-#define FUSE_DEFAULT_USERKERNEL_BUFSIZE    (4096 * 1024)
+#define FUSE_DEFAULT_USERKERNEL_BUFSIZE    (16   * 1024 * 1024)
 
 #define FUSE_LINK_MAX                      LINK_MAX
 #define FUSE_UIO_BACKUP_MAX                8
diff --git a/contrib/macfuse/mount_darwin.c b/contrib/macfuse/mount_darwin.c
index 9d87fca3596..d1d1c34e761 100644
--- a/contrib/macfuse/mount_darwin.c
+++ b/contrib/macfuse/mount_darwin.c
@@ -1,5 +1,5 @@
 /*
- * Derived from mount_bsd.c from the fuse distribution.
+ *  Derived from mount_bsd.c from the fuse distribution.
  *
  *  FUSE: Filesystem in Userspace
  *  Copyright (C) 2005-2006 Csaba Henk <csaba.henk@creo.hu>
@@ -34,323 +34,231 @@
 #include "fuse_param.h"
 #include "fuse_ioctl.h"
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
 
-static long
-fuse_os_version_major_np(void)
-{
-    int ret = 0;
-    long major = 0;
-    char *c = NULL;
-    struct utsname u;
-    size_t oldlen;
-
-    oldlen = sizeof(u.release);
-
-    ret = sysctlbyname("kern.osrelease", u.release, &oldlen, NULL, 0);
-    if (ret != 0) {
-        return -1;
-    }
-
-    c = strchr(u.release, '.');
-    if (c == NULL) {
-        return -1;
-    }
-
-    *c = '\0';
-
-    errno = 0;
-    major = strtol(u.release, NULL, 10);
-    if ((errno == EINVAL) || (errno == ERANGE)) {
-        return -1;
-    }
-
-    return major;
-}
-
-static int
-fuse_running_under_rosetta(void)
-{
-    int result = 0;
-    int is_native = 1;
-    size_t sz = sizeof(result);
-
-    int ret = sysctlbyname("sysctl.proc_native", &result, &sz, NULL, (size_t)0);
-    if ((ret == 0) && !result) {
-        is_native = 0;
-    }
-
-    return !is_native;
-}
-
-static int
-loadkmod(void)
-{
-    int result = -1;
-    int pid, terminated_pid;
-    union wait status;
-    long major;
-
-    major = fuse_os_version_major_np();
-
-    if (major < 9) { /* not Mac OS X 10.5+ */
-        return EINVAL;
-    }
-
-    pid = fork();
-
-    if (pid == 0) {
-        execl(MACFUSE_LOAD_PROG, MACFUSE_LOAD_PROG, NULL);
-
-        /* exec failed */
-        exit(ENOENT);
-    }
-
-    require_action(pid != -1, Return, result = errno);
-
-    while ((terminated_pid = wait4(pid, (int *)&status, 0, NULL)) < 0) {
-        /* retry if EINTR, else break out with error */
-        if (errno != EINTR) {
-            break;
-        }
-    }
-
-    if ((terminated_pid == pid) && (WIFEXITED(status))) {
-        result = WEXITSTATUS(status);
-    } else {
-        result = -1;
-    }
-
-Return:
-    check_noerr_string(result, strerror(errno));
-
-    return result;
-}
-
 int
-gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param)
+gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
+               pid_t *mnt_pid, int status_fd) /* Not used on OS X */
 {
-    int fd, pid;
-    int result;
-    char *fdnam, *dev;
-    const char *mountprog = MACFUSE_MOUNT_PROG;
-    sig_t chldf;
+        int fd       = 0;
+        int pid      = 0;
+        int ret      = 0;
+        char *fdnam  = NULL;
+        char *dev    = NULL;
+        char vstr[4];
+        unsigned vval = 0;
+        int i        = 0;
+
+        const char *mountprog              = OSXFUSE_MOUNT_PROG;
+        sig_t chldf                        = SIG_ERR;
+        char   version[MAXHOSTNAMELEN + 1] = { 0 };
+        size_t version_len                 = MAXHOSTNAMELEN;
+        size_t version_len_desired         = 0;
+        int r                              = 0;
+        char devpath[MAXPATHLEN]           = { 0 };;
+
+        if (!mountpoint) {
+                gf_log ("glustefs-fuse", GF_LOG_ERROR,
+                        "missing or invalid mount point");
+                goto err;
+        }
 
-    /* mount_fusefs should not try to spawn the daemon */
-    setenv("MOUNT_FUSEFS_SAFE", "1", 1);
+        /* mount_fusefs should not try to spawn the daemon */
+        setenv("MOUNT_FUSEFS_SAFE", "1", 1);
 
-    /* to notify mount_fusefs it's called from lib */
-    setenv("MOUNT_FUSEFS_CALL_BY_LIB", "1", 1);
+        /* to notify mount_fusefs it's called from lib */
+        setenv("MOUNT_FUSEFS_CALL_BY_LIB", "1", 1);
 
-    if (!mountpoint) {
-        fprintf(stderr, "missing or invalid mount point\n");
-        return -1;
-    }
+        chldf = signal(SIGCHLD, SIG_DFL); /* So that we can wait4() below. */
 
-    if (fuse_running_under_rosetta()) {
-        fprintf(stderr, "MacFUSE does not work under Rosetta\n");
-        return -1;
-    }
+        if (chldf == SIG_ERR) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "signal() returned SIG_ERR: %s",
+                        strerror(errno));
+                goto err;
+        }
 
-    chldf = signal(SIGCHLD, SIG_DFL); /* So that we can wait4() below. */
+        /* check for user<->kernel match. */
+        ret = sysctlbyname(SYSCTL_OSXFUSE_VERSION_NUMBER, version,
+                              &version_len, NULL, (size_t)0);
+        if (ret != 0) {
+                gf_log ("glustefs-fuse", GF_LOG_ERROR,
+                        "sysctlbyname() returned error: %s",
+                        strerror(errno));
+                goto err;
+        }
 
-    result = loadkmod();
-    if (result == EINVAL)
-        GFFUSE_LOGERR("OS X >= 10.5 (at least Leopard) required");
-    else if (result == 0 || result == ENOENT || result == EBUSY) {
-        /* Module loaded, but now need to check for user<->kernel match. */
+        /* sysctlbyname() includes the trailing '\0' in version_len */
+        version_len_desired = sizeof ("2.x.y");
 
-        char   version[MAXHOSTNAMELEN + 1] = { 0 };
-        size_t version_len = MAXHOSTNAMELEN;
-        size_t version_len_desired = 0;
+        if (version_len != version_len_desired) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "version length mismatch for OSXFUSE %s",
+                        version);
+                ret = -1;
+                goto err;
+        }
 
-        result = sysctlbyname(SYSCTL_MACFUSE_VERSION_NUMBER, version,
-                              &version_len, NULL, (size_t)0);
-        if (result == 0) {
-            /* sysctlbyname() includes the trailing '\0' in version_len */
-            version_len_desired = strlen("2.x.y") + 1;
-
-            if (version_len != version_len_desired)
-                result = -1;
-        } else
-            strcpy(version, "?.?.?");
-        if (result == 0) {
-            char *ep;
-            char vstr[4];
-            unsigned vval;
-            int i;
-
-            for (i = 0; i < 3; i++)
+        for (i = 0; i < 3; i++)
                 vstr[i] = version[2*i];
-            vstr[3] = '\0';
-
-            vval = strtoul(vstr, &ep, 10);
-            if (*ep || vval < 203 || vval > 217)
-                result = -1;
-            else
-                gf_log("glusterfs-fuse", GF_LOG_INFO,
-                       "MacFUSE kext version %s", version);
-        }
-        if (result != 0)
-            GFFUSE_LOGERR("MacFUSE version %s is not supported", version);
-    } else
-        GFFUSE_LOGERR("cannot load MacFUSE kext");
-    if (result != 0)
-        return -1;
-
-    fdnam = getenv("FUSE_DEV_FD");
-
-    if (fdnam) {
-        char *ep;
-
-        fd = strtol(fdnam, &ep, 10);
-        if (*ep != '\0' || fd < 0) {
-            GFFUSE_LOGERR("invalid value given in FUSE_DEV_FD");
-            return -1;
+        vstr[3] = '\0';
+
+        vval = strtoul(vstr, NULL, 10);
+        if (vval < 264) {
+                GFFUSE_LOGERR("OSXFUSE version %s is not supported", version);
+                ret = -1;
+                goto err;
         }
 
-        goto mount;
-    }
+        gf_log("glusterfs-fuse", GF_LOG_INFO,
+               "OSXFUSE kext version supported %s", version);
 
-    dev = getenv("FUSE_DEV_NAME");
-    if (dev) {
-        if ((fd = open(dev, O_RDWR)) < 0) {
-            GFFUSE_LOGERR("failed to open device (%s)", strerror(errno));
-            return -1;
+        fdnam = getenv("FUSE_DEV_FD");
+        if (fdnam) {
+                fd = strtol(fdnam, NULL, 10);
+                if (fd < 0) {
+                        GFFUSE_LOGERR("invalid value given in FUSE_DEV_FD");
+                        ret = -1;
+                        goto err;
+                }
+                goto mount;
         }
-    } else {
-        int r, devidx = -1;
-        char devpath[MAXPATHLEN];
-
-        for (r = 0; r < MACFUSE_NDEVICES; r++) {
-            snprintf(devpath, MAXPATHLEN - 1,
-                     _PATH_DEV MACFUSE_DEVICE_BASENAME "%d", r);
-            fd = open(devpath, O_RDWR);
-            if (fd >= 0) {
-                dev = devpath;
-                devidx = r;
-                break;
-            }
+
+        dev = getenv("FUSE_DEV_NAME");
+        if (!dev) {
+                for (r = 0; r < OSXFUSE_NDEVICES; r++) {
+                        snprintf(devpath, MAXPATHLEN - 1,
+                                 _PATH_DEV OSXFUSE_DEVICE_BASENAME "%d", r);
+                        if ((fd = open(devpath, O_RDWR)) < 0) {
+                                GFFUSE_LOGERR("failed to open device %s (%s)",
+                                              devpath,
+                                              strerror(errno));
+                                goto err;
+                        }
+                        dev = devpath;
+                        goto mount;
+                }
         }
-        if (devidx == -1) {
-            GFFUSE_LOGERR("failed to open device (%s)", strerror(errno));
-            return -1;
+
+        fd = open(dev, O_RDWR);
+        if (fd < 0) {
+                GFFUSE_LOGERR("failed to open device %s (%s)", dev,
+                              strerror(errno));
+                ret = -1;
+                goto err;
         }
-    }
 
 mount:
-    if (getenv("FUSE_NO_MOUNT") || ! mountpoint)
-        goto out;
-
-    signal(SIGCHLD, chldf);
-
-    pid = fork();
-
-    if (pid == -1) {
-        GFFUSE_LOGERR("fork() failed (%s)", strerror(errno));
-        close(fd);
-        return -1;
-    }
-
-    if (pid == 0) {
+        signal(SIGCHLD, chldf);
 
         pid = fork();
         if (pid == -1) {
-            GFFUSE_LOGERR("fork() failed (%s)", strerror(errno));
-            close(fd);
-            exit(1);
+                GFFUSE_LOGERR("fork() failed (%s)", strerror(errno));
+                ret = -1;
+                goto err;
         }
 
         if (pid == 0) {
-            const char *argv[32];
-            int a = 0;
-            char *opts = NULL;
-
-            if (asprintf(&opts, "%s,fssubtype=glusterfs", mnt_param) == -1) {
-                GFFUSE_LOGERR("Out of memory");
-		exit(1);
-            }
-
-            if (! fdnam)
-                asprintf(&fdnam, "%d", fd);
-
-            argv[a++] = mountprog;
-            if (opts) {
-                argv[a++] = "-o";
-                argv[a++] = opts;
-            }
-            argv[a++] = fdnam;
-            argv[a++] = mountpoint;
-            argv[a++] = NULL;
-
-            {
-                char title[MAXPATHLEN + 1] = { 0 };
-                u_int32_t len = MAXPATHLEN;
-                int ret = proc_pidpath(getpid(), title, len);
-                if (ret) {
-                    setenv("MOUNT_FUSEFS_DAEMON_PATH", title, 1);
+                pid = fork();
+                if (pid == -1) {
+                        GFFUSE_LOGERR("fork() failed (%s)", strerror(errno));
+                        ret = -1;
+                        goto err;
                 }
-            }
-            execvp(mountprog, (char **) argv);
-            GFFUSE_LOGERR("MacFUSE: failed to exec mount program (%s)", strerror(errno));
-            exit(1);
-        }
 
-        _exit(0);
-    }
-
-out:
-    return fd;
+                if (pid == 0) {
+                        const char *argv[32];
+                        int a = 0;
+                        char *opts = NULL;
+
+                        if (asprintf(&opts, "%s,fssubtype=glusterfs",
+                                     mnt_param) == -1) {
+                                GFFUSE_LOGERR("asprintf() error: %s",
+                                              strerror(errno));
+                                ret = -1;
+                                goto err;
+                        }
+
+                        if (!fdnam)
+                                asprintf(&fdnam, "%d", fd);
+
+                        argv[a++] = mountprog;
+                        if (opts) {
+                                argv[a++] = "-o";
+                                argv[a++] = opts;
+                        }
+                        argv[a++] = fdnam;
+                        argv[a++] = mountpoint;
+                        argv[a++] = NULL;
+
+                        {
+                                char title[MAXPATHLEN + 1] = { 0 };
+                                u_int32_t len = MAXPATHLEN;
+                                int ret = proc_pidpath(getpid(), title, len);
+                                if (ret) {
+                                        setenv("MOUNT_FUSEFS_DAEMON_PATH",
+                                               title, 1);
+                                }
+                        }
+                        execvp(mountprog, (char **) argv);
+                        GFFUSE_LOGERR("OSXFUSE: failed to exec mount"
+                                      " program (%s)", strerror(errno));
+                        _exit(1);
+                }
+                _exit(0);
+        }
+        ret = fd;
+err:
+        if (ret == -1) {
+                if (fd > 0) {
+                        close(fd);
+                }
+        }
+        return ret;
 }
 
 void
 gf_fuse_unmount(const char *mountpoint, int fd)
 {
-    int ret;
-    struct stat sbuf;
-    char dev[128];
-    char resolved_path[PATH_MAX];
-    char *ep, *rp = NULL;
-
-    unsigned int hs_complete = 0;
+        int ret;
+        struct stat sbuf;
+        char dev[128];
+        char resolved_path[PATH_MAX];
+        char *ep, *rp = NULL;
 
-    ret = ioctl(fd, FUSEDEVIOCGETHANDSHAKECOMPLETE, &hs_complete);
-    if (ret || !hs_complete) {
-        return;
-    }
-    /* XXX does this have any use here? */
-    ret = ioctl(fd,  FUSEDEVIOCSETDAEMONDEAD, &fd);
-    if (ret) {
-        return;
-    }
+        unsigned int hs_complete = 0;
 
-    if (fstat(fd, &sbuf) == -1) {
-        return;
-    }
+        ret = ioctl(fd, FUSEDEVIOCGETHANDSHAKECOMPLETE, &hs_complete);
+        if (ret || !hs_complete) {
+                return;
+        }
 
-    devname_r(sbuf.st_rdev, S_IFCHR, dev, 128);
+        if (fstat(fd, &sbuf) == -1) {
+                return;
+        }
 
-    if (strncmp(dev, MACFUSE_DEVICE_BASENAME,
-                sizeof(MACFUSE_DEVICE_BASENAME) - 1)) {
-        return;
-    }
+        devname_r(sbuf.st_rdev, S_IFCHR, dev, 128);
 
-    strtol(dev + 4, &ep, 10);
-    if (*ep != '\0') {
-        return;
-    }
+        if (strncmp(dev, OSXFUSE_DEVICE_BASENAME,
+                    sizeof(OSXFUSE_DEVICE_BASENAME) - 1)) {
+                return;
+        }
 
-    rp = realpath(mountpoint, resolved_path);
-    if (rp) {
-        ret = unmount(resolved_path, 0);
-    }
+        strtol(dev + sizeof(OSXFUSE_DEVICE_BASENAME) - 1, &ep, 10);
+        if (*ep != '\0') {
+                return;
+        }
 
-    close(fd);
+        rp = realpath(mountpoint, resolved_path);
+        if (rp) {
+                ret = unmount(resolved_path, 0);
+        }
 
-    return;
+        close(fd);
+        return;
 }
diff --git a/contrib/md5/md5.c b/contrib/md5/md5.c
deleted file mode 100644
index 5f0d0d157bf..00000000000
--- a/contrib/md5/md5.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * RFC 1321 compliant MD5 implementation
- *
- * Copyright (C) 2001-2003 Christophe Devine
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "md5.h"
-
-void md5_begin(md_context *ctx)
-{
-	ctx->A = 0x67452301;
-	ctx->B = 0xEFCDAB89;
-	ctx->C = 0x98BADCFE;
-	ctx->D = 0x10325476;
-
-	ctx->totalN = ctx->totalN2 = 0;
-}
-
-static void md5_process(md_context *ctx, const uint8_t data[CSUM_CHUNK])
-{
-	uint32_t X[16], A, B, C, D;
-
-	A = ctx->A;
-	B = ctx->B;
-	C = ctx->C;
-	D = ctx->D;
-
-	X[0] = IVAL(data, 0);
-	X[1] = IVAL(data, 4);
-	X[2] = IVAL(data, 8);
-	X[3] = IVAL(data, 12);
-	X[4] = IVAL(data, 16);
-	X[5] = IVAL(data, 20);
-	X[6] = IVAL(data, 24);
-	X[7] = IVAL(data, 28);
-	X[8] = IVAL(data, 32);
-	X[9] = IVAL(data, 36);
-	X[10] = IVAL(data, 40);
-	X[11] = IVAL(data, 44);
-	X[12] = IVAL(data, 48);
-	X[13] = IVAL(data, 52);
-	X[14] = IVAL(data, 56);
-	X[15] = IVAL(data, 60);
-
-#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n)))
-
-#define P(a,b,c,d,k,s,t) a += F(b,c,d) + X[k] + t, a = S(a,s) + b
-
-#define F(x,y,z) (z ^ (x & (y ^ z)))
-
-	P(A, B, C, D,  0,  7, 0xD76AA478);
-	P(D, A, B, C,  1, 12, 0xE8C7B756);
-	P(C, D, A, B,  2, 17, 0x242070DB);
-	P(B, C, D, A,  3, 22, 0xC1BDCEEE);
-	P(A, B, C, D,  4,  7, 0xF57C0FAF);
-	P(D, A, B, C,  5, 12, 0x4787C62A);
-	P(C, D, A, B,  6, 17, 0xA8304613);
-	P(B, C, D, A,  7, 22, 0xFD469501);
-	P(A, B, C, D,  8,  7, 0x698098D8);
-	P(D, A, B, C,  9, 12, 0x8B44F7AF);
-	P(C, D, A, B, 10, 17, 0xFFFF5BB1);
-	P(B, C, D, A, 11, 22, 0x895CD7BE);
-	P(A, B, C, D, 12,  7, 0x6B901122);
-	P(D, A, B, C, 13, 12, 0xFD987193);
-	P(C, D, A, B, 14, 17, 0xA679438E);
-	P(B, C, D, A, 15, 22, 0x49B40821);
-
-#undef F
-#define F(x,y,z) (y ^ (z & (x ^ y)))
-
-	P(A, B, C, D,  1,  5, 0xF61E2562);
-	P(D, A, B, C,  6,  9, 0xC040B340);
-	P(C, D, A, B, 11, 14, 0x265E5A51);
-	P(B, C, D, A,  0, 20, 0xE9B6C7AA);
-	P(A, B, C, D,  5,  5, 0xD62F105D);
-	P(D, A, B, C, 10,  9, 0x02441453);
-	P(C, D, A, B, 15, 14, 0xD8A1E681);
-	P(B, C, D, A,  4, 20, 0xE7D3FBC8);
-	P(A, B, C, D,  9,  5, 0x21E1CDE6);
-	P(D, A, B, C, 14,  9, 0xC33707D6);
-	P(C, D, A, B,  3, 14, 0xF4D50D87);
-	P(B, C, D, A,  8, 20, 0x455A14ED);
-	P(A, B, C, D, 13,  5, 0xA9E3E905);
-	P(D, A, B, C,  2,  9, 0xFCEFA3F8);
-	P(C, D, A, B,  7, 14, 0x676F02D9);
-	P(B, C, D, A, 12, 20, 0x8D2A4C8A);
-
-#undef F
-#define F(x,y,z) (x ^ y ^ z)
-
-	P(A, B, C, D,  5,  4, 0xFFFA3942);
-	P(D, A, B, C,  8, 11, 0x8771F681);
-	P(C, D, A, B, 11, 16, 0x6D9D6122);
-	P(B, C, D, A, 14, 23, 0xFDE5380C);
-	P(A, B, C, D,  1,  4, 0xA4BEEA44);
-	P(D, A, B, C,  4, 11, 0x4BDECFA9);
-	P(C, D, A, B,  7, 16, 0xF6BB4B60);
-	P(B, C, D, A, 10, 23, 0xBEBFBC70);
-	P(A, B, C, D, 13,  4, 0x289B7EC6);
-	P(D, A, B, C,  0, 11, 0xEAA127FA);
-	P(C, D, A, B,  3, 16, 0xD4EF3085);
-	P(B, C, D, A,  6, 23, 0x04881D05);
-	P(A, B, C, D,  9,  4, 0xD9D4D039);
-	P(D, A, B, C, 12, 11, 0xE6DB99E5);
-	P(C, D, A, B, 15, 16, 0x1FA27CF8);
-	P(B, C, D, A,  2, 23, 0xC4AC5665);
-
-#undef F
-#define F(x,y,z) (y ^ (x | ~z))
-
-	P(A, B, C, D,  0,  6, 0xF4292244);
-	P(D, A, B, C,  7, 10, 0x432AFF97);
-	P(C, D, A, B, 14, 15, 0xAB9423A7);
-	P(B, C, D, A,  5, 21, 0xFC93A039);
-	P(A, B, C, D, 12,  6, 0x655B59C3);
-	P(D, A, B, C,  3, 10, 0x8F0CCC92);
-	P(C, D, A, B, 10, 15, 0xFFEFF47D);
-	P(B, C, D, A,  1, 21, 0x85845DD1);
-	P(A, B, C, D,  8,  6, 0x6FA87E4F);
-	P(D, A, B, C, 15, 10, 0xFE2CE6E0);
-	P(C, D, A, B,  6, 15, 0xA3014314);
-	P(B, C, D, A, 13, 21, 0x4E0811A1);
-	P(A, B, C, D,  4,  6, 0xF7537E82);
-	P(D, A, B, C, 11, 10, 0xBD3AF235);
-	P(C, D, A, B,  2, 15, 0x2AD7D2BB);
-	P(B, C, D, A,  9, 21, 0xEB86D391);
-
-#undef F
-
-	ctx->A += A;
-	ctx->B += B;
-	ctx->C += C;
-	ctx->D += D;
-}
-
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length)
-{
-	uint32_t left, fill;
-
-	if (!length)
-		return;
-
-	left = ctx->totalN & 0x3F;
-	fill = CSUM_CHUNK - left;
-
-	ctx->totalN += length;
-	ctx->totalN &= 0xFFFFFFFF;
-
-	if (ctx->totalN < length)
-		ctx->totalN2++;
-
-	if (left && length >= fill) {
-		memcpy(ctx->buffer + left, input, fill);
-		md5_process(ctx, ctx->buffer);
-		length -= fill;
-		input  += fill;
-		left = 0;
-	}
-
-	while (length >= CSUM_CHUNK) {
-		md5_process(ctx, input);
-		length -= CSUM_CHUNK;
-		input  += CSUM_CHUNK;
-	}
-
-	if (length)
-		memcpy(ctx->buffer + left, input, length);
-}
-
-static uint8_t md5_padding[CSUM_CHUNK] = { 0x80 };
-
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN])
-{
-	uint32_t last, padn;
-	uint32_t high, low;
-	uint8_t msglen[8];
-
-	high = (ctx->totalN >> 29)
-	     | (ctx->totalN2 <<  3);
-	low  = (ctx->totalN <<  3);
-
-	SIVAL(msglen, 0, low);
-	SIVAL(msglen, 4, high);
-
-	last = ctx->totalN & 0x3F;
-	padn = last < 56 ? 56 - last : 120 - last;
-
-	md5_update(ctx, md5_padding, padn);
-	md5_update(ctx, msglen, 8);
-
-	SIVAL(digest, 0, ctx->A);
-	SIVAL(digest, 4, ctx->B);
-	SIVAL(digest, 8, ctx->C);
-	SIVAL(digest, 12, ctx->D);
-}
-
-void get_md5(uint8_t *out, const uint8_t *input, int n)
-{
-	md_context ctx;
-	md5_begin(&ctx);
-	md5_update(&ctx, input, n);
-	md5_result(&ctx, out);
-}
-
-#ifdef TEST_MD5
-
-#include <stdlib.h>
-#include <stdio.h>
-
-/*
- * those are the standard RFC 1321 test vectors
- */
-
-static struct {
-    char *str, *md5;
-} tests[] = {
- { "",
-   "d41d8cd98f00b204e9800998ecf8427e" },
- { "a",
-   "0cc175b9c0f1b6a831c399e269772661" },
- { "abc",
-   "900150983cd24fb0d6963f7d28e17f72" },
- { "message digest",
-   "f96b697d7cb7938d525a2f31aaf161d0" },
- { "abcdefghijklmnopqrstuvwxyz",
-   "c3fcd3d76192e4007dfb496cca67e13b" },
- { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
-   "d174ab98d277d9f5a5611c2c9f419d9f" },
- { "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
-   "57edf4a22be3c955ac49da2e2107b67a" },
- { NULL, NULL }
-};
-
-int main(int argc, char *argv[])
-{
-	FILE *f;
-	int i, j;
-	char output[33];
-	md_context ctx;
-	uint8_t buf[1000];
-	uint8_t md5sum[MD5_DIGEST_LEN];
-
-	if (argc < 2) {
-		printf("\nMD5 Validation Tests:\n\n");
-
-		for (i = 0; tests[i].str; i++) {
-			char *str = tests[i].str;
-			char *chk = tests[i].md5;
-
-			printf("  Test %d ", i + 1);
-
-			get_md5(md5sum, str, strlen(str));
-
-			for (j = 0; j < MD5_DIGEST_LEN; j++)
-				sprintf(output + j * 2, "%02x", md5sum[j]);
-
-			if (memcmp(output, chk, 32)) {
-				printf("failed!\n");
-				return 1;
-			}
-
-			printf("passed.\n");
-		}
-
-		printf("\n");
-		return 0;
-	}
-
-	while (--argc) {
-		if (!(f = fopen(*++argv, "rb"))) {
-			perror("fopen");
-			return 1;
-		}
-
-		md5_begin(&ctx);
-
-		while ((i = fread(buf, 1, sizeof buf, f)) > 0)
-			md5_update(&ctx, buf, i);
-
-		fclose(f);
-
-		md5_result(&ctx, md5sum);
-
-		for (j = 0; j < MD5_DIGEST_LEN; j++)
-			printf("%02x", md5sum[j]);
-
-		printf("  %s\n", *argv);
-	}
-
-	return 0;
-}
-
-#endif
diff --git a/contrib/md5/md5.h b/contrib/md5/md5.h
deleted file mode 100644
index ba8f08dbcfa..00000000000
--- a/contrib/md5/md5.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/* rsync-3.0.6/byteorder.h */
-
-/*
- * Simple byteorder handling.
- *
- * Copyright (C) 1992-1995 Andrew Tridgell
- * Copyright (C) 2007-2008 Wayne Davison
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-#undef CAREFUL_ALIGNMENT
-
-/* We know that the x86 can handle misalignment and has the same
- * byte order (LSB-first) as the 32-bit numbers we transmit. */
-
-#ifdef __i386__
-#define CAREFUL_ALIGNMENT 0
-#endif
-
-#ifndef CAREFUL_ALIGNMENT
-#define CAREFUL_ALIGNMENT 1
-#endif
-
-#define CVAL(buf,pos) (((unsigned char *)(buf))[pos])
-#define UVAL(buf,pos) ((uint32_t)CVAL(buf,pos))
-#define SCVAL(buf,pos,val) (CVAL(buf,pos) = (val))
-
-#if CAREFUL_ALIGNMENT
-#define PVAL(buf,pos) (UVAL(buf,pos)|UVAL(buf,(pos)+1)<<8)
-#define IVAL(buf,pos) (PVAL(buf,pos)|PVAL(buf,(pos)+2)<<16)
-#define SSVALX(buf,pos,val) (CVAL(buf,pos)=(val)&0xFF,CVAL(buf,pos+1)=(val)>>8)
-#define SIVALX(buf,pos,val) (SSVALX(buf,pos,val&0xFFFF),SSVALX(buf,pos+2,val>>16))
-#define SIVAL(buf,pos,val) SIVALX((buf),(pos),((uint32_t)(val)))
-#else
-
-/* this handles things for architectures like the 386 that can handle
-   alignment errors */
-
-/*
-   WARNING: This section is dependent on the length of int32
-   being correct. set CAREFUL_ALIGNMENT if it is not.
-*/
-
-#define IVAL(buf,pos) (*(uint32_t *)((char *)(buf) + (pos)))
-#define SIVAL(buf,pos,val) IVAL(buf,pos)=((uint32_t)(val))
-#endif
-
-/* The include file for both the MD4 and MD5 routines. */
-
-#define MD5_DIGEST_LEN 16
-#define MAX_DIGEST_LEN MD5_DIGEST_LEN
-
-#define CSUM_CHUNK 64
-
-typedef struct {
-	uint32_t A, B, C, D;
-	uint32_t totalN;          /* bit count, lower 32 bits */
-	uint32_t totalN2;         /* bit count, upper 32 bits */
-	uint8_t buffer[CSUM_CHUNK];
-} md_context;
-
-void md5_begin(md_context *ctx);
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length);
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN]);
-
-void get_md5(uint8_t digest[MD5_DIGEST_LEN], const uint8_t *input, int n);
diff --git a/contrib/mount/mntent.c b/contrib/mount/mntent.c
new file mode 100644
index 00000000000..9a7e5f39bdb
--- /dev/null
+++ b/contrib/mount/mntent.c
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 1980, 1989, 1993, 1994
+ *      The Regents of the University of California.  All rights reserved.
+ * Copyright (c) 2001
+ *      David Rufino <daverufino@btinternet.com>
+ * Copyright (c) 2014
+ *      Red Hat, Inc. <http://www.redhat.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by the University of
+ *      California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if !defined(GF_LINUX_HOST_OS)
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/param.h>
+#include <sys/ucred.h>
+#include <sys/mount.h>
+#include "mntent_compat.h"
+
+#ifdef __NetBSD__
+typedef struct statvfs gf_statfs_t;
+#else
+typedef struct statfs gf_statfs_t;
+#endif
+
+typedef struct _mntent_state {
+        struct mntent  mntent;
+        gf_statfs_t   *statfs;
+        int            count;
+        int            pos;
+        /* A buffer big enough to store all defined flags as a string.
+         * Increase it if necessary when more flags are defined. */
+        char           buf[256];
+} mntent_state_t;
+
+typedef struct _mntflag {
+        unsigned long  value;
+        const char    *on;
+        const char    *off;
+} mntflag_t;
+
+static mntflag_t mntflags[] = {
+        { MNT_RDONLY,      "ro",          "rw" },
+        { MNT_SYNCHRONOUS, "sync",        NULL },
+        { MNT_NOEXEC,      "noexec",      NULL },
+        { MNT_NOSUID,      "nosuid",      NULL },
+#if !defined(__FreeBSD__)
+        { MNT_NODEV,       "nodev",       NULL },
+#endif /* __FreeBSD__ */
+        { MNT_UNION,       "union",       NULL },
+        { MNT_ASYNC,       "async",       NULL },
+#if !defined(GF_DARWIN_HOST_OS)
+        { MNT_NOATIME,     "noatime",     NULL },
+#if !defined(__NetBSD__)
+        { MNT_NOCLUSTERR,  "noclusterr",  NULL },
+        { MNT_NOCLUSTERW,  "noclusterw",  NULL },
+        { MNT_NOSYMFOLLOW, "nosymfollow", NULL },
+        { MNT_SUIDDIR,     "suiddir",     NULL },
+#endif /* !__NetBSD__ */
+#endif /* !GF_DARWIN_HOST_OS */
+        { 0,               NULL,          NULL }
+};
+
+char *
+hasmntopt (const struct mntent *mnt, const char *option)
+{
+        char *opt, *optbuf;
+        int len;
+
+        optbuf = strdup(mnt->mnt_opts);
+        if (optbuf == NULL) {
+                return NULL;
+        }
+
+        opt = optbuf;
+        len = 0;
+        while (*opt) {
+                while (opt[len] != 0) {
+                        if (opt[len] == ' ') {
+                                opt[len++] = 0;
+                                break;
+                        }
+                        len++;
+                }
+                if ((*opt != 0) && (strcasecmp(opt, option) == 0)) {
+                        break;
+                }
+                opt += len;
+                len = 0;
+        }
+        free(optbuf);
+        if (len == 0) {
+                return NULL;
+        }
+
+        return opt - optbuf + mnt->mnt_opts;
+}
+
+static int
+writeopt(const char *text, char *buf, int buflen, int pos)
+{
+        int len;
+
+        /* buflen must be > 0 */
+
+        if (text == NULL) {
+                return pos;
+        }
+
+        buf += pos;
+        if (pos > 0) {
+                /* We are sure we have at least one byte to store the space.
+                 * We don't need to check buflen here. */
+                *buf++ = ' ';
+                pos++;
+        }
+        len = strlen(text) + 1;
+        pos += len;
+        if (pos >= buflen) {
+                /* There won't be enough space for the text and the
+                 * terminating null character. We copy as much as we can
+                 * of the text and mark the end of the string with '...' */
+                memcpy(buf, text, buflen - pos + len);
+                if (buflen > 3) {
+                        strcpy(buf + buflen - 4, "...");
+                } else {
+                        strncpy(buf, "...", buflen - 1);
+                        buf[buflen - 1] = 0;
+                }
+                pos = buflen;
+        } else {
+                memcpy(buf, text, len);
+        }
+
+        return pos;
+}
+
+static char *
+flags2opts (int flags, char *buf, int buflen)
+{
+        char other[16];
+        mntflag_t *flg;
+        int pos;
+
+        if (buflen == 0) {
+                return NULL;
+        }
+
+        pos = 0;
+        for (flg = mntflags; flg->value != 0; flg++) {
+                pos = writeopt((flags & flg->value) == 0 ? flg->off : flg->on,
+                               buf, buflen, pos);
+                flags &= ~flg->value;
+        }
+
+        if (flags != 0) {
+                sprintf(other, "[0x%x]", flags);
+                writeopt(other, buf, buflen, pos);
+        }
+
+        return buf;
+}
+
+static void
+statfs_to_mntent (struct mntent *mntent, gf_statfs_t *mntbuf, char *buf,
+                  int buflen)
+{
+        int f_flags;
+
+        mntent->mnt_fsname = mntbuf->f_mntfromname;
+        mntent->mnt_dir = mntbuf->f_mntonname;
+        mntent->mnt_type = mntbuf->f_fstypename;
+
+#ifdef __NetBSD__
+        f_flags = mntbuf->f_flag;
+#else
+        f_flags = mntbuf->f_flags;
+#endif
+        mntent->mnt_opts = flags2opts (f_flags, buf, buflen);
+
+        mntent->mnt_freq = mntent->mnt_passno = 0;
+}
+
+struct mntent *
+getmntent_r (FILE *fp, struct mntent *mntent, char *buf, int buflen)
+{
+        mntent_state_t *state = (mntent_state_t *)fp;
+
+        if (state->pos >= state->count) {
+                return NULL;
+        }
+
+        statfs_to_mntent(mntent, &state->statfs[state->pos++], buf, buflen);
+
+        return mntent;
+}
+
+struct mntent *
+getmntent (FILE *fp)
+{
+        mntent_state_t *state = (mntent_state_t *)fp;
+
+        return getmntent_r(fp, &state->mntent, state->buf,
+                           sizeof(state->buf));
+}
+
+FILE *
+setmntent (const char *filename, const char *type)
+{
+        mntent_state_t *state;
+
+        /* We don't really need to access any file so we'll use the FILE* as
+         * a fake file to store state information.
+         */
+
+        state = malloc(sizeof(mntent_state_t));
+        if (state != NULL) {
+                state->pos = 0;
+                state->count = getmntinfo(&state->statfs, MNT_NOWAIT);
+        }
+
+        return (FILE *)state;
+}
+
+int
+endmntent (FILE *fp)
+{
+        free(fp);
+
+        return 1; /* endmntent() always returns 1 */
+}
+
+#endif /* !GF_LINUX_HOST_OS */
diff --git a/contrib/mount/mntent_compat.h b/contrib/mount/mntent_compat.h
new file mode 100644
index 00000000000..ca82e9aa60f
--- /dev/null
+++ b/contrib/mount/mntent_compat.h
@@ -0,0 +1,37 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _MNTENT_H
+#define _MNTENT_H
+
+#if !defined(GF_LINUX_HOST_OS)
+#include <stdio.h>
+
+struct mntent {
+        char *mnt_fsname;
+        char *mnt_dir;
+        char *mnt_type;
+        char *mnt_opts;
+        int mnt_freq;
+        int mnt_passno;
+};
+
+struct mntent *getmntent (FILE *fp);
+struct mntent *getmntent_r (FILE *fp, struct mntent *result,
+                            char *buffer, int bufsize);
+FILE *setmntent (const char *filename, const char *type);
+int endmntent(FILE *fp);
+char * hasmntopt (const struct mntent *mnt, const char *option);
+
+/* Dummy - /etc/mtab has no meaning on OSX platform */
+#define _PATH_MOUNTED "/etc/mtab"
+
+#endif /* GF_DARWIN_HOST_OS || __NetBSD__ */
+#endif /* _MNTENT_H */
diff --git a/contrib/rbtree/rb.c b/contrib/rbtree/rb.c
index d1339b97d0a..6184c507e72 100644
--- a/contrib/rbtree/rb.c
+++ b/contrib/rbtree/rb.c
@@ -1,26 +1,30 @@
 /* Produced by texiweb from libavl.w. */
 
 /* libavl - library for manipulation of binary trees.
-   Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software
+   Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+   02110-1301 USA.
+
+   This code is also covered by the following earlier license notice:
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-   See the GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-   02111-1307, USA.
-
-   The author may be contacted at <blp@gnu.org> on the Internet, or
-   write to Ben Pfaff, Stanford University, Computer Science Dept., 353
-   Serra Mall, Stanford CA 94305, USA.
 */
 
 #include <assert.h>
diff --git a/contrib/rbtree/rb.h b/contrib/rbtree/rb.h
index c8858b55682..97b44cfd412 100644
--- a/contrib/rbtree/rb.h
+++ b/contrib/rbtree/rb.h
@@ -1,26 +1,30 @@
 /* Produced by texiweb from libavl.w. */
 
 /* libavl - library for manipulation of binary trees.
-   Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software
+   Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+   02110-1301 USA.
+
+   This code is also covered by the following earlier license notice:
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-   See the GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-   02111-1307, USA.
-
-   The author may be contacted at <blp@gnu.org> on the Internet, or
-   write to Ben Pfaff, Stanford University, Computer Science Dept., 353
-   Serra Mall, Stanford CA 94305, USA.
 */
 
 #ifndef RB_H
@@ -51,7 +55,7 @@ void rb_free (struct libavl_allocator *, void *);
 
 /* Maximum RB height. */
 #ifndef RB_MAX_HEIGHT
-#define RB_MAX_HEIGHT 48
+#define RB_MAX_HEIGHT 128
 #endif
 
 /* Tree data structure. */
diff --git a/contrib/timer-wheel/find_last_bit.c b/contrib/timer-wheel/find_last_bit.c
new file mode 100644
index 00000000000..192fee802a8
--- /dev/null
+++ b/contrib/timer-wheel/find_last_bit.c
@@ -0,0 +1,61 @@
+/* bit search implementation
+ *
+ * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * Copyright (C) 2008 IBM Corporation
+ * 'find_last_bit' is written by Rusty Russell <rusty@rustcorp.com.au>
+ * (Inspired by David Howell's find_next_bit implementation)
+ *
+ * Rewritten by Yury Norov <yury.norov@gmail.com> to decrease
+ * size and improve performance, 2015.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+/**
+ * @find_last_bit
+ * optimized implementation of find last bit in
+ */
+
+#ifndef BITS_PER_LONG
+#ifdef __LP64__
+#define BITS_PER_LONG 64
+#else
+#define BITS_PER_LONG 32
+#endif
+#endif
+
+unsigned long gw_tw_fls (unsigned long word)
+{
+        int num = BITS_PER_LONG;
+
+#if BITS_PER_LONG == 64
+        if (!(word & (~0ul << 32))) {
+                num -= 32;
+                word <<= 32;
+        }
+#endif
+        if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
+                num -= 16;
+                word <<= 16;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
+                num -= 8;
+                word <<= 8;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
+                num -= 4;
+                word <<= 4;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
+                num -= 2;
+                word <<= 2;
+        }
+        if (!(word & (~0ul << (BITS_PER_LONG-1))))
+                num -= 1;
+        return num;
+}
diff --git a/contrib/timer-wheel/timer-wheel.c b/contrib/timer-wheel/timer-wheel.c
new file mode 100644
index 00000000000..58e0607bf0c
--- /dev/null
+++ b/contrib/timer-wheel/timer-wheel.c
@@ -0,0 +1,374 @@
+/*
+ *  linux/kernel/timer.c
+ *
+ *  Kernel internal timers
+ *
+ *  Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ */
+/*
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/select.h>
+
+#include "timer-wheel.h"
+
+static inline void
+__gf_tw_add_timer (struct tvec_base *base, struct gf_tw_timer_list *timer)
+{
+        int i;
+        unsigned long idx;
+        unsigned long expires;
+        struct list_head *vec;
+
+        expires = timer->expires;
+
+        idx = expires - base->timer_sec;
+
+        if (idx < TVR_SIZE) {
+                i = expires & TVR_MASK;
+                vec = base->tv1.vec + i;
+        } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
+                i = (expires >> TVR_BITS) & TVN_MASK;
+                vec = base->tv2.vec + i;
+        } else if (idx < 1 << (TVR_BITS + 2*TVN_BITS)) {
+                i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
+                vec = base->tv3.vec + i;
+        } else if (idx < 1 << (TVR_BITS + 3*TVN_BITS)) {
+                i = (expires >> (TVR_BITS + 2*TVN_BITS)) & TVN_MASK;
+                vec = base->tv4.vec + i;
+        } else if (idx < 0) {
+                vec = base->tv1.vec + (base->timer_sec & TVR_MASK);
+        } else {
+                i = (expires >> (TVR_BITS + 3*TVN_BITS)) & TVN_MASK;
+                vec = base->tv5.vec + i;
+        }
+
+        list_add_tail (&timer->entry, vec);
+}
+
+unsigned long gf_tw_find_last_bit(const unsigned long *, unsigned long);
+
+#if defined(__GNUC__) || defined(__clang__)
+static inline unsigned long gf_tw_fls (unsigned long word)
+{
+        return BITS_PER_LONG - __builtin_clzl(word);
+}
+#else
+extern unsigned long gf_tw_fls (unsigned long);
+#endif
+
+static inline unsigned long
+apply_slack(struct tvec_base *base, struct gf_tw_timer_list *timer)
+{
+        long delta;
+        unsigned long mask, expires, expires_limit;
+
+        expires = timer->expires;
+
+        delta = expires - base->timer_sec;
+        if (delta < 256)
+                return expires;
+
+        expires_limit = expires + delta / 256;
+        mask = expires ^ expires_limit;
+        if (mask == 0)
+                return expires;
+
+        int bit = gf_tw_fls (mask);
+        mask = (1UL << bit) - 1;
+
+        expires_limit = expires_limit & ~(mask);
+        return expires_limit;
+}
+
+static inline void
+__gf_tw_detach_timer (struct gf_tw_timer_list *timer)
+{
+        struct list_head *entry = &timer->entry;
+
+        list_del (entry);
+        entry->next = NULL;
+}
+
+static inline int
+cascade (struct tvec_base *base, struct tvec *tv, int index)
+{
+        struct gf_tw_timer_list *timer, *tmp;
+        struct list_head tv_list;
+
+        list_replace_init (tv->vec + index, &tv_list);
+
+        list_for_each_entry_safe (timer, tmp, &tv_list, entry) {
+                __gf_tw_add_timer (base, timer);
+        }
+
+        return index;
+}
+
+#define INDEX(N)  ((base->timer_sec >> (TVR_BITS + N * TVN_BITS)) & TVN_MASK)
+
+/**
+ * run expired timers
+ */
+static inline void
+run_timers (struct tvec_base *base)
+{
+        unsigned long index, call_time;
+        struct gf_tw_timer_list *timer;
+
+        struct list_head work_list;
+        struct list_head *head = &work_list;
+
+        pthread_spin_lock (&base->lock);
+        {
+                index  = base->timer_sec & TVR_MASK;
+
+                if (!index &&
+                    (!cascade (base, &base->tv2, INDEX(0))) &&
+                    (!cascade (base, &base->tv3, INDEX(1))) &&
+                    (!cascade (base, &base->tv4, INDEX(2))))
+                        cascade (base, &base->tv5, INDEX(3));
+
+                call_time = base->timer_sec++;
+                list_replace_init (base->tv1.vec + index, head);
+                while (!list_empty(head)) {
+                        void (*fn)(struct gf_tw_timer_list *, void *, unsigned long);
+                        void *data;
+
+                        timer = list_first_entry (head, struct gf_tw_timer_list, entry);
+                        fn = timer->function;
+                        data = timer->data;
+
+                        __gf_tw_detach_timer (timer);
+                        pthread_spin_unlock(&base->lock);
+                        {
+                            /* It is required to run the actual function outside
+                               of the locked zone, so we don't bother about
+                               locked operations inside that function */
+                            fn(timer, data, call_time);
+                        }
+                        pthread_spin_lock(&base->lock);
+                }
+        }
+        pthread_spin_unlock (&base->lock);
+
+}
+
+void *runner (void *arg)
+{
+        struct timeval tv = {0,};
+        struct tvec_base *base = arg;
+
+        while (1) {
+                run_timers (base);
+
+                tv.tv_sec  = 1;
+                tv.tv_usec = 0;
+                (void) select (0, NULL, NULL, NULL, &tv);
+        }
+
+        return NULL;
+
+}
+
+static inline int timer_pending (struct gf_tw_timer_list *timer)
+{
+        struct list_head *entry = &timer->entry;
+
+        return (entry->next != NULL);
+}
+
+static inline int __detach_if_pending (struct gf_tw_timer_list *timer)
+{
+        if (!timer_pending (timer))
+                return 0;
+
+        __gf_tw_detach_timer (timer);
+        return 1;
+}
+
+static inline int __mod_timer (struct tvec_base *base,
+                               struct gf_tw_timer_list *timer, int pending_only)
+{
+        int ret = 0;
+
+        ret = __detach_if_pending (timer);
+        if (!ret && pending_only)
+                goto done;
+
+        ret = 1;
+        __gf_tw_add_timer (base, timer);
+
+ done:
+        return ret;
+}
+
+/* interface */
+
+/**
+ * Add a timer in the timer wheel
+ */
+void gf_tw_add_timer (struct tvec_base *base, struct gf_tw_timer_list *timer)
+{
+        pthread_spin_lock (&base->lock);
+        {
+                timer->expires += base->timer_sec;
+                timer->expires = apply_slack (base, timer);
+                __gf_tw_add_timer (base, timer);
+        }
+        pthread_spin_unlock (&base->lock);
+}
+
+/**
+ * Remove a timer from the timer wheel
+ */
+int gf_tw_del_timer (struct tvec_base *base, struct gf_tw_timer_list *timer)
+{
+        int ret = 0;
+
+        pthread_spin_lock (&base->lock);
+        {
+                if (timer_pending (timer)) {
+                        ret = 1;
+                        __gf_tw_detach_timer (timer);
+                }
+        }
+        pthread_spin_unlock (&base->lock);
+
+        return ret;
+}
+
+int gf_tw_mod_timer_pending (struct tvec_base *base,
+                             struct gf_tw_timer_list *timer,
+                             unsigned long expires)
+{
+        int ret = 1;
+
+        pthread_spin_lock (&base->lock);
+        {
+                timer->expires = expires + base->timer_sec;
+                timer->expires = apply_slack (base, timer);
+
+                ret = __mod_timer (base, timer, 1);
+        }
+        pthread_spin_unlock (&base->lock);
+
+        return ret;
+}
+
+int gf_tw_mod_timer (struct tvec_base *base,
+                     struct gf_tw_timer_list *timer, unsigned long expires)
+{
+        int ret = 1;
+
+        pthread_spin_lock (&base->lock);
+        {
+                /* fast path optimization */
+                if (timer_pending (timer) && timer->expires == expires)
+                        goto unblock;
+
+                timer->expires = expires + base->timer_sec;
+                timer->expires = apply_slack (base, timer);
+
+                ret = __mod_timer (base, timer, 0);
+        }
+ unblock:
+        pthread_spin_unlock (&base->lock);
+
+        return ret;
+}
+
+int gf_tw_cleanup_timers (struct tvec_base *base)
+{
+        int ret = 0;
+        void *res = NULL;
+
+        /* terminate runner */
+        ret = pthread_cancel (base->runner);
+        if (ret != 0)
+                goto error_return;
+        ret = pthread_join (base->runner, &res);
+        if (ret != 0)
+                goto error_return;
+        if (res != PTHREAD_CANCELED)
+                goto error_return;
+
+        /* destroy lock */
+        ret = pthread_spin_destroy (&base->lock);
+        if (ret != 0)
+                goto error_return;
+
+        /* deallocated timer base */
+        free (base);
+        return 0;
+
+ error_return:
+        return -1;
+}
+
+/**
+ * Initialize various timer wheel lists and spawn a thread that
+ * invokes run_timers()
+ */
+struct tvec_base *gf_tw_init_timers ()
+{
+        int               j    = 0;
+        int               ret  = 0;
+        struct timeval    tv   = {0,};
+        struct tvec_base *base = NULL;
+
+        base = malloc (sizeof (*base));
+        if (!base)
+                goto error_return;
+
+        ret = pthread_spin_init (&base->lock, 0);
+        if (ret != 0)
+                goto error_dealloc;
+
+        for (j = 0; j < TVN_SIZE; j++) {
+                INIT_LIST_HEAD (base->tv5.vec + j);
+                INIT_LIST_HEAD (base->tv4.vec + j);
+                INIT_LIST_HEAD (base->tv3.vec + j);
+                INIT_LIST_HEAD (base->tv2.vec + j);
+        }
+
+        for (j = 0; j < TVR_SIZE; j++) {
+                INIT_LIST_HEAD (base->tv1.vec + j);
+        }
+
+        ret = gettimeofday (&tv, 0);
+        if (ret < 0)
+                goto destroy_lock;
+        base->timer_sec = tv.tv_sec;
+
+        ret = pthread_create (&base->runner, NULL, runner, base);
+        if (ret != 0)
+                goto destroy_lock;
+        return base;
+
+ destroy_lock:
+        (void) pthread_spin_destroy (&base->lock);
+ error_dealloc:
+        free (base);
+ error_return:
+        return NULL;
+}
diff --git a/contrib/timer-wheel/timer-wheel.h b/contrib/timer-wheel/timer-wheel.h
new file mode 100644
index 00000000000..5637735ec22
--- /dev/null
+++ b/contrib/timer-wheel/timer-wheel.h
@@ -0,0 +1,77 @@
+/*
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef __TIMER_WHEEL_H
+#define __TIMER_WHEEL_H
+
+#include "glusterfs/locking.h"
+
+#include "glusterfs/list.h"
+
+#define TVR_BITS  8
+#define TVN_BITS  6
+#define TVR_SIZE  (1 << TVR_BITS)
+#define TVN_SIZE  (1 << TVN_BITS)
+#define TVR_MASK  (TVR_SIZE - 1)
+#define TVN_MASK  (TVN_SIZE - 1)
+
+#define BITS_PER_LONG  64
+
+struct tvec {
+        struct list_head vec[TVN_SIZE];
+};
+
+struct tvec_root {
+        struct list_head vec[TVR_SIZE];
+};
+
+struct tvec_base {
+        pthread_spinlock_t lock;      /* base lock */
+
+        pthread_t runner;             /* run_timer() */
+
+        unsigned long timer_sec;      /* time counter */
+
+        struct tvec_root tv1;
+        struct tvec tv2;
+        struct tvec tv3;
+        struct tvec tv4;
+        struct tvec tv5;
+};
+
+struct gf_tw_timer_list {
+        void *data;
+        unsigned long expires;
+
+        /** callback routine */
+        void (*function)(struct gf_tw_timer_list *, void *, unsigned long);
+
+        struct list_head entry;
+};
+
+/** The API! */
+struct tvec_base *gf_tw_init_timers ();
+int gf_tw_cleanup_timers (struct tvec_base *);
+void gf_tw_add_timer (struct tvec_base *, struct gf_tw_timer_list *);
+int gf_tw_del_timer (struct tvec_base *, struct gf_tw_timer_list *);
+
+int gf_tw_mod_timer_pending (struct tvec_base *,
+                             struct gf_tw_timer_list *, unsigned long);
+
+int gf_tw_mod_timer (struct tvec_base *,
+                     struct gf_tw_timer_list *, unsigned long);
+
+#endif
diff --git a/contrib/umountd/Makefile.am b/contrib/umountd/Makefile.am
new file mode 100644
index 00000000000..b39e000f5aa
--- /dev/null
+++ b/contrib/umountd/Makefile.am
@@ -0,0 +1,11 @@
+sbin_PROGRAMS = umountd
+umountd_SOURCES = umountd.c
+umountd_CFLAGS = $(GF_CFLAGS) -DDATADIR=\"$(localstatedir)\"
+umountd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la ${GF_LDADD}
+umountd_LDFLAGS = $(GF_LDFLAGS)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/contrib/umountd/umountd.c b/contrib/umountd/umountd.c
new file mode 100644
index 00000000000..3f933ecb554
--- /dev/null
+++ b/contrib/umountd/umountd.c
@@ -0,0 +1,255 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <dirent.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/mem-types.h"
+
+static void
+usage (void)
+{
+        fprintf (stderr, "Usage: umountd [-d dev] [-t timeout] [-r] path\n");
+        exit (EXIT_FAILURE);
+}
+
+
+static int
+sanity_check (char *path, dev_t *devp)
+{
+        struct stat st;
+        struct stat parent_st;
+        int ret;
+        char pathtmp[PATH_MAX];
+        char *parent;
+
+        if (path == NULL)
+                usage ();
+
+        if ((ret = stat (path, &st)) != 0) {
+                switch (errno) {
+                case ENOTCONN:
+                        /* volume is stopped */
+                        break;
+                default:
+                        gf_log ("umountd", GF_LOG_ERROR,
+                                "Cannot access %s: %s\n",
+                                path, strerror (errno));
+                        goto out;
+                }
+        }
+
+        /* If dev was not specified, get it from path */
+        if (*devp == -1 && ret == 0)
+                *devp = st.st_dev;
+
+        snprintf (pathtmp, PATH_MAX, "%s", path);
+        parent = dirname (pathtmp);
+
+        if (stat (parent, &parent_st) != 0) {
+                gf_log ("umountd", GF_LOG_ERROR,
+                        "Cannot access %s: %s\n",
+                        parent, strerror (errno));
+                goto out;
+        }
+
+        if (st.st_dev == parent_st.st_dev) {
+                gf_log ("umountd", GF_LOG_ERROR,
+                        "No filesystem mounted on %s\n", path);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static void
+log_rotate (int signum)
+{
+        gf_log_logrotate (1);
+
+        if (signal (SIGHUP, *log_rotate) == SIG_ERR) {
+                gf_log ("umountd", GF_LOG_ERROR, "signal () failed");
+                exit (EXIT_FAILURE);
+        }
+
+        return;
+}
+
+static int
+logging_init (void)
+{
+        glusterfs_ctx_t *ctx;
+        char log_file[PATH_MAX];
+        int ret = -1;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx) {
+                fprintf (stderr, "glusterfs_ctx_new failed\n");
+                goto out;
+        }
+
+        ret = glusterfs_globals_init (ctx);
+        if (ret) {
+                fprintf (stderr, "glusterfs_globals_init failed\n");
+                goto out;
+        }
+
+        THIS->ctx = ctx;
+        xlator_mem_acct_init (THIS, gf_common_mt_end);
+
+        snprintf (log_file, PATH_MAX,
+                  "%s/umountd.log", DEFAULT_LOG_FILE_DIRECTORY);
+
+        ret = gf_log_init (ctx, log_file, "umountd");
+        if (ret) {
+                fprintf (stderr, "gf_log_init failed\n");
+                goto out;
+        }
+
+        if (signal (SIGHUP, *log_rotate) == SIG_ERR) {
+                gf_log ("umountd", GF_LOG_ERROR, "signal () failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+umountd_async (char *path, dev_t dev, int frmdir, int timeout)
+{
+        int                   ret        = -1;
+        struct stat           stbuf      = {0, };
+        int                       unmount_ret = 0;
+
+        do {
+                unmount_ret = unmount (path, 0);
+                if (unmount_ret == 0)
+                        gf_log ("umountd", GF_LOG_INFO, "Unmounted %s", path);
+
+                if (unmount_ret != 0 && errno != EBUSY) {
+                               gf_log ("umountd", GF_LOG_WARNING,
+                                      "umount %s failed: %s",
+                                path, strerror (errno));
+                }
+
+                ret = sys_lstat (path, &stbuf);
+                if (ret != 0) {
+                        gf_log ("umountd", GF_LOG_WARNING,
+                                      "Cannot stat device from %s",
+                                path, strerror (errno));
+                        break;
+                }
+
+                if (stbuf.st_dev != dev) {
+                        if (unmount_ret != 0)
+                                gf_log ("umountd", GF_LOG_INFO,
+                                        "device mismatch "
+                                        "(expect %lld, found %lld), "
+                                        "someone else unmounted %s",
+                                        dev, stbuf.st_dev, path);
+                        ret = 0;
+                        break;
+                }
+
+                sleep (timeout);
+        } while (1/*CONSTCOND*/);
+
+        if (ret) {
+                gf_log ("umountd", GF_LOG_ERROR,
+                        "Asynchronous unmount of %s failed: %s",
+                        path, strerror (errno));
+        } else {
+                if (frmdir) {
+                        ret = rmdir (path);
+                        if (ret)
+                                gf_log ("umountd", GF_LOG_WARNING,
+                                         "rmdir %s failed: %s",
+                                         path, strerror (errno));
+                        else
+                                gf_log ("umountd", GF_LOG_INFO,
+                                         "Removed %s", path);
+                }
+        }
+
+        return ret;
+}
+
+int
+main (int argc, char **argv)
+{
+        char *path = NULL;
+        dev_t dev = -1;
+        int frmdir = 0;
+        int timeout = 30;
+        int f;
+
+        while ((f = getopt (argc, argv, "d:rt:")) != -1) {
+                switch (f) {
+                case 'p':
+                        path = optarg;
+                        break;
+                case 'd':
+                        dev = strtoll (optarg, NULL, 10);
+                        break;
+                case 't':
+                        timeout = atoi (optarg);
+                        break;
+                case 'r':
+                        frmdir = 1;
+                        break;
+                default:
+                        usage ();
+                        break;
+                }
+        }
+
+        argc -= optind;
+        argv += optind;
+
+        if (argc != 1)
+                usage ();
+
+        path = argv[0];
+
+        if (logging_init () != 0)
+                exit (EXIT_FAILURE);
+
+        if (sanity_check (path, &dev) != 0)
+                exit (EXIT_FAILURE);
+
+        if (daemon (0, 0) != 0)
+                exit (EXIT_FAILURE);
+
+        if (umountd_async (path, dev, frmdir, timeout) != 0)
+                exit (EXIT_FAILURE);
+
+        return EXIT_SUCCESS;
+}
diff --git a/contrib/userspace-rcu/rculist-extra.h b/contrib/userspace-rcu/rculist-extra.h
new file mode 100644
index 00000000000..274cf9f9d7a
--- /dev/null
+++ b/contrib/userspace-rcu/rculist-extra.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2002 Free Software Foundation, Inc.
+ * (originally part of the GNU C Library)
+ * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+ *
+ * Copyright (C) 2009 Pierre-Marc Fournier
+ * Conversion to RCU list.
+ * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef URCU_RCULIST_EXTRA_H
+#define URCU_RCULIST_EXTRA_H
+/* Copying this definition from liburcu-0.8 as liburcu-0.7 does not have this
+ * particular list api
+ */
+/* Add new element at the tail of the list. */
+
+static inline
+void cds_list_add_tail_rcu(struct cds_list_head *newp,
+                struct cds_list_head *head)
+{
+        newp->next = head;
+        newp->prev = head->prev;
+        rcu_assign_pointer(head->prev->next, newp);
+        head->prev = newp;
+}
+
+#endif
diff --git a/contrib/userspace-rcu/static-wfcqueue.h b/contrib/userspace-rcu/static-wfcqueue.h
new file mode 100644
index 00000000000..37d14ad674b
--- /dev/null
+++ b/contrib/userspace-rcu/static-wfcqueue.h
@@ -0,0 +1,685 @@
+#ifndef _URCU_WFCQUEUE_STATIC_H
+#define _URCU_WFCQUEUE_STATIC_H
+
+/*
+ * urcu/static/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfcqueue.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Copied from userspace-rcu 0.10 because version 0.7 doesn't contain it. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ *
+ * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ *     [1] [2] [3] [4] [5] [6]
+ * [1]  -   -   -   -   -   -
+ * [2]  -   -   -   -   -   -
+ * [3]  -   -   X   X   X   X
+ * [4]  -   -   X   -   X   X
+ * [5]  -   -   X   X   -   -
+ * [6]  -   -   X   X   -   -
+ *
+ * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
+ *
+ * For convenience, cds_wfcq_dequeue_blocking() and
+ * cds_wfcq_splice_blocking() hold the dequeue lock.
+ *
+ * Besides locking, mutual exclusion of dequeue, splice and iteration
+ * can be ensured by performing all of those operations from a single
+ * thread, without requiring any lock.
+ */
+
+#define WFCQ_ADAPT_ATTEMPTS		10	/* Retry if being set */
+#define WFCQ_WAIT			10	/* Wait 10 ms if being set */
+
+/*
+ * cds_wfcq_node_init: initialize wait-free queue node.
+ */
+static inline void _cds_wfcq_node_init(struct cds_wfcq_node *node)
+{
+	node->next = NULL;
+}
+
+/*
+ * cds_wfcq_init: initialize wait-free queue (with lock). Pair with
+ * cds_wfcq_destroy().
+ */
+static inline void _cds_wfcq_init(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	/* Set queue head and tail */
+	_cds_wfcq_node_init(&head->node);
+	tail->p = &head->node;
+	ret = pthread_mutex_init(&head->lock, NULL);
+	assert(!ret);
+}
+
+/*
+ * cds_wfcq_destroy: destroy wait-free queue (with lock). Pair with
+ * cds_wfcq_init().
+ */
+static inline void _cds_wfcq_destroy(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret = pthread_mutex_destroy(&head->lock);
+	assert(!ret);
+}
+
+/*
+ * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
+ * pair with any destroy function.
+ */
+static inline void ___cds_wfcq_init(struct __cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	/* Set queue head and tail */
+	_cds_wfcq_node_init(&head->node);
+	tail->p = &head->node;
+}
+
+/*
+ * cds_wfcq_empty: return whether wait-free queue is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ *
+ * We perform the test on head->node.next to check if the queue is
+ * possibly empty, but we confirm this by checking if the tail pointer
+ * points to the head node because the tail pointer is the linearisation
+ * point of the enqueuers. Just checking the head next pointer could
+ * make a queue appear empty if an enqueuer is preempted for a long time
+ * between xchg() and setting the previous node's next pointer.
+ */
+static inline bool _cds_wfcq_empty(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	/*
+	 * Queue is empty if no node is pointed by head->node.next nor
+	 * tail->p. Even though the tail->p check is sufficient to find
+	 * out of the queue is empty, we first check head->node.next as a
+	 * common case to ensure that dequeuers do not frequently access
+	 * enqueuer's tail->p cache line.
+	 */
+	return CMM_LOAD_SHARED(head->node.next) == NULL
+		&& CMM_LOAD_SHARED(tail->p) == &head->node;
+}
+
+static inline void _cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	ret = pthread_mutex_lock(&head->lock);
+	assert(!ret);
+}
+
+static inline void _cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	int ret;
+
+	ret = pthread_mutex_unlock(&head->lock);
+	assert(!ret);
+}
+
+static inline bool ___cds_wfcq_append(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *new_head,
+		struct cds_wfcq_node *new_tail)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *old_tail;
+
+	/*
+	 * Implicit memory barrier before uatomic_xchg() orders earlier
+	 * stores to data structure containing node and setting
+	 * node->next to NULL before publication.
+	 */
+	old_tail = uatomic_xchg(&tail->p, new_tail);
+
+	/*
+	 * Implicit memory barrier after uatomic_xchg() orders store to
+	 * q->tail before store to old_tail->next.
+	 *
+	 * At this point, dequeuers see a NULL tail->p->next, which
+	 * indicates that the queue is being appended to. The following
+	 * store will append "node" to the queue from a dequeuer
+	 * perspective.
+	 */
+	CMM_STORE_SHARED(old_tail->next, new_head);
+	/*
+	 * Return false if queue was empty prior to adding the node,
+	 * else return true.
+	 */
+	return old_tail != &head->node;
+}
+
+/*
+ * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
+ *
+ * Issues a full memory barrier before enqueue. No mutual exclusion is
+ * required.
+ *
+ * Returns false if the queue was empty prior to adding the node.
+ * Returns true otherwise.
+ */
+static inline bool _cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *new_tail)
+{
+	return ___cds_wfcq_append(head, tail, new_tail, new_tail);
+}
+
+/*
+ * CDS_WFCQ_WAIT_SLEEP:
+ *
+ * By default, this sleeps for the given @msec milliseconds.
+ * This is a macro which LGPL users may #define themselves before
+ * including wfcqueue.h to override the default behavior (e.g.
+ * to log a warning or perform other background work).
+ */
+#ifndef CDS_WFCQ_WAIT_SLEEP
+#define CDS_WFCQ_WAIT_SLEEP(msec) ___cds_wfcq_wait_sleep(msec)
+#endif
+
+static inline void ___cds_wfcq_wait_sleep(int msec)
+{
+	(void) poll(NULL, 0, msec);
+}
+
+/*
+ * ___cds_wfcq_busy_wait: adaptative busy-wait.
+ *
+ * Returns 1 if nonblocking and needs to block, 0 otherwise.
+ */
+static inline bool
+___cds_wfcq_busy_wait(int *attempt, int blocking)
+{
+	if (!blocking)
+		return 1;
+	if (++(*attempt) >= WFCQ_ADAPT_ATTEMPTS) {
+		CDS_WFCQ_WAIT_SLEEP(WFCQ_WAIT);		/* Wait for 10ms */
+		*attempt = 0;
+	} else {
+		caa_cpu_relax();
+	}
+	return 0;
+}
+
+/*
+ * Waiting for enqueuer to complete enqueue and return the next node.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_node_sync_next(struct cds_wfcq_node *node, int blocking)
+{
+	struct cds_wfcq_node *next;
+	int attempt = 0;
+
+	/*
+	 * Adaptative busy-looping waiting for enqueuer to complete enqueue.
+	 */
+	while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		if (___cds_wfcq_busy_wait(&attempt, blocking))
+			return CDS_WFCQ_WOULDBLOCK;
+	}
+
+	return next;
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_first(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		int blocking)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *node;
+
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail))
+		return NULL;
+	node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+	/* Load head->node.next before loading node's content */
+	cmm_smp_read_barrier_depends();
+	return node;
+}
+
+/*
+ * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if queue is empty, first node otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_first(head, tail, 1);
+}
+
+
+/*
+ * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_first(head, tail, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_next(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node,
+		int blocking)
+{
+	struct cds_wfcq_node *next;
+
+	/*
+	 * Even though the following tail->p check is sufficient to find
+	 * out if we reached the end of the queue, we first check
+	 * node->next as a common case to ensure that iteration on nodes
+	 * do not frequently access enqueuer's tail->p cache line.
+	 */
+	if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		/* Load node->next before tail->p */
+		cmm_smp_rmb();
+		if (CMM_LOAD_SHARED(tail->p) == node)
+			return NULL;
+		next = ___cds_wfcq_node_sync_next(node, blocking);
+	}
+	/* Load node->next before loading next's content */
+	cmm_smp_read_barrier_depends();
+	return next;
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of queue, non-NULL next queue node
+ * otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node)
+{
+	return ___cds_wfcq_next(head, tail, node, 1);
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail,
+		struct cds_wfcq_node *node)
+{
+	return ___cds_wfcq_next(head, tail, node, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state(cds_wfcq_head_ptr_t u_head,
+		struct cds_wfcq_tail *tail,
+		int *state,
+		int blocking)
+{
+	struct __cds_wfcq_head *head = u_head._h;
+	struct cds_wfcq_node *node, *next;
+
+	if (state)
+		*state = 0;
+
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail)) {
+		return NULL;
+	}
+
+	node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+	if (!blocking && node == CDS_WFCQ_WOULDBLOCK) {
+		return CDS_WFCQ_WOULDBLOCK;
+	}
+
+	if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		/*
+		 * @node is probably the only node in the queue.
+		 * Try to move the tail to &q->head.
+		 * q->head.next is set to NULL here, and stays
+		 * NULL if the cmpxchg succeeds. Should the
+		 * cmpxchg fail due to a concurrent enqueue, the
+		 * q->head.next will be set to the next node.
+		 * The implicit memory barrier before
+		 * uatomic_cmpxchg() orders load node->next
+		 * before loading q->tail.
+		 * The implicit memory barrier before uatomic_cmpxchg
+		 * orders load q->head.next before loading node's
+		 * content.
+		 */
+		_cds_wfcq_node_init(&head->node);
+		if (uatomic_cmpxchg(&tail->p, node, &head->node) == node) {
+			if (state)
+				*state |= CDS_WFCQ_STATE_LAST;
+			return node;
+		}
+		next = ___cds_wfcq_node_sync_next(node, blocking);
+		/*
+		 * In nonblocking mode, if we would need to block to
+		 * get node's next, set the head next node pointer
+		 * (currently NULL) back to its original value.
+		 */
+		if (!blocking && next == CDS_WFCQ_WOULDBLOCK) {
+			head->node.next = node;
+			return CDS_WFCQ_WOULDBLOCK;
+		}
+	}
+
+	/*
+	 * Move queue head forward.
+	 */
+	head->node.next = next;
+
+	/* Load q->head.next before loading node's content */
+	cmm_smp_read_barrier_depends();
+	return node;
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_blocking: dequeue node from queue, with state.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	return ___cds_wfcq_dequeue_with_state(head, tail, state, 1);
+}
+
+/*
+ * ___cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_blocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_blocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_nonblocking: dequeue node, with state.
+ *
+ * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
+ * if it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	return ___cds_wfcq_dequeue_with_state(head, tail, state, 0);
+}
+
+/*
+ * ___cds_wfcq_dequeue_nonblocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_nonblocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_nonblocking(cds_wfcq_head_ptr_t head,
+		struct cds_wfcq_tail *tail)
+{
+	return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_splice: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice(
+		cds_wfcq_head_ptr_t u_dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t u_src_q_head,
+		struct cds_wfcq_tail *src_q_tail,
+		int blocking)
+{
+	struct __cds_wfcq_head *dest_q_head = u_dest_q_head._h;
+	struct __cds_wfcq_head *src_q_head = u_src_q_head._h;
+	struct cds_wfcq_node *head, *tail;
+	int attempt = 0;
+
+	/*
+	 * Initial emptiness check to speed up cases where queue is
+	 * empty: only require loads to check if queue is empty.
+	 */
+	if (_cds_wfcq_empty(__cds_wfcq_head_cast(src_q_head), src_q_tail))
+		return CDS_WFCQ_RET_SRC_EMPTY;
+
+	for (;;) {
+		/*
+		 * Open-coded _cds_wfcq_empty() by testing result of
+		 * uatomic_xchg, as well as tail pointer vs head node
+		 * address.
+		 */
+		head = uatomic_xchg(&src_q_head->node.next, NULL);
+		if (head)
+			break;	/* non-empty */
+		if (CMM_LOAD_SHARED(src_q_tail->p) == &src_q_head->node)
+			return CDS_WFCQ_RET_SRC_EMPTY;
+		if (___cds_wfcq_busy_wait(&attempt, blocking))
+			return CDS_WFCQ_RET_WOULDBLOCK;
+	}
+
+	/*
+	 * Memory barrier implied before uatomic_xchg() orders store to
+	 * src_q->head before store to src_q->tail. This is required by
+	 * concurrent enqueue on src_q, which exchanges the tail before
+	 * updating the previous tail's next pointer.
+	 */
+	tail = uatomic_xchg(&src_q_tail->p, &src_q_head->node);
+
+	/*
+	 * Append the spliced content of src_q into dest_q. Does not
+	 * require mutual exclusion on dest_q (wait-free).
+	 */
+	if (___cds_wfcq_append(__cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+			head, tail))
+		return CDS_WFCQ_RET_DEST_NON_EMPTY;
+	else
+		return CDS_WFCQ_RET_DEST_EMPTY;
+}
+
+/*
+ * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_blocking(
+		cds_wfcq_head_ptr_t dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+		src_q_head, src_q_tail, 1);
+}
+
+/*
+ * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Same as __cds_wfcq_splice_blocking, but returns
+ * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_nonblocking(
+		cds_wfcq_head_ptr_t dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		cds_wfcq_head_ptr_t src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+		src_q_head, src_q_tail, 0);
+}
+
+/*
+ * cds_wfcq_dequeue_with_state_blocking: dequeue a node from a wait-free queue.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_splice_blocking and dequeue lock is
+ * ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail, int *state)
+{
+	struct cds_wfcq_node *retval;
+
+	_cds_wfcq_dequeue_lock(head, tail);
+	retval = ___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_cast(head),
+			tail, state);
+	_cds_wfcq_dequeue_unlock(head, tail);
+	return retval;
+}
+
+/*
+ * cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as cds_wfcq_dequeue_blocking, but without saving state.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head,
+		struct cds_wfcq_tail *tail)
+{
+	return _cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
+ * ensured.
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+_cds_wfcq_splice_blocking(
+		struct cds_wfcq_head *dest_q_head,
+		struct cds_wfcq_tail *dest_q_tail,
+		struct cds_wfcq_head *src_q_head,
+		struct cds_wfcq_tail *src_q_tail)
+{
+	enum cds_wfcq_ret ret;
+
+	_cds_wfcq_dequeue_lock(src_q_head, src_q_tail);
+	ret = ___cds_wfcq_splice_blocking(cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+			cds_wfcq_head_cast(src_q_head), src_q_tail);
+	_cds_wfcq_dequeue_unlock(src_q_head, src_q_tail);
+	return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_STATIC_H */
diff --git a/contrib/userspace-rcu/static-wfstack.h b/contrib/userspace-rcu/static-wfstack.h
new file mode 100644
index 00000000000..29b81c3aac3
--- /dev/null
+++ b/contrib/userspace-rcu/static-wfstack.h
@@ -0,0 +1,455 @@
+#ifndef _URCU_STATIC_WFSTACK_H
+#define _URCU_STATIC_WFSTACK_H
+
+/*
+ * urcu/static/wfstack.h
+ *
+ * Userspace RCU library - Stack with with wait-free push, blocking traversal.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfstack.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't support a stack
+ * without mutex. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CDS_WFS_END			((void *) 0x1UL)
+#define CDS_WFS_ADAPT_ATTEMPTS		10	/* Retry if being set */
+#define CDS_WFS_WAIT			10	/* Wait 10 ms if being set */
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ *                       cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ *                      iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ *                      cds_wfs_push  __cds_wfs_pop  __cds_wfs_pop_all
+ * cds_wfs_push               -              -                  -
+ * __cds_wfs_pop              -              X                  X
+ * __cds_wfs_pop_all          -              X                  -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+/*
+ * cds_wfs_node_init: initialize wait-free stack node.
+ */
+static inline
+void _cds_wfs_node_init(struct cds_wfs_node *node)
+{
+	node->next = NULL;
+}
+
+/*
+ * __cds_wfs_init: initialize wait-free stack. Don't pair with
+ * any destroy function.
+ */
+static inline void ___cds_wfs_init(struct __cds_wfs_stack *s)
+{
+	s->head = CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_init: initialize wait-free stack. Pair with
+ * cds_wfs_destroy().
+ */
+static inline
+void _cds_wfs_init(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	s->head = CDS_WFS_END;
+	ret = pthread_mutex_init(&s->lock, NULL);
+	assert(!ret);
+}
+
+/*
+ * cds_wfs_destroy: destroy wait-free stack. Pair with
+ * cds_wfs_init().
+ */
+static inline
+void _cds_wfs_destroy(struct cds_wfs_stack *s)
+{
+	int ret = pthread_mutex_destroy(&s->lock);
+	assert(!ret);
+}
+
+static inline bool ___cds_wfs_end(void *node)
+{
+	return node == CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_empty: return whether wait-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+static inline bool _cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+
+	return ___cds_wfs_end(CMM_LOAD_SHARED(s->head));
+}
+
+/*
+ * cds_wfs_push: push a node into the stack.
+ *
+ * Issues a full memory barrier before push. No mutual exclusion is
+ * required.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+static inline
+int _cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+	struct cds_wfs_head *old_head, *new_head;
+
+	assert(node->next == NULL);
+	new_head = caa_container_of(node, struct cds_wfs_head, node);
+	/*
+	 * uatomic_xchg() implicit memory barrier orders earlier stores
+	 * to node (setting it to NULL) before publication.
+	 */
+	old_head = uatomic_xchg(&s->head, new_head);
+	/*
+	 * At this point, dequeuers see a NULL node->next, they should
+	 * busy-wait until node->next is set to old_head.
+	 */
+	CMM_STORE_SHARED(node->next, &old_head->node);
+	return !___cds_wfs_end(old_head);
+}
+
+/*
+ * Waiting for push to complete enqueue and return the next node.
+ */
+static inline struct cds_wfs_node *
+___cds_wfs_node_sync_next(struct cds_wfs_node *node, int blocking)
+{
+	struct cds_wfs_node *next;
+	int attempt = 0;
+
+	/*
+	 * Adaptative busy-looping waiting for push to complete.
+	 */
+	while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+		if (!blocking)
+			return CDS_WFS_WOULDBLOCK;
+		if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
+			(void) poll(NULL, 0, CDS_WFS_WAIT);	/* Wait for 10ms */
+			attempt = 0;
+		} else {
+			caa_cpu_relax();
+		}
+	}
+
+	return next;
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop(cds_wfs_stack_ptr_t u_stack, int *state, int blocking)
+{
+	struct cds_wfs_head *head, *new_head;
+	struct cds_wfs_node *next;
+	struct __cds_wfs_stack *s = u_stack._s;
+
+	if (state)
+		*state = 0;
+	for (;;) {
+		head = CMM_LOAD_SHARED(s->head);
+		if (___cds_wfs_end(head)) {
+			return NULL;
+		}
+		next = ___cds_wfs_node_sync_next(&head->node, blocking);
+		if (!blocking && next == CDS_WFS_WOULDBLOCK) {
+			return CDS_WFS_WOULDBLOCK;
+		}
+		new_head = caa_container_of(next, struct cds_wfs_head, node);
+		if (uatomic_cmpxchg(&s->head, head, new_head) == head) {
+			if (state && ___cds_wfs_end(new_head))
+				*state |= CDS_WFS_STATE_LAST;
+			return &head->node;
+		}
+		if (!blocking) {
+			return CDS_WFS_WOULDBLOCK;
+		}
+		/* busy-loop if head changed under us */
+	}
+}
+
+/*
+ * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_wfs_pop_blocking needs to be synchronized using one of the
+ * following techniques:
+ *
+ * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
+ *    section. The caller must wait for a grace period to pass before
+ *    freeing the returned node or modifying the cds_wfs_node structure.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ *     __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ *    and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+	return ___cds_wfs_pop(u_stack, state, 1);
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
+{
+	return ___cds_wfs_pop_with_state_blocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_with_state_blocking, but returns
+ * CDS_WFS_WOULDBLOCK if it needs to block.
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+	return ___cds_wfs_pop(u_stack, state, 0);
+}
+
+/*
+ * __cds_wfs_pop_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
+{
+	return ___cds_wfs_pop_with_state_nonblocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_wfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_wfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_wfs_pop_blocking:
+ *
+ * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
+ *    section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
+ *    must wait for a grace period to pass before freeing the returned
+ *    node or modifying the cds_wfs_node structure. However, no RCU
+ *    read-side critical section is needed around __cds_wfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ *     __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ *    and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_wfs_head *
+___cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
+{
+	struct __cds_wfs_stack *s = u_stack._s;
+	struct cds_wfs_head *head;
+
+	/*
+	 * Implicit memory barrier after uatomic_xchg() matches implicit
+	 * memory barrier before uatomic_xchg() in cds_wfs_push. It
+	 * ensures that all nodes of the returned list are consistent.
+	 * There is no need to issue memory barriers when iterating on
+	 * the returned list, because the full memory barrier issued
+	 * prior to each uatomic_cmpxchg, which each write to head, are
+	 * taking care to order writes to each node prior to the full
+	 * memory barrier after this uatomic_xchg().
+	 */
+	head = uatomic_xchg(&s->head, CDS_WFS_END);
+	if (___cds_wfs_end(head))
+		return NULL;
+	return head;
+}
+
+/*
+ * cds_wfs_pop_lock: lock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_lock(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	ret = pthread_mutex_lock(&s->lock);
+	assert(!ret);
+}
+
+/*
+ * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_unlock(struct cds_wfs_stack *s)
+{
+	int ret;
+
+	ret = pthread_mutex_unlock(&s->lock);
+	assert(!ret);
+}
+
+/*
+ * Call __cds_wfs_pop_with_state_blocking with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
+{
+	struct cds_wfs_node *retnode;
+
+	_cds_wfs_pop_lock(s);
+	retnode = ___cds_wfs_pop_with_state_blocking(s, state);
+	_cds_wfs_pop_unlock(s);
+	return retnode;
+}
+
+/*
+ * Call _cds_wfs_pop_with_state_blocking without saving any state.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+	return _cds_wfs_pop_with_state_blocking(s, NULL);
+}
+
+/*
+ * Call __cds_wfs_pop_all with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_head *
+_cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
+{
+	struct cds_wfs_head *rethead;
+
+	_cds_wfs_pop_lock(s);
+	rethead = ___cds_wfs_pop_all(s);
+	_cds_wfs_pop_unlock(s);
+	return rethead;
+}
+
+/*
+ * cds_wfs_first: get first node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if popped stack is empty, top stack node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_first(struct cds_wfs_head *head)
+{
+	if (___cds_wfs_end(head))
+		return NULL;
+	return &head->node;
+}
+
+static inline struct cds_wfs_node *
+___cds_wfs_next(struct cds_wfs_node *node, int blocking)
+{
+	struct cds_wfs_node *next;
+
+	next = ___cds_wfs_node_sync_next(node, blocking);
+	/*
+	 * CDS_WFS_WOULDBLOCK != CSD_WFS_END, so we can check for end
+	 * even if ___cds_wfs_node_sync_next returns CDS_WFS_WOULDBLOCK,
+	 * and still return CDS_WFS_WOULDBLOCK.
+	 */
+	if (___cds_wfs_end(next))
+		return NULL;
+	return next;
+}
+
+/*
+ * cds_wfs_next_blocking: get next node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of popped stack, non-NULL next stack
+ * node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_blocking(struct cds_wfs_node *node)
+{
+	return ___cds_wfs_next(node, 1);
+}
+
+
+/*
+ * cds_wfs_next_nonblocking: get next node of a popped stack.
+ *
+ * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
+ * needs to block.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_nonblocking(struct cds_wfs_node *node)
+{
+	return ___cds_wfs_next(node, 0);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_WFSTACK_H */
diff --git a/contrib/userspace-rcu/wfcqueue.h b/contrib/userspace-rcu/wfcqueue.h
new file mode 100644
index 00000000000..0292585ac79
--- /dev/null
+++ b/contrib/userspace-rcu/wfcqueue.h
@@ -0,0 +1,216 @@
+#ifndef _URCU_WFCQUEUE_H
+#define _URCU_WFCQUEUE_H
+
+/*
+ * urcu/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't contain it.
+ * The non-LGPL section has been removed. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ */
+
+#define CDS_WFCQ_WOULDBLOCK	((struct cds_wfcq_node *) -1UL)
+
+enum cds_wfcq_ret {
+	CDS_WFCQ_RET_WOULDBLOCK =	-1,
+	CDS_WFCQ_RET_DEST_EMPTY =	0,
+	CDS_WFCQ_RET_DEST_NON_EMPTY =	1,
+	CDS_WFCQ_RET_SRC_EMPTY = 	2,
+};
+
+enum cds_wfcq_state {
+	CDS_WFCQ_STATE_LAST =		(1U << 0),
+};
+
+struct cds_wfcq_node {
+	struct cds_wfcq_node *next;
+};
+
+/*
+ * Do not put head and tail on the same cache-line if concurrent
+ * enqueue/dequeue are expected from many CPUs. This eliminates
+ * false-sharing between enqueue and dequeue.
+ */
+struct __cds_wfcq_head {
+	struct cds_wfcq_node node;
+};
+
+struct cds_wfcq_head {
+	struct cds_wfcq_node node;
+	pthread_mutex_t lock;
+};
+
+#ifndef __cplusplus
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfcq_head and struct __cds_wfcq_head on any of those two
+ * types.
+ */
+typedef union {
+	struct __cds_wfcq_head *_h;
+	struct cds_wfcq_head *h;
+} __attribute__((__transparent_union__)) cds_wfcq_head_ptr_t;
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct __cds_wfcq_head *__cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+	return head;
+}
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct cds_wfcq_head *cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+	return head;
+}
+#else /* #ifndef __cplusplus */
+
+/* C++ ignores transparent union. */
+typedef union {
+	struct __cds_wfcq_head *_h;
+	struct cds_wfcq_head *h;
+} cds_wfcq_head_ptr_t;
+
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t __cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+	cds_wfcq_head_ptr_t ret = { ._h = head };
+	return ret;
+}
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+	cds_wfcq_head_ptr_t ret = { .h = head };
+	return ret;
+}
+#endif /* #else #ifndef __cplusplus */
+
+struct cds_wfcq_tail {
+	struct cds_wfcq_node *p;
+};
+
+#include "static-wfcqueue.h"
+
+#define cds_wfcq_node_init		_cds_wfcq_node_init
+#define cds_wfcq_init			_cds_wfcq_init
+#define __cds_wfcq_init			___cds_wfcq_init
+#define cds_wfcq_destroy		_cds_wfcq_destroy
+#define cds_wfcq_empty			_cds_wfcq_empty
+#define cds_wfcq_enqueue		_cds_wfcq_enqueue
+
+/* Dequeue locking */
+#define cds_wfcq_dequeue_lock		_cds_wfcq_dequeue_lock
+#define cds_wfcq_dequeue_unlock		_cds_wfcq_dequeue_unlock
+
+/* Locking performed within cds_wfcq calls. */
+#define cds_wfcq_dequeue_blocking	_cds_wfcq_dequeue_blocking
+#define cds_wfcq_dequeue_with_state_blocking	\
+					_cds_wfcq_dequeue_with_state_blocking
+#define cds_wfcq_splice_blocking	_cds_wfcq_splice_blocking
+#define cds_wfcq_first_blocking		_cds_wfcq_first_blocking
+#define cds_wfcq_next_blocking		_cds_wfcq_next_blocking
+
+/* Locking ensured by caller by holding cds_wfcq_dequeue_lock() */
+#define __cds_wfcq_dequeue_blocking	___cds_wfcq_dequeue_blocking
+#define __cds_wfcq_dequeue_with_state_blocking	\
+					___cds_wfcq_dequeue_with_state_blocking
+#define __cds_wfcq_splice_blocking	___cds_wfcq_splice_blocking
+#define __cds_wfcq_first_blocking	___cds_wfcq_first_blocking
+#define __cds_wfcq_next_blocking	___cds_wfcq_next_blocking
+
+/*
+ * Locking ensured by caller by holding cds_wfcq_dequeue_lock().
+ * Non-blocking: deque, first, next return CDS_WFCQ_WOULDBLOCK if they
+ * need to block. splice returns nonzero if it needs to block.
+ */
+#define __cds_wfcq_dequeue_nonblocking	___cds_wfcq_dequeue_nonblocking
+#define __cds_wfcq_dequeue_with_state_nonblocking	\
+				___cds_wfcq_dequeue_with_state_nonblocking
+#define __cds_wfcq_splice_nonblocking	___cds_wfcq_splice_nonblocking
+#define __cds_wfcq_first_nonblocking	___cds_wfcq_first_nonblocking
+#define __cds_wfcq_next_nonblocking	___cds_wfcq_next_nonblocking
+
+/*
+ * __cds_wfcq_for_each_blocking: Iterate over all nodes in a queue,
+ * without dequeuing them.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking(head, tail, node)		\
+	for (node = __cds_wfcq_first_blocking(head, tail);	\
+		node != NULL;					\
+		node = __cds_wfcq_next_blocking(head, tail, node))
+
+/*
+ * __cds_wfcq_for_each_blocking_safe: Iterate over all nodes in a queue,
+ * without dequeuing them. Safe against deletion.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ * @n: struct cds_wfcq_node pointer holding the next pointer (used
+ *     internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking_safe(head, tail, node, n)		       \
+	for (node = __cds_wfcq_first_blocking(head, tail),		       \
+			n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL); \
+		node != NULL;						       \
+		node = n, n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_H */
diff --git a/contrib/userspace-rcu/wfstack.h b/contrib/userspace-rcu/wfstack.h
new file mode 100644
index 00000000000..738fd1cfd33
--- /dev/null
+++ b/contrib/userspace-rcu/wfstack.h
@@ -0,0 +1,178 @@
+#ifndef _URCU_WFSTACK_H
+#define _URCU_WFSTACK_H
+
+/*
+ * urcu/wfstack.h
+ *
+ * Userspace RCU library - Stack with wait-free push, blocking traversal.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Adapted from userspace-rcu 0.10 because version 0.7 doesn't support a stack
+ * without mutex. The non-LGPL section has been removed. */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ *                       cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ *                      iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ *                      cds_wfs_push  __cds_wfs_pop  __cds_wfs_pop_all
+ * cds_wfs_push               -              -                  -
+ * __cds_wfs_pop              -              X                  X
+ * __cds_wfs_pop_all          -              X                  -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+#define CDS_WFS_WOULDBLOCK	((void *) -1UL)
+
+enum cds_wfs_state {
+	CDS_WFS_STATE_LAST =		(1U << 0),
+};
+
+/*
+ * struct cds_wfs_node is returned by __cds_wfs_pop, and also used as
+ * iterator on stack. It is not safe to dereference the node next
+ * pointer when returned by __cds_wfs_pop_blocking.
+ */
+struct cds_wfs_node {
+	struct cds_wfs_node *next;
+};
+
+/*
+ * struct cds_wfs_head is returned by __cds_wfs_pop_all, and can be used
+ * to begin iteration on the stack. "node" needs to be the first field of
+ * cds_wfs_head, so the end-of-stack pointer value can be used for both
+ * types.
+ */
+struct cds_wfs_head {
+	struct cds_wfs_node node;
+};
+
+struct __cds_wfs_stack {
+	struct cds_wfs_head *head;
+};
+
+struct cds_wfs_stack {
+	struct cds_wfs_head *head;
+	pthread_mutex_t lock;
+};
+
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfs_stack and struct __cds_wfs_stack on any of those two
+ * types.
+ */
+typedef union {
+	struct __cds_wfs_stack *_s;
+	struct cds_wfs_stack *s;
+} __attribute__((__transparent_union__)) cds_wfs_stack_ptr_t;
+
+#include "static-wfstack.h"
+
+#define cds_wfs_node_init		_cds_wfs_node_init
+#define cds_wfs_init			_cds_wfs_init
+#define cds_wfs_destroy			_cds_wfs_destroy
+#define __cds_wfs_init			___cds_wfs_init
+#define cds_wfs_empty			_cds_wfs_empty
+#define cds_wfs_push			_cds_wfs_push
+
+/* Locking performed internally */
+#define cds_wfs_pop_blocking		_cds_wfs_pop_blocking
+#define cds_wfs_pop_with_state_blocking	_cds_wfs_pop_with_state_blocking
+#define cds_wfs_pop_all_blocking	_cds_wfs_pop_all_blocking
+
+/*
+ * For iteration on cds_wfs_head returned by __cds_wfs_pop_all or
+ * cds_wfs_pop_all_blocking.
+ */
+#define cds_wfs_first			_cds_wfs_first
+#define cds_wfs_next_blocking		_cds_wfs_next_blocking
+#define cds_wfs_next_nonblocking	_cds_wfs_next_nonblocking
+
+/* Pop locking with internal mutex */
+#define cds_wfs_pop_lock		_cds_wfs_pop_lock
+#define cds_wfs_pop_unlock		_cds_wfs_pop_unlock
+
+/* Synchronization ensured by the caller. See synchronization table. */
+#define __cds_wfs_pop_blocking		___cds_wfs_pop_blocking
+#define __cds_wfs_pop_with_state_blocking	\
+					___cds_wfs_pop_with_state_blocking
+#define __cds_wfs_pop_nonblocking	___cds_wfs_pop_nonblocking
+#define __cds_wfs_pop_with_state_nonblocking	\
+					___cds_wfs_pop_with_state_nonblocking
+#define __cds_wfs_pop_all		___cds_wfs_pop_all
+
+#ifdef __cplusplus
+}
+#endif
+
+/*
+ * cds_wfs_for_each_blocking: Iterate over all nodes returned by
+ * __cds_wfs_pop_all().
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking(head, node)			\
+	for (node = cds_wfs_first(head);			\
+		node != NULL;					\
+		node = cds_wfs_next_blocking(node))
+
+/*
+ * cds_wfs_for_each_blocking_safe: Iterate over all nodes returned by
+ * __cds_wfs_pop_all(). Safe against deletion.
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ * @n: struct cds_wfs_node pointer holding the next pointer (used
+ *     internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking_safe(head, node, n)			   \
+	for (node = cds_wfs_first(head),				   \
+			n = (node ? cds_wfs_next_blocking(node) : NULL);   \
+		node != NULL;						   \
+		node = n, n = (node ? cds_wfs_next_blocking(node) : NULL))
+
+#endif /* _URCU_WFSTACK_H */
diff --git a/contrib/uuid/clear.c b/contrib/uuid/clear.c
deleted file mode 100644
index 2d91fee9399..00000000000
--- a/contrib/uuid/clear.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * clear.c -- Clear a UUID
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "string.h"
-
-#include "uuidP.h"
-
-void uuid_clear(uuid_t uu)
-{
-	memset(uu, 0, 16);
-}
-
diff --git a/contrib/uuid/compare.c b/contrib/uuid/compare.c
deleted file mode 100644
index f28a72678cf..00000000000
--- a/contrib/uuid/compare.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * compare.c --- compare whether or not two UUID's are the same
- *
- * Returns 0 if the two UUID's are different, and 1 if they are the same.
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-#include <string.h>
-
-#define UUCMP(u1,u2) if (u1 != u2) return((u1 < u2) ? -1 : 1);
-
-int uuid_compare(const uuid_t uu1, const uuid_t uu2)
-{
-	struct uuid	uuid1, uuid2;
-
-	uuid_unpack(uu1, &uuid1);
-	uuid_unpack(uu2, &uuid2);
-
-	UUCMP(uuid1.time_low, uuid2.time_low);
-	UUCMP(uuid1.time_mid, uuid2.time_mid);
-	UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version);
-	UUCMP(uuid1.clock_seq, uuid2.clock_seq);
-	return memcmp(uuid1.node, uuid2.node, 6);
-}
-
diff --git a/contrib/uuid/copy.c b/contrib/uuid/copy.c
deleted file mode 100644
index ead33aa26e8..00000000000
--- a/contrib/uuid/copy.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * copy.c --- copy UUIDs
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-
-void uuid_copy(uuid_t dst, const uuid_t src)
-{
-	unsigned char		*cp1;
-	const unsigned char	*cp2;
-	int			i;
-
-	for (i=0, cp1 = dst, cp2 = src; i < 16; i++)
-		*cp1++ = *cp2++;
-}
diff --git a/contrib/uuid/gen_uuid.c b/contrib/uuid/gen_uuid.c
deleted file mode 100644
index b3eda9de387..00000000000
--- a/contrib/uuid/gen_uuid.c
+++ /dev/null
@@ -1,679 +0,0 @@
-/*
- * gen_uuid.c --- generate a DCE-compatible uuid
- *
- * Copyright (C) 1996, 1997, 1998, 1999 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-/*
- * Force inclusion of SVID stuff since we need it if we're compiling in
- * gcc-wall wall mode
- */
-#define _SVID_SOURCE
-
-#include "config.h"
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-#include <string.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <sys/types.h>
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#include <sys/wait.h>
-#include <sys/stat.h>
-#ifdef HAVE_SYS_FILE_H
-#include <sys/file.h>
-#endif
-#ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
-#endif
-#ifdef HAVE_NET_IF_H
-#include <net/if.h>
-#endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_NET_IF_DL_H
-#include <net/if_dl.h>
-#endif
-#if defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)
-#include <sys/syscall.h>
-#endif
-#ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
-#endif
-#include <limits.h>
-
-#include "uuidP.h"
-#include "uuidd.h"
-
-#ifdef HAVE_SRANDOM
-#define srand(x) 	srandom(x)
-#define rand() 		random()
-#endif
-
-#ifdef TLS
-#define THREAD_LOCAL static TLS
-#else
-#define THREAD_LOCAL static
-#endif
-
-#if defined(__linux__) && defined(__NR_gettid) && defined(HAVE_JRAND48)
-#define DO_JRAND_MIX
-THREAD_LOCAL unsigned short jrand_seed[3];
-#endif
-
-#ifndef OPEN_MAX
-#define OPEN_MAX 1024
-#endif
-
-#ifdef _WIN32
-static void gettimeofday (struct timeval *tv, void *dummy)
-{
-	FILETIME	ftime;
-	uint64_t	n;
-
-	GetSystemTimeAsFileTime (&ftime);
-	n = (((uint64_t) ftime.dwHighDateTime << 32)
-	     + (uint64_t) ftime.dwLowDateTime);
-	if (n) {
-		n /= 10;
-		n -= ((369 * 365 + 89) * (uint64_t) 86400) * 1000000;
-	}
-
-	tv->tv_sec = n / 1000000;
-	tv->tv_usec = n % 1000000;
-}
-
-static int getuid (void)
-{
-	return 1;
-}
-#endif
-
-static int get_random_fd(void)
-{
-	struct timeval	tv;
-	static int	fd = -2;
-	int		i;
-
-	if (fd == -2) {
-		gettimeofday(&tv, 0);
-#ifndef _WIN32
-		fd = open("/dev/urandom", O_RDONLY);
-		if (fd == -1)
-			fd = open("/dev/random", O_RDONLY | O_NONBLOCK);
-		if (fd >= 0) {
-			i = fcntl(fd, F_GETFD);
-			if (i >= 0)
-				fcntl(fd, F_SETFD, i | FD_CLOEXEC);
-		}
-#endif
-		srand((getpid() << 16) ^ getuid() ^ tv.tv_sec ^ tv.tv_usec);
-#ifdef DO_JRAND_MIX
-		jrand_seed[0] = getpid() ^ (tv.tv_sec & 0xFFFF);
-		jrand_seed[1] = getppid() ^ (tv.tv_usec & 0xFFFF);
-		jrand_seed[2] = (tv.tv_sec ^ tv.tv_usec) >> 16;
-#endif
-	}
-	/* Crank the random number generator a few times */
-	gettimeofday(&tv, 0);
-	for (i = (tv.tv_sec ^ tv.tv_usec) & 0x1F; i > 0; i--)
-		rand();
-	return fd;
-}
-
-
-/*
- * Generate a series of random bytes.  Use /dev/urandom if possible,
- * and if not, use srandom/random.
- */
-static void get_random_bytes(void *buf, int nbytes)
-{
-	int i, n = nbytes, fd = get_random_fd();
-	int lose_counter = 0;
-	unsigned char *cp = (unsigned char *) buf;
-#ifdef DO_JRAND_MIX
-	unsigned short tmp_seed[3];
-#endif
-	if (fd >= 0) {
-		while (n > 0) {
-			i = read(fd, cp, n);
-			if (i <= 0) {
-				if (lose_counter++ > 16)
-					break;
-				continue;
-			}
-			n -= i;
-			cp += i;
-			lose_counter = 0;
-		}
-	}
-
-	/*
-	 * We do this all the time, but this is the only source of
-	 * randomness if /dev/random/urandom is out to lunch.
-	 */
-	for (cp = buf, i = 0; i < nbytes; i++)
-		*cp++ ^= (rand() >> 7) & 0xFF;
-#ifdef DO_JRAND_MIX
-	memcpy(tmp_seed, jrand_seed, sizeof(tmp_seed));
-	jrand_seed[2] = jrand_seed[2] ^ syscall(__NR_gettid);
-	for (cp = buf, i = 0; i < nbytes; i++)
-		*cp++ ^= (jrand48(tmp_seed) >> 7) & 0xFF;
-	memcpy(jrand_seed, tmp_seed,
-	       sizeof(jrand_seed)-sizeof(unsigned short));
-#endif
-
-	return;
-}
-
-/*
- * Get the ethernet hardware address, if we can find it...
- *
- * XXX for a windows version, probably should use GetAdaptersInfo:
- * http://www.codeguru.com/cpp/i-n/network/networkinformation/article.php/c5451
- * commenting out get_node_id just to get gen_uuid to compile under windows
- * is not the right way to go!
- */
-static int get_node_id(unsigned char *node_id)
-{
-#ifdef HAVE_NET_IF_H
-	int 		sd;
-	struct ifreq 	ifr, *ifrp;
-	struct ifconf 	ifc;
-	char buf[1024];
-	int		n, i;
-	unsigned char 	*a;
-#ifdef HAVE_NET_IF_DL_H
-	struct sockaddr_dl *sdlp;
-#endif
-
-/*
- * BSD 4.4 defines the size of an ifreq to be
- * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len
- * However, under earlier systems, sa_len isn't present, so the size is
- * just sizeof(struct ifreq)
- */
-#ifdef HAVE_SA_LEN
-#ifndef max
-#define max(a,b) ((a) > (b) ? (a) : (b))
-#endif
-#define ifreq_size(i) max(sizeof(struct ifreq),\
-     sizeof((i).ifr_name)+(i).ifr_addr.sa_len)
-#else
-#define ifreq_size(i) sizeof(struct ifreq)
-#endif /* HAVE_SA_LEN*/
-
-	sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
-	if (sd < 0) {
-		return -1;
-	}
-	memset(buf, 0, sizeof(buf));
-	ifc.ifc_len = sizeof(buf);
-	ifc.ifc_buf = buf;
-	if (ioctl (sd, SIOCGIFCONF, (char *)&ifc) < 0) {
-		close(sd);
-		return -1;
-	}
-	n = ifc.ifc_len;
-	for (i = 0; i < n; i+= ifreq_size(*ifrp) ) {
-		ifrp = (struct ifreq *)((char *) ifc.ifc_buf+i);
-		strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ);
-#ifdef SIOCGIFHWADDR
-		if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0)
-			continue;
-		a = (unsigned char *) &ifr.ifr_hwaddr.sa_data;
-#else
-#ifdef SIOCGENADDR
-		if (ioctl(sd, SIOCGENADDR, &ifr) < 0)
-			continue;
-		a = (unsigned char *) ifr.ifr_enaddr;
-#else
-#ifdef HAVE_NET_IF_DL_H
-		sdlp = (struct sockaddr_dl *) &ifrp->ifr_addr;
-		if ((sdlp->sdl_family != AF_LINK) || (sdlp->sdl_alen != 6))
-			continue;
-		a = (unsigned char *) &sdlp->sdl_data[sdlp->sdl_nlen];
-#else
-		/*
-		 * XXX we don't have a way of getting the hardware
-		 * address
-		 */
-		close(sd);
-		return 0;
-#endif /* HAVE_NET_IF_DL_H */
-#endif /* SIOCGENADDR */
-#endif /* SIOCGIFHWADDR */
-		if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5])
-			continue;
-		if (node_id) {
-			memcpy(node_id, a, 6);
-			close(sd);
-			return 1;
-		}
-	}
-	close(sd);
-#endif
-	return 0;
-}
-
-/* Assume that the gettimeofday() has microsecond granularity */
-#define MAX_ADJUSTMENT 10
-
-static int get_clock(uint32_t *clock_high, uint32_t *clock_low,
-		     uint16_t *ret_clock_seq, int *num)
-{
-	THREAD_LOCAL int		adjustment = 0;
-	THREAD_LOCAL struct timeval	last = {0, 0};
-	THREAD_LOCAL int		state_fd = -2;
-	THREAD_LOCAL FILE		*state_f;
-	THREAD_LOCAL uint16_t		clock_seq;
-	struct timeval 			tv;
-	struct flock			fl;
-	uint64_t			clock_reg;
-	mode_t				save_umask;
-	int				len;
-
-	if (state_fd == -2) {
-		save_umask = umask(0);
-		state_fd = open("/var/lib/libuuid/clock.txt",
-				O_RDWR|O_CREAT, 0660);
-		(void) umask(save_umask);
-		state_f = fdopen(state_fd, "r+");
-		if (!state_f) {
-			close(state_fd);
-			state_fd = -1;
-		}
-	}
-	fl.l_type = F_WRLCK;
-	fl.l_whence = SEEK_SET;
-	fl.l_start = 0;
-	fl.l_len = 0;
-	fl.l_pid = 0;
-	if (state_fd >= 0) {
-		rewind(state_f);
-		while (fcntl(state_fd, F_SETLKW, &fl) < 0) {
-			if ((errno == EAGAIN) || (errno == EINTR))
-				continue;
-			fclose(state_f);
-			close(state_fd);
-			state_fd = -1;
-			break;
-		}
-	}
-	if (state_fd >= 0) {
-		unsigned int cl;
-		unsigned long tv1, tv2;
-		int a;
-
-		if (fscanf(state_f, "clock: %04x tv: %lu %lu adj: %d\n",
-			   &cl, &tv1, &tv2, &a) == 4) {
-			clock_seq = cl & 0x3FFF;
-			last.tv_sec = tv1;
-			last.tv_usec = tv2;
-			adjustment = a;
-		}
-	}
-
-	if ((last.tv_sec == 0) && (last.tv_usec == 0)) {
-		get_random_bytes(&clock_seq, sizeof(clock_seq));
-		clock_seq &= 0x3FFF;
-		gettimeofday(&last, 0);
-		last.tv_sec--;
-	}
-
-try_again:
-	gettimeofday(&tv, 0);
-	if ((tv.tv_sec < last.tv_sec) ||
-	    ((tv.tv_sec == last.tv_sec) &&
-	     (tv.tv_usec < last.tv_usec))) {
-		clock_seq = (clock_seq+1) & 0x3FFF;
-		adjustment = 0;
-		last = tv;
-	} else if ((tv.tv_sec == last.tv_sec) &&
-	    (tv.tv_usec == last.tv_usec)) {
-		if (adjustment >= MAX_ADJUSTMENT)
-			goto try_again;
-		adjustment++;
-	} else {
-		adjustment = 0;
-		last = tv;
-	}
-
-	clock_reg = tv.tv_usec*10 + adjustment;
-	clock_reg += ((uint64_t) tv.tv_sec)*10000000;
-	clock_reg += (((uint64_t) 0x01B21DD2) << 32) + 0x13814000;
-
-	if (num && (*num > 1)) {
-		adjustment += *num - 1;
-		last.tv_usec += adjustment / 10;
-		adjustment = adjustment % 10;
-		last.tv_sec += last.tv_usec / 1000000;
-		last.tv_usec = last.tv_usec % 1000000;
-	}
-
-	if (state_fd > 0) {
-		rewind(state_f);
-		len = fprintf(state_f, 
-			      "clock: %04x tv: %016lu %08lu adj: %08d\n",
-			      clock_seq, last.tv_sec, last.tv_usec, adjustment);
-		fflush(state_f);
-		if (ftruncate(state_fd, len) < 0) {
-			fprintf(state_f, "                   \n");
-			fflush(state_f);
-		}
-		rewind(state_f);
-		fl.l_type = F_UNLCK;
-		fcntl(state_fd, F_SETLK, &fl);
-	}
-
-	*clock_high = clock_reg >> 32;
-	*clock_low = clock_reg;
-	*ret_clock_seq = clock_seq;
-	return 0;
-}
-
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-static ssize_t read_all(int fd, char *buf, size_t count)
-{
-	ssize_t ret;
-	ssize_t c = 0;
-	int tries = 0;
-
-	memset(buf, 0, count);
-	while (count > 0) {
-		ret = read(fd, buf, count);
-		if (ret <= 0) {
-			if ((errno == EAGAIN || errno == EINTR || ret == 0) &&
-			    (tries++ < 5))
-				continue;
-			return c ? c : -1;
-		}
-		if (ret > 0)
-			tries = 0;
-		count -= ret;
-		buf += ret;
-		c += ret;
-	}
-	return c;
-}
-#endif
-
-/*
- * Close all file descriptors
- */
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-static void close_all_fds(void)
-{
-	int i, max;
-
-#if defined(HAVE_SYSCONF) && defined(_SC_OPEN_MAX)
-	max = sysconf(_SC_OPEN_MAX);
-#elif defined(HAVE_GETDTABLESIZE)
-	max = getdtablesize();
-#elif defined(HAVE_GETRLIMIT) && defined(RLIMIT_NOFILE)
-	struct rlimit rl;
-
-	getrlimit(RLIMIT_NOFILE, &rl);
-	max = rl.rlim_cur;
-#else
-	max = OPEN_MAX;
-#endif
-
-	for (i=0; i < max; i++) {
-		close(i);
-		if (i <= 2)
-			open("/dev/null", O_RDWR);
-	}
-}
-#endif
-
-
-/*
- * Try using the uuidd daemon to generate the UUID
- *
- * Returns 0 on success, non-zero on failure.
- */
-static int get_uuid_via_daemon(int op, uuid_t out, int *num)
-{
-#if defined(USE_UUIDD) && defined(HAVE_SYS_UN_H)
-	char op_buf[64];
-	int op_len;
-	int s;
-	ssize_t ret;
-	int32_t reply_len = 0, expected = 16;
-	struct sockaddr_un srv_addr;
-	struct stat st;
-	pid_t pid;
-	static const char *uuidd_path = UUIDD_PATH;
-	static int access_ret = -2;
-	static int start_attempts = 0;
-
-	if ((s = socket(AF_UNIX, SOCK_STREAM, 0)) < 0)
-		return -1;
-
-	srv_addr.sun_family = AF_UNIX;
-	strcpy(srv_addr.sun_path, UUIDD_SOCKET_PATH);
-
-	if (connect(s, (const struct sockaddr *) &srv_addr,
-		    sizeof(struct sockaddr_un)) < 0) {
-		if (access_ret == -2)
-			access_ret = access(uuidd_path, X_OK);
-		if (access_ret == 0)
-			access_ret = stat(uuidd_path, &st);
-		if (access_ret == 0 && (st.st_mode & (S_ISUID | S_ISGID)) == 0)
-			access_ret = access(UUIDD_DIR, W_OK);
-		if (access_ret == 0 && start_attempts++ < 5) {
-			if ((pid = fork()) == 0) {
-				close_all_fds();
-				execl(uuidd_path, "uuidd", "-qT", "300",
-				      (char *) NULL);
-				exit(1);
-			}
-			(void) waitpid(pid, 0, 0);
-			if (connect(s, (const struct sockaddr *) &srv_addr,
-				    sizeof(struct sockaddr_un)) < 0)
-				goto fail;
-		} else
-			goto fail;
-	}
-	op_buf[0] = op;
-	op_len = 1;
-	if (op == UUIDD_OP_BULK_TIME_UUID) {
-		memcpy(op_buf+1, num, sizeof(*num));
-		op_len += sizeof(*num);
-		expected += sizeof(*num);
-	}
-
-	ret = write(s, op_buf, op_len);
-	if (ret < 1)
-		goto fail;
-
-	ret = read_all(s, (char *) &reply_len, sizeof(reply_len));
-	if (ret < 0)
-		goto fail;
-
-	if (reply_len != expected)
-		goto fail;
-
-	ret = read_all(s, op_buf, reply_len);
-
-	if (op == UUIDD_OP_BULK_TIME_UUID)
-		memcpy(op_buf+16, num, sizeof(int));
-
-	memcpy(out, op_buf, 16);
-
-	close(s);
-	return ((ret == expected) ? 0 : -1);
-
-fail:
-	close(s);
-#endif
-	return -1;
-}
-
-void uuid__generate_time(uuid_t out, int *num)
-{
-	static unsigned char node_id[6];
-	static int has_init = 0;
-	struct uuid uu;
-	uint32_t	clock_mid;
-
-	if (!has_init) {
-		if (get_node_id(node_id) <= 0) {
-			get_random_bytes(node_id, 6);
-			/*
-			 * Set multicast bit, to prevent conflicts
-			 * with IEEE 802 addresses obtained from
-			 * network cards
-			 */
-			node_id[0] |= 0x01;
-		}
-		has_init = 1;
-	}
-	get_clock(&clock_mid, &uu.time_low, &uu.clock_seq, num);
-	uu.clock_seq |= 0x8000;
-	uu.time_mid = (uint16_t) clock_mid;
-	uu.time_hi_and_version = ((clock_mid >> 16) & 0x0FFF) | 0x1000;
-	memcpy(uu.node, node_id, 6);
-	uuid_pack(&uu, out);
-}
-
-void uuid_generate_time(uuid_t out)
-{
-#ifdef TLS
-	THREAD_LOCAL int		num = 0;
-	THREAD_LOCAL struct uuid	uu;
-	THREAD_LOCAL time_t		last_time = 0;
-	time_t				now;
-
-	if (num > 0) {
-		now = time(0);
-		if (now > last_time+1)
-			num = 0;
-	}
-	if (num <= 0) {
-		num = 1000;
-		if (get_uuid_via_daemon(UUIDD_OP_BULK_TIME_UUID,
-					out, &num) == 0) {
-			last_time = time(0);
-			uuid_unpack(out, &uu);
-			num--;
-			return;
-		}
-		num = 0;
-	}
-	if (num > 0) {
-		uu.time_low++;
-		if (uu.time_low == 0) {
-			uu.time_mid++;
-			if (uu.time_mid == 0)
-				uu.time_hi_and_version++;
-		}
-		num--;
-		uuid_pack(&uu, out);
-		return;
-	}
-#else
-	if (get_uuid_via_daemon(UUIDD_OP_TIME_UUID, out, 0) == 0)
-		return;
-#endif
-
-	uuid__generate_time(out, 0);
-}
-
-
-void uuid__generate_random(uuid_t out, int *num)
-{
-	uuid_t	buf;
-	struct uuid uu;
-	int i, n;
-
-	if (!num || !*num)
-		n = 1;
-	else
-		n = *num;
-
-	for (i = 0; i < n; i++) {
-		get_random_bytes(buf, sizeof(buf));
-		uuid_unpack(buf, &uu);
-
-		uu.clock_seq = (uu.clock_seq & 0x3FFF) | 0x8000;
-		uu.time_hi_and_version = (uu.time_hi_and_version & 0x0FFF)
-			| 0x4000;
-		uuid_pack(&uu, out);
-		out += sizeof(uuid_t);
-	}
-}
-
-void uuid_generate_random(uuid_t out)
-{
-	int	num = 1;
-	/* No real reason to use the daemon for random uuid's -- yet */
-
-	uuid__generate_random(out, &num);
-}
-
-
-/*
- * This is the generic front-end to uuid_generate_random and
- * uuid_generate_time.  It uses uuid_generate_random only if
- * /dev/urandom is available, since otherwise we won't have
- * high-quality randomness.
- */
-void uuid_generate(uuid_t out)
-{
-	if (get_random_fd() >= 0)
-		uuid_generate_random(out);
-	else
-		uuid_generate_time(out);
-}
diff --git a/contrib/uuid/gen_uuid_nt.c b/contrib/uuid/gen_uuid_nt.c
deleted file mode 100644
index aa44bfd3d7d..00000000000
--- a/contrib/uuid/gen_uuid_nt.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * gen_uuid_nt.c -- Use NT api to generate uuid
- *
- * Written by Andrey Shedel (andreys@ns.cr.cyco.com)
- */
-
-
-#include "uuidP.h"
-
-#pragma warning(push,4)
-
-#pragma comment(lib, "ntdll.lib")
-
-//
-// Here is a nice example why it's not a good idea
-// to use native API in ordinary applications.
-// Number of parameters in function below was changed from 3 to 4
-// for NT5.
-//
-//
-// NTSYSAPI
-// NTSTATUS
-// NTAPI
-// NtAllocateUuids(
-//     OUT PULONG p1,
-//     OUT PULONG p2,
-//     OUT PULONG p3,
-//     OUT PUCHAR Seed // 6 bytes
-//   );
-//
-//
-
-unsigned long
-__stdcall
-NtAllocateUuids(
-   void* p1,  // 8 bytes
-   void* p2,  // 4 bytes
-   void* p3   // 4 bytes
-   );
-
-typedef
-unsigned long
-(__stdcall*
-NtAllocateUuids_2000)(
-   void* p1,  // 8 bytes
-   void* p2,  // 4 bytes
-   void* p3,  // 4 bytes
-   void* seed // 6 bytes
-   );
-
-
-
-//
-// Nice, but instead of including ntddk.h ot winnt.h
-// I should define it here because they MISSED __stdcall in those headers.
-//
-
-__declspec(dllimport)
-struct _TEB*
-__stdcall
-NtCurrentTeb(void);
-
-
-//
-// The only way to get version information from the system is to examine
-// one stored in PEB. But it's pretty dangerouse because this value could
-// be altered in image header.
-//
-
-static
-int
-Nt5(void)
-{
-	//return NtCuttentTeb()->Peb->OSMajorVersion >= 5;
-	return (int)*(int*)((char*)(int)(*(int*)((char*)NtCurrentTeb() + 0x30)) + 0xA4) >= 5;
-}
-
-
-
-
-void uuid_generate(uuid_t out)
-{
-	if(Nt5())
-	{
-		unsigned char seed[6];
-		((NtAllocateUuids_2000)NtAllocateUuids)(out, ((char*)out)+8, ((char*)out)+12, &seed[0] );
-	}
-	else
-	{
-		NtAllocateUuids(out, ((char*)out)+8, ((char*)out)+12);
-	}
-}
diff --git a/contrib/uuid/isnull.c b/contrib/uuid/isnull.c
deleted file mode 100644
index 931e7e7dba2..00000000000
--- a/contrib/uuid/isnull.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * isnull.c --- Check whether or not the UUID is null
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuidP.h"
-
-/* Returns 1 if the uuid is the NULL uuid */
-int uuid_is_null(const uuid_t uu)
-{
-	const unsigned char 	*cp;
-	int			i;
-
-	for (i=0, cp = uu; i < 16; i++)
-		if (*cp++)
-			return 0;
-	return 1;
-}
-
diff --git a/contrib/uuid/pack.c b/contrib/uuid/pack.c
deleted file mode 100644
index 097516d2e2f..00000000000
--- a/contrib/uuid/pack.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Internal routine for packing UUID's
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <string.h>
-#include "uuidP.h"
-
-void uuid_pack(const struct uuid *uu, uuid_t ptr)
-{
-	uint32_t	tmp;
-	unsigned char	*out = ptr;
-
-	tmp = uu->time_low;
-	out[3] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[2] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[1] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[0] = (unsigned char) tmp;
-
-	tmp = uu->time_mid;
-	out[5] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[4] = (unsigned char) tmp;
-
-	tmp = uu->time_hi_and_version;
-	out[7] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[6] = (unsigned char) tmp;
-
-	tmp = uu->clock_seq;
-	out[9] = (unsigned char) tmp;
-	tmp >>= 8;
-	out[8] = (unsigned char) tmp;
-
-	memcpy(out+10, uu->node, 6);
-}
-
diff --git a/contrib/uuid/parse.c b/contrib/uuid/parse.c
deleted file mode 100644
index 074383efae7..00000000000
--- a/contrib/uuid/parse.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * parse.c --- UUID parsing
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-
-#include "uuidP.h"
-
-int uuid_parse(const char *in, uuid_t uu)
-{
-	struct uuid	uuid;
-	int 		i;
-	const char	*cp;
-	char		buf[3];
-
-	if (strlen(in) != 36)
-		return -1;
-	for (i=0, cp = in; i <= 36; i++,cp++) {
-		if ((i == 8) || (i == 13) || (i == 18) ||
-		    (i == 23)) {
-			if (*cp == '-')
-				continue;
-			else
-				return -1;
-		}
-		if (i== 36)
-			if (*cp == 0)
-				continue;
-		if (!isxdigit(*cp))
-			return -1;
-	}
-	uuid.time_low = strtoul(in, NULL, 16);
-	uuid.time_mid = strtoul(in+9, NULL, 16);
-	uuid.time_hi_and_version = strtoul(in+14, NULL, 16);
-	uuid.clock_seq = strtoul(in+19, NULL, 16);
-	cp = in+24;
-	buf[2] = 0;
-	for (i=0; i < 6; i++) {
-		buf[0] = *cp++;
-		buf[1] = *cp++;
-		uuid.node[i] = strtoul(buf, NULL, 16);
-	}
-
-	uuid_pack(&uuid, uu);
-	return 0;
-}
diff --git a/contrib/uuid/tst_uuid.c b/contrib/uuid/tst_uuid.c
deleted file mode 100644
index e03138f7d18..00000000000
--- a/contrib/uuid/tst_uuid.c
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * tst_uuid.c --- test program from the UUID library
- *
- * Copyright (C) 1996, 1997, 1998 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "uuid.h"
-
-static int test_uuid(const char * uuid, int isValid)
-{
-	static const char * validStr[2] = {"invalid", "valid"};
-	uuid_t uuidBits;
-	int parsedOk;
-
-	parsedOk = uuid_parse(uuid, uuidBits) == 0;
-
-	printf("%s is %s", uuid, validStr[isValid]);
-	if (parsedOk != isValid) {
-		printf(" but uuid_parse says %s\n", validStr[parsedOk]);
-		return 1;
-	}
-	printf(", OK\n");
-	return 0;
-}
-
-#ifdef __GNUC__
-#define ATTR(x) __attribute__(x)
-#else
-#define ATTR(x)
-#endif
-
-int
-main(int argc ATTR((unused)) , char **argv ATTR((unused)))
-{
-	uuid_t		buf, tst;
-	char		str[100];
-	struct timeval	tv;
-	time_t		time_reg;
-	unsigned char	*cp;
-	int i;
-	int failed = 0;
-	int type, variant;
-
-	uuid_generate(buf);
-	uuid_unparse(buf, str);
-	printf("UUID generate = %s\n", str);
-	printf("UUID: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = uuid_type(buf); 	variant = uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	printf("\n");
-
-	uuid_generate_random(buf);
-	uuid_unparse(buf, str);
-	printf("UUID random string = %s\n", str);
-	printf("UUID: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = uuid_type(buf); 	variant = uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	if (type != 4) {
-		printf("Incorrect UUID type; was expecting "
-		       "4 (random type)!\n");
-		failed++;
-	}
-	printf("\n");
-
-	uuid_generate_time(buf);
-	uuid_unparse(buf, str);
-	printf("UUID string = %s\n", str);
-	printf("UUID time: ");
-	for (i=0, cp = (unsigned char *) &buf; i < 16; i++) {
-		printf("%02x", *cp++);
-	}
-	printf("\n");
-	type = uuid_type(buf); 	variant = uuid_variant(buf);
-	printf("UUID type = %d, UUID variant = %d\n", type, variant);
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Incorrect UUID Variant; was expecting DCE!\n");
-		failed++;
-	}
-	if (type != 1) {
-		printf("Incorrect UUID type; was expecting "
-		       "1 (time-based type)!\\n");
-		failed++;
-	}
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	time_reg = uuid_time(buf, &tv);
-	printf("UUID time is: (%ld, %ld): %s\n", tv.tv_sec, tv.tv_usec,
-	       ctime(&time_reg));
-	uuid_parse(str, tst);
-	if (!uuid_compare(buf, tst))
-		printf("UUID parse and compare succeeded.\n");
-	else {
-		printf("UUID parse and compare failed!\n");
-		failed++;
-	}
-	uuid_clear(tst);
-	if (uuid_is_null(tst))
-		printf("UUID clear and is null succeeded.\n");
-	else {
-		printf("UUID clear and is null failed!\n");
-		failed++;
-	}
-	uuid_copy(buf, tst);
-	if (!uuid_compare(buf, tst))
-		printf("UUID copy and compare succeeded.\n");
-	else {
-		printf("UUID copy and compare failed!\n");
-		failed++;
-	}
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981b", 1);
-	failed += test_uuid("84949CC5-4701-4A84-895B-354C584A981B", 1);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981bc", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981", 0);
-	failed += test_uuid("84949cc5x4701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc504701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-470104a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a840895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b0354c584a981b", 0);
-	failed += test_uuid("g4949cc5-4701-4a84-895b-354c584a981b", 0);
-	failed += test_uuid("84949cc5-4701-4a84-895b-354c584a981g", 0);
-
-	if (failed) {
-		printf("%d failures.\n", failed);
-		exit(1);
-	}
-	return 0;
-}
diff --git a/contrib/uuid/unpack.c b/contrib/uuid/unpack.c
deleted file mode 100644
index beaaff3ca8a..00000000000
--- a/contrib/uuid/unpack.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Internal routine for unpacking UUID
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <string.h>
-#include "uuidP.h"
-
-void uuid_unpack(const uuid_t in, struct uuid *uu)
-{
-	const uint8_t	*ptr = in;
-	uint32_t		tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_low = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_mid = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->time_hi_and_version = tmp;
-
-	tmp = *ptr++;
-	tmp = (tmp << 8) | *ptr++;
-	uu->clock_seq = tmp;
-
-	memcpy(uu->node, ptr, 6);
-}
-
diff --git a/contrib/uuid/unparse.c b/contrib/uuid/unparse.c
deleted file mode 100644
index a95bbb04258..00000000000
--- a/contrib/uuid/unparse.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * unparse.c -- convert a UUID to string
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include <stdio.h>
-
-#include "uuidP.h"
-
-static const char *fmt_lower =
-	"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x";
-
-static const char *fmt_upper =
-	"%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X";
-
-#ifdef UUID_UNPARSE_DEFAULT_UPPER
-#define FMT_DEFAULT fmt_upper
-#else
-#define FMT_DEFAULT fmt_lower
-#endif
-
-static void uuid_unparse_x(const uuid_t uu, char *out, const char *fmt)
-{
-	struct uuid uuid;
-
-	uuid_unpack(uu, &uuid);
-	sprintf(out, fmt,
-		uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,
-		uuid.clock_seq >> 8, uuid.clock_seq & 0xFF,
-		uuid.node[0], uuid.node[1], uuid.node[2],
-		uuid.node[3], uuid.node[4], uuid.node[5]);
-}
-
-void uuid_unparse_lower(const uuid_t uu, char *out)
-{
-	uuid_unparse_x(uu, out,	fmt_lower);
-}
-
-void uuid_unparse_upper(const uuid_t uu, char *out)
-{
-	uuid_unparse_x(uu, out,	fmt_upper);
-}
-
-void uuid_unparse(const uuid_t uu, char *out)
-{
-	uuid_unparse_x(uu, out, FMT_DEFAULT);
-}
diff --git a/contrib/uuid/uuid.h b/contrib/uuid/uuid.h
deleted file mode 100644
index ab006652fc0..00000000000
--- a/contrib/uuid/uuid.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Public include file for the UUID library
- *
- * Copyright (C) 1996, 1997, 1998 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifndef _UUID_UUID_H
-#define _UUID_UUID_H
-
-#include "config.h"
-#include <sys/types.h>
-#ifndef _WIN32
-#include <sys/time.h>
-#endif
-#include <time.h>
-
-typedef unsigned char uuid_t[16];
-
-/* UUID Variant definitions */
-#define UUID_VARIANT_NCS 	0
-#define UUID_VARIANT_DCE 	1
-#define UUID_VARIANT_MICROSOFT	2
-#define UUID_VARIANT_OTHER	3
-
-/* UUID Type definitions */
-#define UUID_TYPE_DCE_TIME   1
-#define UUID_TYPE_DCE_RANDOM 4
-
-/* Allow UUID constants to be defined */
-#ifdef __GNUC__
-#define UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \
-	static const uuid_t name __attribute__ ((unused)) = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15}
-#else
-#define UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \
-	static const uuid_t name = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15}
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* clear.c */
-void uuid_clear(uuid_t uu);
-
-/* compare.c */
-int uuid_compare(const uuid_t uu1, const uuid_t uu2);
-
-/* copy.c */
-void uuid_copy(uuid_t dst, const uuid_t src);
-
-/* gen_uuid.c */
-void uuid_generate(uuid_t out);
-void uuid_generate_random(uuid_t out);
-void uuid_generate_time(uuid_t out);
-
-/* isnull.c */
-int uuid_is_null(const uuid_t uu);
-
-/* parse.c */
-int uuid_parse(const char *in, uuid_t uu);
-
-/* unparse.c */
-void uuid_unparse(const uuid_t uu, char *out);
-void uuid_unparse_lower(const uuid_t uu, char *out);
-void uuid_unparse_upper(const uuid_t uu, char *out);
-
-/* uuid_time.c */
-time_t uuid_time(const uuid_t uu, struct timeval *ret_tv);
-int uuid_type(const uuid_t uu);
-int uuid_variant(const uuid_t uu);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _UUID_UUID_H */
diff --git a/contrib/uuid/uuidP.h b/contrib/uuid/uuidP.h
deleted file mode 100644
index 9a2de6132fe..00000000000
--- a/contrib/uuid/uuidP.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * uuid.h -- private header file for uuids
- *
- * Copyright (C) 1996, 1997 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#include "uuid.h"
-#ifdef HAVE_INTTYPES_H
-#include <inttypes.h>
-#else
-#include "uuid_types.h"
-#endif
-#include <sys/types.h>
-
-
-/*
- * Offset between 15-Oct-1582 and 1-Jan-70
- */
-#define TIME_OFFSET_HIGH 0x01B21DD2
-#define TIME_OFFSET_LOW  0x13814000
-
-struct uuid {
-	uint32_t	time_low;
-	uint16_t	time_mid;
-	uint16_t	time_hi_and_version;
-	uint16_t	clock_seq;
-	uint8_t	node[6];
-};
-
-
-/*
- * prototypes
- */
-void uuid_pack(const struct uuid *uu, uuid_t ptr);
-void uuid_unpack(const uuid_t in, struct uuid *uu);
diff --git a/contrib/uuid/uuid_time.c b/contrib/uuid/uuid_time.c
deleted file mode 100644
index f25f5c90fe5..00000000000
--- a/contrib/uuid/uuid_time.c
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * uuid_time.c --- Interpret the time field from a uuid.  This program
- * 	violates the UUID abstraction barrier by reaching into the guts
- *	of a UUID and interpreting it.
- *
- * Copyright (C) 1998, 1999 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifdef _WIN32
-#define _WIN32_WINNT 0x0500
-#include <windows.h>
-#define UUID MYUUID
-#endif
-
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <stdlib.h>
-#include <sys/types.h>
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#include <time.h>
-
-#include "uuidP.h"
-
-time_t uuid_time(const uuid_t uu, struct timeval *ret_tv)
-{
-	struct timeval		tv;
-	struct uuid		uuid;
-	uint32_t		high;
-	uint64_t		clock_reg;
-
-	uuid_unpack(uu, &uuid);
-
-	high = uuid.time_mid | ((uuid.time_hi_and_version & 0xFFF) << 16);
-	clock_reg = uuid.time_low | ((uint64_t) high << 32);
-
-	clock_reg -= (((uint64_t) 0x01B21DD2) << 32) + 0x13814000;
-	tv.tv_sec = clock_reg / 10000000;
-	tv.tv_usec = (clock_reg % 10000000) / 10;
-
-	if (ret_tv)
-		*ret_tv = tv;
-
-	return tv.tv_sec;
-}
-
-int uuid_type(const uuid_t uu)
-{
-	struct uuid		uuid;
-
-	uuid_unpack(uu, &uuid);
-	return ((uuid.time_hi_and_version >> 12) & 0xF);
-}
-
-int uuid_variant(const uuid_t uu)
-{
-	struct uuid		uuid;
-	int			var;
-
-	uuid_unpack(uu, &uuid);
-	var = uuid.clock_seq;
-
-	if ((var & 0x8000) == 0)
-		return UUID_VARIANT_NCS;
-	if ((var & 0x4000) == 0)
-		return UUID_VARIANT_DCE;
-	if ((var & 0x2000) == 0)
-		return UUID_VARIANT_MICROSOFT;
-	return UUID_VARIANT_OTHER;
-}
-
-#ifdef DEBUG
-static const char *variant_string(int variant)
-{
-	switch (variant) {
-	case UUID_VARIANT_NCS:
-		return "NCS";
-	case UUID_VARIANT_DCE:
-		return "DCE";
-	case UUID_VARIANT_MICROSOFT:
-		return "Microsoft";
-	default:
-		return "Other";
-	}
-}
-
-
-int
-main(int argc, char **argv)
-{
-	uuid_t		buf;
-	time_t		time_reg;
-	struct timeval	tv;
-	int		type, variant;
-
-	if (argc != 2) {
-		fprintf(stderr, "Usage: %s uuid\n", argv[0]);
-		exit(1);
-	}
-	if (uuid_parse(argv[1], buf)) {
-		fprintf(stderr, "Invalid UUID: %s\n", argv[1]);
-		exit(1);
-	}
-	variant = uuid_variant(buf);
-	type = uuid_type(buf);
-	time_reg = uuid_time(buf, &tv);
-
-	printf("UUID variant is %d (%s)\n", variant, variant_string(variant));
-	if (variant != UUID_VARIANT_DCE) {
-		printf("Warning: This program only knows how to interpret "
-		       "DCE UUIDs.\n\tThe rest of the output is likely "
-		       "to be incorrect!!\n");
-	}
-	printf("UUID type is %d", type);
-	switch (type) {
-	case 1:
-		printf(" (time based)\n");
-		break;
-	case 2:
-		printf(" (DCE)\n");
-		break;
-	case 3:
-		printf(" (name-based)\n");
-		break;
-	case 4:
-		printf(" (random)\n");
-		break;
-	default:
-		printf("\n");
-	}
-	if (type != 1) {
-		printf("Warning: not a time-based UUID, so UUID time "
-		       "decoding will likely not work!\n");
-	}
-	printf("UUID time is: (%ld, %ld): %s\n", tv.tv_sec, tv.tv_usec,
-	       ctime(&time_reg));
-
-	return 0;
-}
-#endif
diff --git a/contrib/uuid/uuid_types.h b/contrib/uuid/uuid_types.h
deleted file mode 100644
index 3e2290b4682..00000000000
--- a/contrib/uuid/uuid_types.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* 
- * If linux/types.h is already been included, assume it has defined
- * everything we need.  (cross fingers)  Other header files may have 
- * also defined the types that we need.
- */
-#if (!defined(_STDINT_H) && !defined(_UUID_STDINT_H))
-#define _UUID_STDINT_H
-
-typedef unsigned char uint8_t;
-typedef signed char int8_t;
-
-#if (4 == 8)
-typedef int		int64_t;
-typedef unsigned int	uint64_t;
-#elif (8 == 8)
-typedef long		int64_t;
-typedef unsigned long	uint64_t;
-#elif (8 == 8)
-#if defined(__GNUC__)
-typedef __signed__ long long 	int64_t;
-#else
-typedef signed long long 	int64_t;
-#endif
-typedef unsigned long long	uint64_t;
-#endif
-
-#if (4 == 2)
-typedef	int		int16_t;
-typedef	unsigned int	uint16_t;
-#elif (2 == 2)
-typedef	short		int16_t;
-typedef	unsigned short	uint16_t;
-#else
-  ?==error: undefined 16 bit type
-#endif
-
-#if (4 == 4)
-typedef	int		int32_t;
-typedef	unsigned int	uint32_t;
-#elif (8 == 4)
-typedef	long		int32_t;
-typedef	unsigned long	uint32_t;
-#elif (2 == 4)
-typedef	short		int32_t;
-typedef	unsigned short	uint32_t;
-#else
- ?== error: undefined 32 bit type
-#endif
-
-#endif
diff --git a/contrib/uuid/uuidd.h b/contrib/uuid/uuidd.h
deleted file mode 100644
index c71f4b78835..00000000000
--- a/contrib/uuid/uuidd.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Definitions used by the uuidd daemon
- *
- * Copyright (C) 2007 Theodore Ts'o.
- *
- * %Begin-Header%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- * %End-Header%
- */
-
-#ifndef _UUID_UUIDD_H
-#define _UUID_UUIDD_H
-
-#define UUIDD_DIR		"/var/lib/libuuid"
-#define UUIDD_SOCKET_PATH	UUIDD_DIR "/request"
-#define UUIDD_PIDFILE_PATH	UUIDD_DIR "/uuidd.pid"
-#define UUIDD_PATH		"/usr/sbin/uuidd"
-
-#define UUIDD_OP_GETPID			0
-#define UUIDD_OP_GET_MAXOP		1
-#define UUIDD_OP_TIME_UUID		2
-#define UUIDD_OP_RANDOM_UUID		3
-#define UUIDD_OP_BULK_TIME_UUID		4
-#define UUIDD_OP_BULK_RANDOM_UUID	5
-#define UUIDD_MAX_OP			UUIDD_OP_BULK_RANDOM_UUID
-
-extern void uuid__generate_time(uuid_t out, int *num);
-extern void uuid__generate_random(uuid_t out, int *num);
-
-#endif /* _UUID_UUID_H */
diff --git a/contrib/xxhash/xxhash.c b/contrib/xxhash/xxhash.c
new file mode 100644
index 00000000000..56f80f8811d
--- /dev/null
+++ b/contrib/xxhash/xxhash.c
@@ -0,0 +1,1029 @@
+/*
+*  xxHash - Fast Hash algorithm
+*  Copyright (C) 2012-2016, Yann Collet
+*
+*  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+*  Redistribution and use in source and binary forms, with or without
+*  modification, are permitted provided that the following conditions are
+*  met:
+*
+*  * Redistributions of source code must retain the above copyright
+*  notice, this list of conditions and the following disclaimer.
+*  * Redistributions in binary form must reproduce the above
+*  copyright notice, this list of conditions and the following disclaimer
+*  in the documentation and/or other materials provided with the
+*  distribution.
+*
+*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+*  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+*  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+*  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+*  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+*  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+*  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+*  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+*  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*  You can contact the author at :
+*  - xxHash homepage: http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/* *************************************
+*  Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ *            It can generate buggy code on targets which do not support unaligned memory accesses.
+ *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
+                        || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \
+                        || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define XXH_FORCE_MEMORY_ACCESS 2
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
+                    || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
+                    || defined(__ARM_ARCH_7S__) ))
+#    define XXH_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault.
+ * When this macro is enabled, xxHash actively checks input for null pointer.
+ * It it is, result for null input pointers is the same as a null-length input.
+ */
+#ifndef XXH_ACCEPT_NULL_INPUT_POINTER   /* can be defined externally */
+#  define XXH_ACCEPT_NULL_INPUT_POINTER 0
+#endif
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
+#  define XXH_FORCE_NATIVE_FORMAT 0
+#endif
+
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash;
+ * set it to 0 when the input is guaranteed to be aligned,
+ * or when alignment doesn't matter for performance.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+#  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#    define XXH_FORCE_ALIGN_CHECK 0
+#  else
+#    define XXH_FORCE_ALIGN_CHECK 1
+#  endif
+#endif
+
+
+/* *************************************
+*  Includes & Memory related functions
+***************************************/
+/*! Modify the local functions below should you wish to use some other memory routines
+*   for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#include <assert.h>   /* assert */
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/* *************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/* *************************************
+*  Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+# else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+# endif
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* *************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+static int XXH_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
+#endif
+
+
+/* ***************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+static U32 XXH_readBE32(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
+
+/* *************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)  { enum { XXH_sa = 1/(int)(!!(c)) }; }  /* use after variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+*  32-bit hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 =  668265263U;
+static const U32 PRIME32_5 =  374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+    seed += input * PRIME32_2;
+    seed  = XXH_rotl32(seed, 13);
+    seed *= PRIME32_1;
+    return seed;
+}
+
+/* mix all bits */
+static U32 XXH32_avalanche(U32 h32)
+{
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+    return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+static U32
+XXH32_finalize(U32 h32, const void* ptr, size_t len,
+                XXH_endianess endian, XXH_alignment align)
+
+{
+    const BYTE* p = (const BYTE*)ptr;
+#define PROCESS1             \
+    h32 += (*p) * PRIME32_5; \
+    p++;                     \
+    h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+
+#define PROCESS4                         \
+    h32 += XXH_get32bits(p) * PRIME32_3; \
+    p+=4;                                \
+    h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+
+    switch(len&15)  /* or switch(bEnd - p) */
+    {
+      case 12:      PROCESS4;
+                    /* fallthrough */
+      case 8:       PROCESS4;
+                    /* fallthrough */
+      case 4:       PROCESS4;
+                    return XXH32_avalanche(h32);
+
+      case 13:      PROCESS4;
+                    /* fallthrough */
+      case 9:       PROCESS4;
+                    /* fallthrough */
+      case 5:       PROCESS4;
+                    PROCESS1;
+                    return XXH32_avalanche(h32);
+
+      case 14:      PROCESS4;
+                    /* fallthrough */
+      case 10:      PROCESS4;
+                    /* fallthrough */
+      case 6:       PROCESS4;
+                    PROCESS1;
+                    PROCESS1;
+                    return XXH32_avalanche(h32);
+
+      case 15:      PROCESS4;
+                    /* fallthrough */
+      case 11:      PROCESS4;
+                    /* fallthrough */
+      case 7:       PROCESS4;
+                    /* fallthrough */
+      case 3:       PROCESS1;
+                    /* fallthrough */
+      case 2:       PROCESS1;
+                    /* fallthrough */
+      case 1:       PROCESS1;
+                    /* fallthrough */
+      case 0:       return XXH32_avalanche(h32);
+    }
+    assert(0);
+    return h32;   /* reaching this point is deemed impossible */
+}
+
+
+FORCE_INLINE U32
+XXH32_endian_align(const void* input, size_t len, U32 seed,
+                    XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16) {
+        const BYTE* const limit = bEnd - 15;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do {
+            v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+            v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+            v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+            v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+        } while (p < limit);
+
+        h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
+            + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    } else {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32)len;
+
+    return XXH32_finalize(h32, p, len&15, endian, align);
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+
+/*======   Hash streaming   ======*/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
+{
+    XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + PRIME32_1 + PRIME32_2;
+    state.v2 = seed + PRIME32_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME32_1;
+    /* do not write into reserved, planned to be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+    return XXH_OK;
+}
+
+
+FORCE_INLINE
+XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    state->total_len_32 += (unsigned)len;
+    state->large_len |= (len>=16) | (state->total_len_32>=16);
+
+    if (state->memsize + len < 16)  {   /* fill in tmp buffer */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (unsigned)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* some data left from previous update */
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {   const U32* p32 = state->mem32;
+            state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+            state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+            state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+            state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian));
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16) {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do {
+            v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+            v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+            v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+            v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+FORCE_INLINE U32
+XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+{
+    U32 h32;
+
+    if (state->large_len) {
+        h32 = XXH_rotl32(state->v1, 1)
+            + XXH_rotl32(state->v2, 7)
+            + XXH_rotl32(state->v3, 12)
+            + XXH_rotl32(state->v4, 18);
+    } else {
+        h32 = state->v3 /* == seed */ + PRIME32_5;
+    }
+
+    h32 += state->total_len_32;
+
+    return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned);
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*======   Canonical representation   ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+*   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+*   These functions allow transformation of hash result into and from its canonical format.
+*   This way, hash values can be written into a file or buffer, remaining comparable across different systems.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+    return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+*  64-bit hash functions
+*********************************************************************/
+
+/*======   Memory access   ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+    typedef uint64_t U64;
+# else
+    /* if compiler doesn't support unsigned long long, replace by another 64-bit type */
+    typedef unsigned long long U64;
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val;
+    memcpy(&val, memPtr, sizeof(val));
+    return val;
+}
+
+#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap64 _byteswap_uint64
+#elif GCC_VERSION >= 403
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*======   xxh64   ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 =  1609587929392839161ULL;
+static const U64 PRIME64_4 =  9650029242287828579ULL;
+static const U64 PRIME64_5 =  2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+    acc += input * PRIME64_2;
+    acc  = XXH_rotl64(acc, 31);
+    acc *= PRIME64_1;
+    return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+    val  = XXH64_round(0, val);
+    acc ^= val;
+    acc  = acc * PRIME64_1 + PRIME64_4;
+    return acc;
+}
+
+static U64 XXH64_avalanche(U64 h64)
+{
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+    return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+static U64
+XXH64_finalize(U64 h64, const void* ptr, size_t len,
+               XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)ptr;
+
+#define PROCESS1_64          \
+    h64 ^= (*p) * PRIME64_5; \
+    p++;                     \
+    h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+
+#define PROCESS4_64          \
+    h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \
+    p+=4;                    \
+    h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+
+#define PROCESS8_64 {        \
+    U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \
+    p+=8;                    \
+    h64 ^= k1;               \
+    h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \
+}
+
+    switch(len&31) {
+      case 24: PROCESS8_64;
+                    /* fallthrough */
+      case 16: PROCESS8_64;
+                    /* fallthrough */
+      case  8: PROCESS8_64;
+               return XXH64_avalanche(h64);
+
+      case 28: PROCESS8_64;
+                    /* fallthrough */
+      case 20: PROCESS8_64;
+                    /* fallthrough */
+      case 12: PROCESS8_64;
+                    /* fallthrough */
+      case  4: PROCESS4_64;
+               return XXH64_avalanche(h64);
+
+      case 25: PROCESS8_64;
+                    /* fallthrough */
+      case 17: PROCESS8_64;
+                    /* fallthrough */
+      case  9: PROCESS8_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 29: PROCESS8_64;
+                    /* fallthrough */
+      case 21: PROCESS8_64;
+                    /* fallthrough */
+      case 13: PROCESS8_64;
+                    /* fallthrough */
+      case  5: PROCESS4_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 26: PROCESS8_64;
+                    /* fallthrough */
+      case 18: PROCESS8_64;
+                    /* fallthrough */
+      case 10: PROCESS8_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 30: PROCESS8_64;
+                    /* fallthrough */
+      case 22: PROCESS8_64;
+                    /* fallthrough */
+      case 14: PROCESS8_64;
+                    /* fallthrough */
+      case  6: PROCESS4_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 27: PROCESS8_64;
+                    /* fallthrough */
+      case 19: PROCESS8_64;
+                    /* fallthrough */
+      case 11: PROCESS8_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               PROCESS1_64;
+               return XXH64_avalanche(h64);
+
+      case 31: PROCESS8_64;
+                    /* fallthrough */
+      case 23: PROCESS8_64;
+                    /* fallthrough */
+      case 15: PROCESS8_64;
+                    /* fallthrough */
+      case  7: PROCESS4_64;
+                    /* fallthrough */
+      case  3: PROCESS1_64;
+                    /* fallthrough */
+      case  2: PROCESS1_64;
+                    /* fallthrough */
+      case  1: PROCESS1_64;
+                    /* fallthrough */
+      case  0: return XXH64_avalanche(h64);
+    }
+
+    /* impossible to reach */
+    assert(0);
+    return 0;  /* unreachable, but some compilers complain without it */
+}
+
+FORCE_INLINE U64
+XXH64_endian_align(const void* input, size_t len, U64 seed,
+                XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+    if (p==NULL) {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do {
+            v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+            v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+            v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+            v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+        } while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+
+    } else {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    return XXH64_finalize(h64, p, len, endian, align);
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if (XXH_FORCE_ALIGN_CHECK) {
+        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
+            if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+                return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+            else
+                return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }   }
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*======   Hash Streaming   ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+    memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+    XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+    memset(&state, 0, sizeof(state));
+    state.v1 = seed + PRIME64_1 + PRIME64_2;
+    state.v2 = seed + PRIME64_2;
+    state.v3 = seed + 0;
+    state.v4 = seed - PRIME64_1;
+     /* do not write into reserved, planned to be removed in a future version */
+    memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+    return XXH_OK;
+}
+
+FORCE_INLINE
+XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+    if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+        return XXH_OK;
+#else
+        return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32) {  /* fill in tmp buffer */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize) {   /* tmp buffer is full */
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+        state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+        state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+        state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd) {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do {
+            v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+            v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+            v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+            v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+        } while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd) {
+        XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+        state->memsize = (unsigned)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+{
+    U64 h64;
+
+    if (state->total_len >= 32) {
+        U64 const v1 = state->v1;
+        U64 const v2 = state->v2;
+        U64 const v3 = state->v3;
+        U64 const v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+        h64 = XXH64_mergeRound(h64, v1);
+        h64 = XXH64_mergeRound(h64, v2);
+        h64 = XXH64_mergeRound(h64, v3);
+        h64 = XXH64_mergeRound(h64, v4);
+    } else {
+        h64  = state->v3 /*seed*/ + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned);
+}
+
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation   ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+    memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+    return XXH_readBE64(src);
+}
+
+#endif  /* XXH_NO_LONG_LONG */
diff --git a/contrib/xxhash/xxhash.h b/contrib/xxhash/xxhash.h
new file mode 100644
index 00000000000..d6bad943358
--- /dev/null
+++ b/contrib/xxhash/xxhash.h
@@ -0,0 +1,328 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2016, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bit version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bit applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************
+*  Definitions
+******************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/* ****************************
+ *  API modifier
+ ******************************/
+/** XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ *  This is useful to include xxhash functions in `static` mode
+ *  in order to inline them, and remove their symbol from the public list.
+ *  Inlining can offer dramatic performance improvement on small keys.
+ *  Methodology :
+ *     #define XXH_INLINE_ALL
+ *     #include "xxhash.h"
+ * `xxhash.c` is automatically included.
+ *  It's not useful to compile and link it as a separate module.
+ */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  ifndef XXH_STATIC_LINKING_ONLY
+#    define XXH_STATIC_LINKING_ONLY
+#  endif
+#  if defined(__GNUC__)
+#    define XXH_PUBLIC_API static __inline __attribute__((unused))
+#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#    define XXH_PUBLIC_API static inline
+#  elif defined(_MSC_VER)
+#    define XXH_PUBLIC_API static __inline
+#  else
+     /* this version may generate warnings for unused static functions */
+#    define XXH_PUBLIC_API static
+#  endif
+#else
+#  define XXH_PUBLIC_API   /* do nothing */
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+/*! XXH_NAMESPACE, aka Namespace Emulation :
+ *
+ * If you want to include _and expose_ xxHash functions from within your own library,
+ * but also want to avoid symbol collisions with other libraries which may also include xxHash,
+ *
+ * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+ * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+ *
+ * Note that no change is required within the calling program as long as it includes `xxhash.h` :
+ * regular symbol name will be automatically translated by this header.
+ */
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
+
+
+/* *************************************
+*  Version
+***************************************/
+#define XXH_VERSION_MAJOR    0
+#define XXH_VERSION_MINOR    6
+#define XXH_VERSION_RELEASE  5
+#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/*-**********************************************************************
+*  32-bit hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
+    Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
+
+/*
+ * Streaming functions generate the xxHash of an input provided in multiple segments.
+ * Note that, for small input, they are slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * XXH state must first be allocated, using XXH*_createState() .
+ *
+ * Start a new hash by initializing state with a seed, using XXH*_reset().
+ *
+ * Then, feed the hash state by calling XXH*_update() as many times as necessary.
+ * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using XXH*_digest().
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a digest,
+ * and generate some new hashes later on, by calling again XXH*_digest().
+ *
+ * When done, free XXH state space if it was allocated dynamically.
+ */
+
+/*======   Canonical representation   ======*/
+
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+ * The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+ * These functions allow transformation of hash result into and from its canonical format.
+ * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+ */
+
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+*  64-bit hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
+
+/*! XXH64() :
+    Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".
+    "seed" can be used to alter the result predictably.
+    This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).
+*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*======   Streaming   ======*/
+typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
+
+/*======   Canonical representation   ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif  /* XXH_NO_LONG_LONG */
+
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+   This section contains declarations which are not guaranteed to remain stable.
+   They may change in future versions, becoming incompatible with a different version of the library.
+   These declarations should only be used with static linking.
+   Never use them in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only present to allow
+ * static allocation of XXH state, on stack or in a struct for example.
+ * Never **ever** use members directly. */
+
+#if !defined (__VMS) \
+  && (defined (__cplusplus) \
+  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#   include <stdint.h>
+
+struct XXH32_state_s {
+   uint32_t total_len_32;
+   uint32_t large_len;
+   uint32_t v1;
+   uint32_t v2;
+   uint32_t v3;
+   uint32_t v4;
+   uint32_t mem32[4];
+   uint32_t memsize;
+   uint32_t reserved;   /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+struct XXH64_state_s {
+   uint64_t total_len;
+   uint64_t v1;
+   uint64_t v2;
+   uint64_t v3;
+   uint64_t v4;
+   uint64_t mem64[4];
+   uint32_t memsize;
+   uint32_t reserved[2];          /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+
+# else
+
+struct XXH32_state_s {
+   unsigned total_len_32;
+   unsigned large_len;
+   unsigned v1;
+   unsigned v2;
+   unsigned v3;
+   unsigned v4;
+   unsigned mem32[4];
+   unsigned memsize;
+   unsigned reserved;   /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH32_state_t */
+
+#   ifndef XXH_NO_LONG_LONG  /* remove 64-bit support */
+struct XXH64_state_s {
+   unsigned long long total_len;
+   unsigned long long v1;
+   unsigned long long v2;
+   unsigned long long v3;
+   unsigned long long v4;
+   unsigned long long mem64[4];
+   unsigned memsize;
+   unsigned reserved[2];     /* never read nor write, might be removed in a future version */
+};   /* typedef'd to XXH64_state_t */
+#    endif
+
+# endif
+
+
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
+#endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* XXHASH_H_5627135585666179 */
diff --git a/contrib/xxhash/xxhsum.c b/contrib/xxhash/xxhsum.c
new file mode 100644
index 00000000000..69931f727f0
--- /dev/null
+++ b/contrib/xxhash/xxhsum.c
@@ -0,0 +1,1301 @@
+/*
+*  xxhsum - Command line interface for xxhash algorithms
+*  Copyright (C) Yann Collet 2012-2016
+*
+*  GPL v2 License
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License as published by
+*  the Free Software Foundation; either version 2 of the License, or
+*  (at your option) any later version.
+*
+*  This program is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU General Public License along
+*  with this program; if not, write to the Free Software Foundation, Inc.,
+*  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*
+*  You can contact the author at :
+*  - xxHash homepage : http://www.xxhash.com
+*  - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* xxhsum :
+ * Provides hash value of a file content, or a list of files, or stdin
+ * Display convention is Big Endian, for both 32 and 64 bits algorithms
+ */
+
+#ifndef XXHASH_C_2097394837
+#define XXHASH_C_2097394837
+
+/* ************************************
+ *  Compiler Options
+ **************************************/
+/* MS Visual */
+#if defined(_MSC_VER) || defined(_WIN32)
+#  define _CRT_SECURE_NO_WARNINGS   /* removes visual warnings */
+#endif
+
+/* Under Linux at least, pull in the *64 commands */
+#ifndef _LARGEFILE64_SOURCE
+#  define _LARGEFILE64_SOURCE
+#endif
+
+
+/* ************************************
+ *  Includes
+ **************************************/
+#include <stdlib.h>     /* malloc, calloc, free, exit */
+#include <stdio.h>      /* fprintf, fopen, ftello64, fread, stdin, stdout, _fileno (when present) */
+#include <string.h>     /* strcmp */
+#include <sys/types.h>  /* stat, stat64, _stat64 */
+#include <sys/stat.h>   /* stat, stat64, _stat64 */
+#include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>     /* assert */
+
+#define XXH_STATIC_LINKING_ONLY   /* *_state_t */
+#include "xxhash.h"
+
+
+/* ************************************
+ *  OS-Specific Includes
+ **************************************/
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>    /* _O_BINARY */
+#  include <io.h>       /* _setmode, _isatty */
+#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#  include <unistd.h>   /* isatty, STDIN_FILENO */
+#  define SET_BINARY_MODE(file)
+#  define IS_CONSOLE(stdStream) isatty(STDIN_FILENO)
+#endif
+
+#if !defined(S_ISREG)
+#  define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
+#endif
+
+
+/* ************************************
+*  Basic Types
+**************************************/
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#   include <stdint.h>
+    typedef uint8_t  BYTE;
+    typedef uint16_t U16;
+    typedef uint32_t U32;
+    typedef  int32_t S32;
+    typedef uint64_t U64;
+#  else
+    typedef unsigned char      BYTE;
+    typedef unsigned short     U16;
+    typedef unsigned int       U32;
+    typedef   signed int       S32;
+    typedef unsigned long long U64;
+#  endif
+#endif
+
+static unsigned BMK_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+
+
+/* *************************************
+ *  Constants
+ ***************************************/
+#define LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE
+#define QUOTE(str) #str
+#define EXPAND_AND_QUOTE(str) QUOTE(str)
+#define PROGRAM_VERSION EXPAND_AND_QUOTE(LIB_VERSION)
+static const int g_nbBits = (int)(sizeof(void*)*8);
+static const char g_lename[] = "little endian";
+static const char g_bename[] = "big endian";
+#define ENDIAN_NAME (BMK_isLittleEndian() ? g_lename : g_bename)
+static const char author[] = "Yann Collet";
+#define WELCOME_MESSAGE(exename) "%s %s (%i-bits %s), by %s \n", \
+                    exename, PROGRAM_VERSION, g_nbBits, ENDIAN_NAME, author
+
+#define KB *( 1<<10)
+#define MB *( 1<<20)
+#define GB *(1U<<30)
+
+static size_t XXH_DEFAULT_SAMPLE_SIZE = 100 KB;
+#define NBLOOPS    3                              /* Default number of benchmark iterations */
+#define TIMELOOP_S 1
+#define TIMELOOP  (TIMELOOP_S * CLOCKS_PER_SEC)   /* Minimum timing per iteration */
+#define XXHSUM32_DEFAULT_SEED 0                   /* Default seed for algo_xxh32 */
+#define XXHSUM64_DEFAULT_SEED 0                   /* Default seed for algo_xxh64 */
+
+#define MAX_MEM    (2 GB - 64 MB)
+
+static const char stdinName[] = "-";
+typedef enum { algo_xxh32, algo_xxh64 } algoType;
+static const algoType g_defaultAlgo = algo_xxh64;    /* required within main() & usage() */
+
+/* <16 hex char> <SPC> <SPC> <filename> <'\0'>
+ * '4096' is typical Linux PATH_MAX configuration. */
+#define DEFAULT_LINE_LENGTH (sizeof(XXH64_hash_t) * 2 + 2 + 4096 + 1)
+
+/* Maximum acceptable line length. */
+#define MAX_LINE_LENGTH (32 KB)
+
+
+/* ************************************
+ *  Display macros
+ **************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYRESULT(...)   fprintf(stdout, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) do { if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); } while (0)
+static int g_displayLevel = 2;
+
+
+/* ************************************
+ *  Local variables
+ **************************************/
+static U32 g_nbIterations = NBLOOPS;
+
+
+/* ************************************
+ *  Benchmark Functions
+ **************************************/
+static clock_t BMK_clockSpan( clock_t start )
+{
+    return clock() - start;   /* works even if overflow; Typical max span ~ 30 mn */
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+    size_t const step = 64 MB;
+    void* testmem = NULL;
+
+    requiredMem = (((requiredMem >> 26) + 1) << 26);
+    requiredMem += 2*step;
+    if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+    while (!testmem) {
+        if (requiredMem > step) requiredMem -= step;
+        else requiredMem >>= 1;
+        testmem = malloc ((size_t)requiredMem);
+    }
+    free (testmem);
+
+    /* keep some space available */
+    if (requiredMem > step) requiredMem -= step;
+    else requiredMem >>= 1;
+
+    return (size_t)requiredMem;
+}
+
+
+static U64 BMK_GetFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+#endif
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    return (U64)statbuf.st_size;
+}
+
+typedef U32 (*hashFunction)(const void* buffer, size_t bufferSize, U32 seed);
+
+static U32 localXXH32(const void* buffer, size_t bufferSize, U32 seed) { return XXH32(buffer, bufferSize, seed); }
+
+static U32 localXXH64(const void* buffer, size_t bufferSize, U32 seed) { return (U32)XXH64(buffer, bufferSize, seed); }
+
+static void BMK_benchHash(hashFunction h, const char* hName, const void* buffer, size_t bufferSize)
+{
+    U32 nbh_perIteration = ((300 MB) / (bufferSize+1)) + 1;  /* first loop conservatively aims for 300 MB/s */
+    U32 iterationNb;
+    double fastestH = 100000000.;
+
+    DISPLAYLEVEL(2, "\r%70s\r", "");       /* Clean display line */
+    if (g_nbIterations<1) g_nbIterations=1;
+    for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) {
+        U32 r=0;
+        clock_t cStart;
+
+        DISPLAYLEVEL(2, "%1i-%-17.17s : %10u ->\r", iterationNb, hName, (U32)bufferSize);
+        cStart = clock();
+        while (clock() == cStart);   /* starts clock() at its exact beginning */
+        cStart = clock();
+
+        {   U32 i;
+            for (i=0; i<nbh_perIteration; i++)
+                r += h(buffer, bufferSize, i);
+        }
+        if (r==0) DISPLAYLEVEL(3,".\r");  /* do something with r to avoid compiler "optimizing" away hash function */
+        {   double const timeS = ((double)BMK_clockSpan(cStart) / CLOCKS_PER_SEC) / nbh_perIteration;
+            if (timeS < fastestH) fastestH = timeS;
+            DISPLAYLEVEL(2, "%1i-%-17.17s : %10u -> %8.0f it/s (%7.1f MB/s) \r",
+                    iterationNb, hName, (U32)bufferSize,
+                    (double)1 / fastestH,
+                    ((double)bufferSize / (1<<20)) / fastestH );
+        }
+        assert(fastestH > 1./2000000000);  /* avoid U32 overflow */
+        nbh_perIteration = (U32)(1 / fastestH) + 1;  /* adjust nbh_perIteration to last roughtly one second */
+    }
+    DISPLAYLEVEL(1, "%-19.19s : %10u -> %8.0f it/s (%7.1f MB/s) \n", hName, (U32)bufferSize,
+        (double)1 / fastestH,
+        ((double)bufferSize / (1<<20)) / fastestH);
+    if (g_displayLevel<1)
+        DISPLAYLEVEL(0, "%u, ", (U32)((double)1 / fastestH));
+}
+
+
+/* BMK_benchMem():
+ * specificTest : 0 == run all tests, 1+ run only specific test
+ * buffer : is supposed 8-bytes aligned (if malloc'ed, it should be)
+ * the real allocated size of buffer is supposed to be >= (bufferSize+3).
+ * @return : 0 on success, 1 if error (invalid mode selected) */
+static int BMK_benchMem(const void* buffer, size_t bufferSize, U32 specificTest)
+{
+    assert((((size_t)buffer) & 8) == 0);  /* ensure alignment */
+
+    /* XXH32 bench */
+    if ((specificTest==0) | (specificTest==1))
+        BMK_benchHash(localXXH32, "XXH32", buffer, bufferSize);
+
+    /* Bench XXH32 on Unaligned input */
+    if ((specificTest==0) | (specificTest==2))
+        BMK_benchHash(localXXH32, "XXH32 unaligned", ((const char*)buffer)+1, bufferSize);
+
+    /* Bench XXH64 */
+    if ((specificTest==0) | (specificTest==3))
+        BMK_benchHash(localXXH64, "XXH64", buffer, bufferSize);
+
+    /* Bench XXH64 on Unaligned input */
+    if ((specificTest==0) | (specificTest==4))
+        BMK_benchHash(localXXH64, "XXH64 unaligned", ((const char*)buffer)+3, bufferSize);
+
+    if (specificTest > 4) {
+        DISPLAY("benchmark mode invalid \n");
+        return 1;
+    }
+    return 0;
+}
+
+
+static size_t BMK_selectBenchedSize(const char* fileName)
+{   U64 const inFileSize = BMK_GetFileSize(fileName);
+    size_t benchedSize = (size_t) BMK_findMaxMem(inFileSize);
+    if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+    if (benchedSize < inFileSize) {
+        DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20));
+    }
+    return benchedSize;
+}
+
+
+static int BMK_benchFiles(const char** fileNamesTable, int nbFiles, U32 specificTest)
+{
+    int result = 0;
+    int fileIdx;
+
+    for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
+        const char* const inFileName = fileNamesTable[fileIdx];
+        FILE* const inFile = fopen( inFileName, "rb" );
+        size_t const benchedSize = BMK_selectBenchedSize(inFileName);
+        char* const buffer = (char*)calloc(benchedSize+16+3, 1);
+        void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF);  /* align on next 16 bytes */
+
+        /* Checks */
+        if ((inFile==NULL) || (inFileName==NULL)) {
+            DISPLAY("Pb opening %s\n", inFileName);
+            free(buffer);
+            return 11;
+        }
+        if(!buffer) {
+            DISPLAY("\nError: not enough memory!\n");
+            fclose(inFile);
+            return 12;
+        }
+
+        /* Fill input buffer */
+        DISPLAYLEVEL(1, "\rLoading %s...        \n", inFileName);
+        {   size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile);
+            fclose(inFile);
+            if(readSize != benchedSize) {
+                DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
+                free(buffer);
+                return 13;
+        }   }
+
+        /* bench */
+        result |= BMK_benchMem(alignedBuffer, benchedSize, specificTest);
+
+        free(buffer);
+    }
+
+    return result;
+}
+
+
+
+static int BMK_benchInternal(size_t keySize, int specificTest)
+{
+    void* const buffer = calloc(keySize+16+3, 1);
+    void* const alignedBuffer = ((char*)buffer+15) - (((size_t)((char*)buffer+15)) & 0xF);  /* align on next 16 bytes */
+    if(!buffer) {
+        DISPLAY("\nError: not enough memory!\n");
+        return 12;
+    }
+
+    /* bench */
+    DISPLAYLEVEL(1, "Sample of ");
+    if (keySize > 10 KB) {
+        DISPLAYLEVEL(1, "%u KB", (U32)(keySize >> 10));
+    } else {
+        DISPLAYLEVEL(1, "%u bytes", (U32)keySize);
+    }
+    DISPLAYLEVEL(1, "...        \n");
+
+    {   int const result = BMK_benchMem(alignedBuffer, keySize, specificTest);
+        free(buffer);
+        return result;
+    }
+}
+
+
+static void BMK_checkResult(U32 r1, U32 r2)
+{
+    static int nbTests = 1;
+    if (r1==r2) {
+        DISPLAYLEVEL(3, "\rTest%3i : %08X == %08X   ok   ", nbTests, r1, r2);
+    } else {
+        DISPLAY("\rERROR : Test%3i : %08X <> %08X   !!!!!   \n", nbTests, r1, r2);
+        exit(1);
+    }
+    nbTests++;
+}
+
+
+static void BMK_checkResult64(U64 r1, U64 r2)
+{
+    static int nbTests = 1;
+    if (r1!=r2) {
+        DISPLAY("\rERROR : Test%3i : 64-bit values non equals   !!!!!   \n", nbTests);
+        DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2);
+        exit(1);
+    }
+    nbTests++;
+}
+
+
+static void BMK_testSequence64(void* sentence, size_t len, U64 seed, U64 Nresult)
+{
+    XXH64_state_t state;
+    U64 Dresult;
+    size_t pos;
+
+    Dresult = XXH64(sentence, len, seed);
+    BMK_checkResult64(Dresult, Nresult);
+
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, sentence, len);
+    Dresult = XXH64_digest(&state);
+    BMK_checkResult64(Dresult, Nresult);
+
+    XXH64_reset(&state, seed);
+    for (pos=0; pos<len; pos++)
+        XXH64_update(&state, ((char*)sentence)+pos, 1);
+    Dresult = XXH64_digest(&state);
+    BMK_checkResult64(Dresult, Nresult);
+}
+
+
+static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult)
+{
+    XXH32_state_t state;
+    U32 Dresult;
+    size_t pos;
+
+    Dresult = XXH32(sequence, len, seed);
+    BMK_checkResult(Dresult, Nresult);
+
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, sequence, len);
+    Dresult = XXH32_digest(&state);
+    BMK_checkResult(Dresult, Nresult);
+
+    XXH32_reset(&state, seed);
+    for (pos=0; pos<len; pos++)
+        XXH32_update(&state, ((const char*)sequence)+pos, 1);
+    Dresult = XXH32_digest(&state);
+    BMK_checkResult(Dresult, Nresult);
+}
+
+
+#define SANITY_BUFFER_SIZE 101
+static void BMK_sanityCheck(void)
+{
+    static const U32 prime = 2654435761U;
+    BYTE sanityBuffer[SANITY_BUFFER_SIZE];
+    U32 byteGen = prime;
+
+    int i;
+    for (i=0; i<SANITY_BUFFER_SIZE; i++) {
+        sanityBuffer[i] = (BYTE)(byteGen>>24);
+        byteGen *= byteGen;
+    }
+
+    BMK_testSequence(NULL,          0, 0,     0x02CC5D05);
+    BMK_testSequence(NULL,          0, prime, 0x36B78AE7);
+    BMK_testSequence(sanityBuffer,  1, 0,     0xB85CBEE5);
+    BMK_testSequence(sanityBuffer,  1, prime, 0xD5845D64);
+    BMK_testSequence(sanityBuffer, 14, 0,     0xE5AA0AB4);
+    BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D);
+    BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0,     0x1F1AA412);
+    BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2);
+
+    BMK_testSequence64(NULL        ,  0, 0,     0xEF46DB3751D8E999ULL);
+    BMK_testSequence64(NULL        ,  0, prime, 0xAC75FDA2929B17EFULL);
+    BMK_testSequence64(sanityBuffer,  1, 0,     0x4FCE394CC88952D8ULL);
+    BMK_testSequence64(sanityBuffer,  1, prime, 0x739840CB819FA723ULL);
+    BMK_testSequence64(sanityBuffer, 14, 0,     0xCFFA8DB881BC3A3DULL);
+    BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL);
+    BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0,     0x0EAB543384F878ADULL);
+    BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL);
+
+    DISPLAYLEVEL(3, "\r%70s\r", "");       /* Clean display line */
+    DISPLAYLEVEL(3, "Sanity check -- all tests ok\n");
+}
+
+
+/* ********************************************************
+*  File Hashing
+**********************************************************/
+
+static void BMK_display_LittleEndian(const void* ptr, size_t length)
+{
+    const BYTE* p = (const BYTE*)ptr;
+    size_t idx;
+    for (idx=length-1; idx<length; idx--)    /* intentional underflow to negative to detect end */
+        DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_display_BigEndian(const void* ptr, size_t length)
+{
+    const BYTE* p = (const BYTE*)ptr;
+    size_t idx;
+    for (idx=0; idx<length; idx++)
+        DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* inFile, void* buffer, size_t blockSize)
+{
+    XXH64_state_t state64;
+    XXH32_state_t state32;
+    size_t readSize;
+
+    /* Init */
+    XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED);
+    XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED);
+
+    /* Load file & update hash */
+    readSize = 1;
+    while (readSize) {
+        readSize = fread(buffer, 1, blockSize, inFile);
+        switch(hashType)
+        {
+        case algo_xxh32:
+            XXH32_update(&state32, buffer, readSize);
+            break;
+        case algo_xxh64:
+            XXH64_update(&state64, buffer, readSize);
+            break;
+        default:
+            break;
+        }
+    }
+
+    switch(hashType)
+    {
+    case algo_xxh32:
+        {   U32 const h32 = XXH32_digest(&state32);
+            memcpy(xxhHashValue, &h32, sizeof(h32));
+            break;
+        }
+    case algo_xxh64:
+        {   U64 const h64 = XXH64_digest(&state64);
+            memcpy(xxhHashValue, &h64, sizeof(h64));
+            break;
+        }
+    default:
+            break;
+    }
+}
+
+
+typedef enum { big_endian, little_endian} endianess;
+
+static int BMK_hash(const char* fileName,
+                    const algoType hashType,
+                    const endianess displayEndianess)
+{
+    FILE*  inFile;
+    size_t const blockSize = 64 KB;
+    void*  buffer;
+    U32    h32 = 0;
+    U64    h64 = 0;
+
+    /* Check file existence */
+    if (fileName == stdinName) {
+        inFile = stdin;
+        SET_BINARY_MODE(stdin);
+    }
+    else
+        inFile = fopen( fileName, "rb" );
+    if (inFile==NULL) {
+        DISPLAY( "Pb opening %s\n", fileName);
+        return 1;
+    }
+
+    /* Memory allocation & restrictions */
+    buffer = malloc(blockSize);
+    if(!buffer) {
+        DISPLAY("\nError: not enough memory!\n");
+        fclose(inFile);
+        return 1;
+    }
+
+    /* loading notification */
+    {   const size_t fileNameSize = strlen(fileName);
+        const char* const fileNameEnd = fileName + fileNameSize;
+        const int maxInfoFilenameSize = (int)(fileNameSize > 30 ? 30 : fileNameSize);
+        int infoFilenameSize = 1;
+        while ((infoFilenameSize < maxInfoFilenameSize)
+            && (fileNameEnd[-1-infoFilenameSize] != '/')
+            && (fileNameEnd[-1-infoFilenameSize] != '\\') )
+              infoFilenameSize++;
+        DISPLAY("\rLoading %s...  \r", fileNameEnd - infoFilenameSize);
+
+        /* Load file & update hash */
+        switch(hashType)
+        {
+        case algo_xxh32:
+            BMK_hashStream(&h32, algo_xxh32, inFile, buffer, blockSize);
+            break;
+        case algo_xxh64:
+            BMK_hashStream(&h64, algo_xxh64, inFile, buffer, blockSize);
+            break;
+        default:
+            break;
+        }
+
+        fclose(inFile);
+        free(buffer);
+        DISPLAY("%s             \r", fileNameEnd - infoFilenameSize);  /* erase line */
+    }
+
+    /* display Hash */
+    switch(hashType)
+    {
+    case algo_xxh32:
+        {   XXH32_canonical_t hcbe32;
+            XXH32_canonicalFromHash(&hcbe32, h32);
+            displayEndianess==big_endian ?
+                BMK_display_BigEndian(&hcbe32, sizeof(hcbe32)) : BMK_display_LittleEndian(&hcbe32, sizeof(hcbe32));
+            DISPLAYRESULT("  %s\n", fileName);
+            break;
+        }
+    case algo_xxh64:
+        {   XXH64_canonical_t hcbe64;
+            XXH64_canonicalFromHash(&hcbe64, h64);
+            displayEndianess==big_endian ?
+                BMK_display_BigEndian(&hcbe64, sizeof(hcbe64)) : BMK_display_LittleEndian(&hcbe64, sizeof(hcbe64));
+            DISPLAYRESULT("  %s\n", fileName);
+            break;
+        }
+    default:
+            break;
+    }
+
+    return 0;
+}
+
+
+static int BMK_hashFiles(const char** fnList, int fnTotal,
+                         algoType hashType, endianess displayEndianess)
+{
+    int fnNb;
+    int result = 0;
+
+    if (fnTotal==0)
+        return BMK_hash(stdinName, hashType, displayEndianess);
+
+    for (fnNb=0; fnNb<fnTotal; fnNb++)
+        result += BMK_hash(fnList[fnNb], hashType, displayEndianess);
+    DISPLAY("\r%70s\r", "");
+    return result;
+}
+
+
+typedef enum {
+    GetLine_ok,
+    GetLine_eof,
+    GetLine_exceedMaxLineLength,
+    GetLine_outOfMemory,
+} GetLineResult;
+
+typedef enum {
+    CanonicalFromString_ok,
+    CanonicalFromString_invalidFormat,
+} CanonicalFromStringResult;
+
+typedef enum {
+    ParseLine_ok,
+    ParseLine_invalidFormat,
+} ParseLineResult;
+
+typedef enum {
+    LineStatus_hashOk,
+    LineStatus_hashFailed,
+    LineStatus_failedToOpen,
+} LineStatus;
+
+typedef union {
+    XXH32_canonical_t xxh32;
+    XXH64_canonical_t xxh64;
+} Canonical;
+
+typedef struct {
+    Canonical   canonical;
+    const char* filename;
+    int         xxhBits;    /* canonical type : 32:xxh32, 64:xxh64 */
+} ParsedLine;
+
+typedef struct {
+    unsigned long   nProperlyFormattedLines;
+    unsigned long   nImproperlyFormattedLines;
+    unsigned long   nMismatchedChecksums;
+    unsigned long   nOpenOrReadFailures;
+    unsigned long   nMixedFormatLines;
+    int             xxhBits;
+    int             quit;
+} ParseFileReport;
+
+typedef struct {
+    const char*     inFileName;
+    FILE*           inFile;
+    int             lineMax;
+    char*           lineBuf;
+    size_t          blockSize;
+    char*           blockBuf;
+    int             strictMode;
+    int             statusOnly;
+    int             warn;
+    int             quiet;
+    ParseFileReport report;
+} ParseFileArg;
+
+
+/*  Read line from stream.
+    Returns GetLine_ok, if it reads line successfully.
+    Returns GetLine_eof, if stream reaches EOF.
+    Returns GetLine_exceedMaxLineLength, if line length is longer than MAX_LINE_LENGTH.
+    Returns GetLine_outOfMemory, if line buffer memory allocation failed.
+ */
+static GetLineResult getLine(char** lineBuf, int* lineMax, FILE* inFile)
+{
+    GetLineResult result = GetLine_ok;
+    int len = 0;
+
+    if ((*lineBuf == NULL) || (*lineMax<1)) {
+        free(*lineBuf);  /* in case it's != NULL */
+        *lineMax = 0;
+        *lineBuf = (char*)malloc(DEFAULT_LINE_LENGTH);
+        if(*lineBuf == NULL) return GetLine_outOfMemory;
+        *lineMax = DEFAULT_LINE_LENGTH;
+    }
+
+    for (;;) {
+        const int c = fgetc(inFile);
+        if (c == EOF) {
+            /* If we meet EOF before first character, returns GetLine_eof,
+             * otherwise GetLine_ok.
+             */
+            if (len == 0) result = GetLine_eof;
+            break;
+        }
+
+        /* Make enough space for len+1 (for final NUL) bytes. */
+        if (len+1 >= *lineMax) {
+            char* newLineBuf = NULL;
+            int newBufSize = *lineMax;
+
+            newBufSize += (newBufSize/2) + 1; /* x 1.5 */
+            if (newBufSize > MAX_LINE_LENGTH) newBufSize = MAX_LINE_LENGTH;
+            if (len+1 >= newBufSize) return GetLine_exceedMaxLineLength;
+
+            newLineBuf = (char*) realloc(*lineBuf, newBufSize);
+            if (newLineBuf == NULL) return GetLine_outOfMemory;
+
+            *lineBuf = newLineBuf;
+            *lineMax = newBufSize;
+        }
+
+        if (c == '\n') break;
+        (*lineBuf)[len++] = (char) c;
+    }
+
+    (*lineBuf)[len] = '\0';
+    return result;
+}
+
+
+/*  Converts one hexadecimal character to integer.
+ *  Returns -1, if given character is not hexadecimal.
+ */
+static int charToHex(char c)
+{
+    int result = -1;
+    if (c >= '0' && c <= '9') {
+        result = (int) (c - '0');
+    } else if (c >= 'A' && c <= 'F') {
+        result = (int) (c - 'A') + 0x0a;
+    } else if (c >= 'a' && c <= 'f') {
+        result = (int) (c - 'a') + 0x0a;
+    }
+    return result;
+}
+
+
+/*  Converts XXH32 canonical hexadecimal string hashStr to big endian unsigned char array dst.
+ *  Returns CANONICAL_FROM_STRING_INVALID_FORMAT, if hashStr is not well formatted.
+ *  Returns CANONICAL_FROM_STRING_OK, if hashStr is parsed successfully.
+ */
+static CanonicalFromStringResult canonicalFromString(unsigned char* dst,
+                                                     size_t dstSize,
+                                                     const char* hashStr)
+{
+    size_t i;
+    for (i = 0; i < dstSize; ++i) {
+        int h0, h1;
+
+        h0 = charToHex(hashStr[i*2 + 0]);
+        if (h0 < 0) return CanonicalFromString_invalidFormat;
+
+        h1 = charToHex(hashStr[i*2 + 1]);
+        if (h1 < 0) return CanonicalFromString_invalidFormat;
+
+        dst[i] = (unsigned char) ((h0 << 4) | h1);
+    }
+    return CanonicalFromString_ok;
+}
+
+
+/*  Parse single line of xxHash checksum file.
+ *  Returns PARSE_LINE_ERROR_INVALID_FORMAT, if line is not well formatted.
+ *  Returns PARSE_LINE_OK if line is parsed successfully.
+ *  And members of parseLine will be filled by parsed values.
+ *
+ *  - line must be ended with '\0'.
+ *  - Since parsedLine.filename will point within given argument `line`,
+ *    users must keep `line`s content during they are using parsedLine.
+ *
+ *  Given xxHash checksum line should have the following format:
+ *
+ *      <8 or 16 hexadecimal char> <space> <space> <filename...> <'\0'>
+ */
+static ParseLineResult parseLine(ParsedLine* parsedLine, const char* line)
+{
+    const char* const firstSpace = strchr(line, ' ');
+    const char* const secondSpace = firstSpace + 1;
+
+    parsedLine->filename = NULL;
+    parsedLine->xxhBits = 0;
+
+    if (firstSpace == NULL || *secondSpace != ' ') return ParseLine_invalidFormat;
+
+    switch (firstSpace - line)
+    {
+    case 8:
+        {   XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32;
+            if (canonicalFromString(xxh32c->digest, sizeof(xxh32c->digest), line)
+                != CanonicalFromString_ok) {
+                return ParseLine_invalidFormat;
+            }
+            parsedLine->xxhBits = 32;
+            break;
+        }
+
+    case 16:
+        {   XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64;
+            if (canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), line)
+                != CanonicalFromString_ok) {
+                return ParseLine_invalidFormat;
+            }
+            parsedLine->xxhBits = 64;
+            break;
+        }
+
+    default:
+            return ParseLine_invalidFormat;
+            break;
+    }
+
+    parsedLine->filename = secondSpace + 1;
+    return ParseLine_ok;
+}
+
+
+/*!  Parse xxHash checksum file.
+ */
+static void parseFile1(ParseFileArg* parseFileArg)
+{
+    const char* const inFileName = parseFileArg->inFileName;
+    ParseFileReport* const report = &parseFileArg->report;
+
+    unsigned long lineNumber = 0;
+    memset(report, 0, sizeof(*report));
+
+    while (!report->quit) {
+        FILE* fp = NULL;
+        LineStatus lineStatus = LineStatus_hashFailed;
+        GetLineResult getLineResult;
+        ParsedLine parsedLine;
+        memset(&parsedLine, 0, sizeof(parsedLine));
+
+        lineNumber++;
+        if (lineNumber == 0) {
+            /* This is unlikely happen, but md5sum.c has this
+             * error check. */
+            DISPLAY("%s : too many checksum lines\n", inFileName);
+            report->quit = 1;
+            break;
+        }
+
+        getLineResult = getLine(&parseFileArg->lineBuf, &parseFileArg->lineMax,
+                                parseFileArg->inFile);
+        if (getLineResult != GetLine_ok) {
+            if (getLineResult == GetLine_eof) break;
+
+            switch (getLineResult)
+            {
+            case GetLine_ok:
+            case GetLine_eof:
+                /* These cases never happen.  See above getLineResult related "if"s.
+                   They exist just for make gcc's -Wswitch-enum happy. */
+                break;
+
+            default:
+                DISPLAY("%s : %lu: unknown error\n", inFileName, lineNumber);
+                break;
+
+            case GetLine_exceedMaxLineLength:
+                DISPLAY("%s : %lu: too long line\n", inFileName, lineNumber);
+                break;
+
+            case GetLine_outOfMemory:
+                DISPLAY("%s : %lu: out of memory\n", inFileName, lineNumber);
+                break;
+            }
+            report->quit = 1;
+            break;
+        }
+
+        if (parseLine(&parsedLine, parseFileArg->lineBuf) != ParseLine_ok) {
+            report->nImproperlyFormattedLines++;
+            if (parseFileArg->warn) {
+                DISPLAY("%s : %lu: improperly formatted XXHASH checksum line\n"
+                    , inFileName, lineNumber);
+            }
+            continue;
+        }
+
+        if (report->xxhBits != 0 && report->xxhBits != parsedLine.xxhBits) {
+            /* Don't accept xxh32/xxh64 mixed file */
+            report->nImproperlyFormattedLines++;
+            report->nMixedFormatLines++;
+            if (parseFileArg->warn) {
+                DISPLAY("%s : %lu: improperly formatted XXHASH checksum line (XXH32/64)\n"
+                    , inFileName, lineNumber);
+            }
+            continue;
+        }
+
+        report->nProperlyFormattedLines++;
+        if (report->xxhBits == 0) {
+            report->xxhBits = parsedLine.xxhBits;
+        }
+
+        fp = fopen(parsedLine.filename, "rb");
+        if (fp == NULL) {
+            lineStatus = LineStatus_failedToOpen;
+        } else {
+            lineStatus = LineStatus_hashFailed;
+            switch (parsedLine.xxhBits)
+            {
+            case 32:
+                {   XXH32_hash_t xxh;
+                    BMK_hashStream(&xxh, algo_xxh32, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+                    if (xxh == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) {
+                        lineStatus = LineStatus_hashOk;
+                }   }
+                break;
+
+            case 64:
+                {   XXH64_hash_t xxh;
+                    BMK_hashStream(&xxh, algo_xxh64, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+                    if (xxh == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) {
+                        lineStatus = LineStatus_hashOk;
+                }   }
+                break;
+
+            default:
+                break;
+            }
+            fclose(fp);
+        }
+
+        switch (lineStatus)
+        {
+        default:
+            DISPLAY("%s : unknown error\n", inFileName);
+            report->quit = 1;
+            break;
+
+        case LineStatus_failedToOpen:
+            report->nOpenOrReadFailures++;
+            if (!parseFileArg->statusOnly) {
+                DISPLAYRESULT("%s : %lu: FAILED open or read %s\n"
+                    , inFileName, lineNumber, parsedLine.filename);
+            }
+            break;
+
+        case LineStatus_hashOk:
+        case LineStatus_hashFailed:
+            {   int b = 1;
+                if (lineStatus == LineStatus_hashOk) {
+                    /* If --quiet is specified, don't display "OK" */
+                    if (parseFileArg->quiet) b = 0;
+                } else {
+                    report->nMismatchedChecksums++;
+                }
+
+                if (b && !parseFileArg->statusOnly) {
+                    DISPLAYRESULT("%s: %s\n", parsedLine.filename
+                        , lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
+            }   }
+            break;
+        }
+    }   /* while (!report->quit) */
+}
+
+
+/*  Parse xxHash checksum file.
+ *  Returns 1, if all procedures were succeeded.
+ *  Returns 0, if any procedures was failed.
+ *
+ *  If strictMode != 0, return error code if any line is invalid.
+ *  If statusOnly != 0, don't generate any output.
+ *  If warn != 0, print a warning message to stderr.
+ *  If quiet != 0, suppress "OK" line.
+ *
+ *  "All procedures are succeeded" means:
+ *    - Checksum file contains at least one line and less than SIZE_T_MAX lines.
+ *    - All files are properly opened and read.
+ *    - All hash values match with its content.
+ *    - (strict mode) All lines in checksum file are consistent and well formatted.
+ *
+ */
+static int checkFile(const char* inFileName,
+                     const endianess displayEndianess,
+                     U32 strictMode,
+                     U32 statusOnly,
+                     U32 warn,
+                     U32 quiet)
+{
+    int result = 0;
+    FILE* inFile = NULL;
+    ParseFileArg parseFileArgBody;
+    ParseFileArg* const parseFileArg = &parseFileArgBody;
+    ParseFileReport* const report = &parseFileArg->report;
+
+    if (displayEndianess != big_endian) {
+        /* Don't accept little endian */
+        DISPLAY( "Check file mode doesn't support little endian\n" );
+        return 0;
+    }
+
+    /* note : stdinName is special constant pointer.  It is not a string. */
+    if (inFileName == stdinName) {
+        /* note : Since we expect text input for xxhash -c mode,
+         * Don't set binary mode for stdin */
+        inFile = stdin;
+    } else {
+        inFile = fopen( inFileName, "rt" );
+    }
+
+    if (inFile == NULL) {
+        DISPLAY( "Pb opening %s\n", inFileName);
+        return 0;
+    }
+
+    parseFileArg->inFileName    = inFileName;
+    parseFileArg->inFile        = inFile;
+    parseFileArg->lineMax       = DEFAULT_LINE_LENGTH;
+    parseFileArg->lineBuf       = (char*) malloc((size_t) parseFileArg->lineMax);
+    parseFileArg->blockSize     = 64 * 1024;
+    parseFileArg->blockBuf      = (char*) malloc(parseFileArg->blockSize);
+    parseFileArg->strictMode    = strictMode;
+    parseFileArg->statusOnly    = statusOnly;
+    parseFileArg->warn          = warn;
+    parseFileArg->quiet         = quiet;
+
+    parseFile1(parseFileArg);
+
+    free(parseFileArg->blockBuf);
+    free(parseFileArg->lineBuf);
+
+    if (inFile != stdin) fclose(inFile);
+
+    /* Show error/warning messages.  All messages are copied from md5sum.c
+     */
+    if (report->nProperlyFormattedLines == 0) {
+        DISPLAY("%s: no properly formatted XXHASH checksum lines found\n", inFileName);
+    } else if (!statusOnly) {
+        if (report->nImproperlyFormattedLines) {
+            DISPLAYRESULT("%lu lines are improperly formatted\n"
+                , report->nImproperlyFormattedLines);
+        }
+        if (report->nOpenOrReadFailures) {
+            DISPLAYRESULT("%lu listed files could not be read\n"
+                , report->nOpenOrReadFailures);
+        }
+        if (report->nMismatchedChecksums) {
+            DISPLAYRESULT("%lu computed checksums did NOT match\n"
+                , report->nMismatchedChecksums);
+    }   }
+
+    /* Result (exit) code logic is copied from
+     * gnu coreutils/src/md5sum.c digest_check() */
+    result =   report->nProperlyFormattedLines != 0
+            && report->nMismatchedChecksums == 0
+            && report->nOpenOrReadFailures == 0
+            && (!strictMode || report->nImproperlyFormattedLines == 0)
+            && report->quit == 0;
+    return result;
+}
+
+
+static int checkFiles(const char** fnList, int fnTotal,
+                      const endianess displayEndianess,
+                      U32 strictMode,
+                      U32 statusOnly,
+                      U32 warn,
+                      U32 quiet)
+{
+    int ok = 1;
+
+    /* Special case for stdinName "-",
+     * note: stdinName is not a string.  It's special pointer. */
+    if (fnTotal==0) {
+        ok &= checkFile(stdinName, displayEndianess, strictMode, statusOnly, warn, quiet);
+    } else {
+        int fnNb;
+        for (fnNb=0; fnNb<fnTotal; fnNb++)
+            ok &= checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, warn, quiet);
+    }
+    return ok ? 0 : 1;
+}
+
+
+/* ********************************************************
+*  Main
+**********************************************************/
+
+static int usage(const char* exename)
+{
+    DISPLAY( WELCOME_MESSAGE(exename) );
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [arg] [filenames]\n", exename);
+    DISPLAY( "When no filename provided, or - provided : use stdin as input\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default: %i)\n", (int)g_defaultAlgo);
+    DISPLAY( " -c  : read xxHash sums from the [filenames] and check them\n");
+    DISPLAY( " -h  : help \n");
+    return 0;
+}
+
+
+static int usage_advanced(const char* exename)
+{
+    usage(exename);
+    DISPLAY( "Advanced :\n");
+    DISPLAY( " --little-endian : hash printed using little endian convention (default: big endian)\n");
+    DISPLAY( " -V, --version   : display version\n");
+    DISPLAY( " -h, --help      : display long help and exit\n");
+    DISPLAY( " -b  : benchmark mode \n");
+    DISPLAY( " -i# : number of iterations (benchmark mode; default %i)\n", g_nbIterations);
+    DISPLAY( "\n");
+    DISPLAY( "The following four options are useful only when verifying checksums (-c):\n");
+    DISPLAY( "--strict : don't print OK for each successfully verified file\n");
+    DISPLAY( "--status : don't output anything, status code shows success\n");
+    DISPLAY( "--quiet  : exit non-zero for improperly formatted checksum lines\n");
+    DISPLAY( "--warn   : warn about improperly formatted checksum lines\n");
+    return 0;
+}
+
+static int badusage(const char* exename)
+{
+    DISPLAY("Wrong parameters\n");
+    usage(exename);
+    return 1;
+}
+
+/*! readU32FromChar() :
+   @return : unsigned integer value read from input in `char` format,
+             0 is no figure at *stringPtr position.
+    Interprets K, KB, KiB, M, MB and MiB suffix.
+    Modifies `*stringPtr`, advancing it to position where reading stopped.
+    Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+    unsigned result = 0;
+    while ((**stringPtr >='0') && (**stringPtr <='9'))
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        result <<= 10;
+        if (**stringPtr=='M') result <<= 10;
+        (*stringPtr)++ ;
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
+    return result;
+}
+
+int main(int argc, const char** argv)
+{
+    int i, filenamesStart = 0;
+    const char* const exename = argv[0];
+    U32 benchmarkMode = 0;
+    U32 fileCheckMode = 0;
+    U32 strictMode    = 0;
+    U32 statusOnly    = 0;
+    U32 warn          = 0;
+    U32 quiet         = 0;
+    U32 specificTest  = 0;
+    size_t keySize    = XXH_DEFAULT_SAMPLE_SIZE;
+    algoType algo     = g_defaultAlgo;
+    endianess displayEndianess = big_endian;
+
+    /* special case : xxh32sum default to 32 bits checksum */
+    if (strstr(exename, "xxh32sum") != NULL) algo = algo_xxh32;
+
+    for(i=1; i<argc; i++) {
+        const char* argument = argv[i];
+
+        if(!argument) continue;   /* Protection, if argument empty */
+
+        if (!strcmp(argument, "--little-endian")) { displayEndianess = little_endian; continue; }
+        if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; }
+        if (!strcmp(argument, "--strict")) { strictMode = 1; continue; }
+        if (!strcmp(argument, "--status")) { statusOnly = 1; continue; }
+        if (!strcmp(argument, "--quiet")) { quiet = 1; continue; }
+        if (!strcmp(argument, "--warn")) { warn = 1; continue; }
+        if (!strcmp(argument, "--help")) { return usage_advanced(exename); }
+        if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE(exename)); return 0; }
+
+        if (*argument!='-') {
+            if (filenamesStart==0) filenamesStart=i;   /* only supports a continuous list of filenames */
+            continue;
+        }
+
+        /* command selection */
+        argument++;   /* note : *argument=='-' */
+
+        while (*argument!=0) {
+            switch(*argument)
+            {
+            /* Display version */
+            case 'V':
+                DISPLAY(WELCOME_MESSAGE(exename)); return 0;
+
+            /* Display help on usage */
+            case 'h':
+                return usage_advanced(exename);
+
+            /* select hash algorithm */
+            case 'H':
+                algo = (algoType)(argument[1] - '0');
+                argument+=2;
+                break;
+
+            /* File check mode */
+            case 'c':
+                fileCheckMode=1;
+                argument++;
+                break;
+
+            /* Warning mode (file check mode only, alias of "--warning") */
+            case 'w':
+                warn=1;
+                argument++;
+                break;
+
+            /* Trigger benchmark mode */
+            case 'b':
+                argument++;
+                benchmarkMode = 1;
+                specificTest = readU32FromChar(&argument);   /* select one specific test (hidden option) */
+                break;
+
+            /* Modify Nb Iterations (benchmark only) */
+            case 'i':
+                argument++;
+                g_nbIterations = readU32FromChar(&argument);
+                break;
+
+            /* Modify Block size (benchmark only) */
+            case 'B':
+                argument++;
+                keySize = readU32FromChar(&argument);
+                break;
+
+            /* Modify verbosity of benchmark output (hidden option) */
+            case 'q':
+                argument++;
+                g_displayLevel--;
+                break;
+
+            default:
+                return badusage(exename);
+            }
+        }
+    }   /* for(i=1; i<argc; i++) */
+
+    /* Check benchmark mode */
+    if (benchmarkMode) {
+        DISPLAYLEVEL(2, WELCOME_MESSAGE(exename) );
+        BMK_sanityCheck();
+        if (filenamesStart==0) return BMK_benchInternal(keySize, specificTest);
+        return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart, specificTest);
+    }
+
+    /* Check if input is defined as console; trigger an error in this case */
+    if ( (filenamesStart==0) && IS_CONSOLE(stdin) ) return badusage(exename);
+
+    if (filenamesStart==0) filenamesStart = argc;
+    if (fileCheckMode) {
+        return checkFiles(argv+filenamesStart, argc-filenamesStart,
+                          displayEndianess, strictMode, statusOnly, warn, quiet);
+    } else {
+        return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess);
+    }
+}
+
+#endif /* XXHASH_C_2097394837 */
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 0eb348c189d..de68c20b4d7 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,13 +1,9 @@
-EXTRA_DIST = glusterfs.vol.sample glusterfsd.vol.sample glusterfs.8 mount.glusterfs.8\
-	porting_guide.txt authentication.txt coding-standard.pdf get_put_api_using_xattr.txt \
-	translator-options.txt mac-related-xattrs.txt replicate.pdf glusterd.vol gluster.8 \
-        glusterd.8 glusterfs-volgen.8
+EXTRA_DIST = glusterfs.8 mount.glusterfs.8 gluster.8 \
+			glusterd.8 glusterfsd.8
 
-SUBDIRS = examples hacker-guide
+man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8
+if WITH_SERVER
+man8_MANS += glusterd.8 glusterfsd.8
+endif
 
-voldir = $(sysconfdir)/glusterfs
-vol_DATA = glusterfs.vol.sample glusterfsd.vol.sample glusterd.vol
-
-man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 glusterfs-volgen.8
-
-CLEANFILES = 
+CLEANFILES =
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 00000000000..6aa28642ef4
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,26 @@
+## Developer Guide
+
+Gluster's contributors can check about the internals by visiting [Developer Guide Section](developer-guide). While it is not 'comprehensive', it can help you to get started.
+
+Also while coding, keep [Coding Standard](developer-guide/coding-standard.md) in mind.
+
+When you are ready to commit the changes, make sure you meet our [Commit message standard](developer-guide/commit-guidelines.md).
+
+## Admin Guide ##
+
+The gluster administration guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable admin guide can be found [here](http://docs.gluster.org/en/latest/Administrator%20Guide/).
+
+The doc patch has to be sent against the above mentioned repository.
+
+## Features/Spec ##
+
+The Gluster features which are 'in progress' or implemented can be found at [github](https://github.com/gluster/glusterfs-specs).
+
+## Upgrade Guide ##
+
+The gluster upgrade guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable upgrade guide can be found [here](http://docs.gluster.org/en/latest/Upgrade-Guide)
+
+The doc patch has to be sent against the above mentioned repository.
+
+
+For more details about the docuemntation workflow please refer [this discussion](https://www.mail-archive.com/gluster-users@gluster.org/msg21168.html)
diff --git a/doc/authentication.txt b/doc/authentication.txt
deleted file mode 100644
index 70aafd93334..00000000000
--- a/doc/authentication.txt
+++ /dev/null
@@ -1,112 +0,0 @@
-
-* Authentication is provided by two modules addr and login. Login based authentication uses username/password from client for authentication. Each module returns either ACCEPT, REJCET or DONT_CARE. DONT_CARE is returned if the input authentication information to the module is not concerned to its working. The theory behind authentication is that "none of the auth modules should return REJECT and atleast one of them should return ACCEPT"
-
-* Currently all the authentication related information is passed un-encrypted over the network from client to server.
-
-----------------------------------------------------------------------------------------------------
-* options provided in protocol/client:
-	* for username/password based authentication:
-	      option username <username>
-	      option password <password>
-	* client can have only one set of username/password
-	* for addr based authentication:
-	      * no options required in protocol/client. Client has to bind to privileged port (port < 1024 ) which means the process in which protocol/client is loaded has to be run as root.
-
-----------------------------------------------------------------------------------------------------
-* options provided in protocol/server:
-	* for username/password based authentication:
-	      option auth.login.<brick>.allow [comma seperated list of usernames using which clients can connect to volume <brick>]
-	      option auth.login.<username>.password <password> #specify password <password> for username <username>
-	* for addr based authentication:
-	      option auth.addr.<brick>.allow [comma seperated list of ip-addresses/unix-paths from which clients are allowed to connect to volume <brick>]
-	      option auth.addr.<brick>.reject [comma seperated list of ip-addresses/unix-paths from which clients are not allowed to connect to volume <brick>]
-	* negation operator '!' is used to invert the sense of matching.
-	  Eg., option auth.addr.brick.allow !a.b.c.d #do not allow client from a.b.c.d to connect to volume brick
-	       option auth.addr.brick.reject !w.x.y.z #allow client from w.x.y.z to connect to volume brick
-	* wildcard '*' can be used to match any ip-address/unix-path
-
-----------------------------------------------------------------------------------------------------
-
-* Usecases:
-
-* username/password based authentication only
-      protocol/client:
-	option username foo
-	option password foo-password
-	option remote-subvolume foo-brick
-
-      protocol/server:
-	option auth.login.foo-brick.allow foo,who #,other users allowed to connect to foo-brick
-	option auth.login.foo.password foo-password
-	option auth.login.who.password who-password
-
-      * in protocol/server, dont specify ip from which client is connecting in auth.addr.foo-brick.reject list
-
-****************************************************************************************************
-
-* ip based authentication only
-      protocol/client:
-	option remote-subvolume foo-brick
-	* Client is connecting from a.b.c.d
-      
-      protocol/server:
-	option auth.addr.foo-brick.allow a.b.c.d,e.f.g.h,i.j.k.l #, other ip addresses from which clients are allowed to connect to foo-brick
-
-****************************************************************************************************
-* ip and username/password based authentication
-  * allow only "user foo from a.b.c.d"
-    protocol/client:
-	option username foo
-	option password foo-password
-	option remote-subvolume foo-brick
-
-    protocol/server:
-	option auth.login.foo-brick.allow foo
-	option auth.login.foo.password foo-password
-	option auth.addr.foo-brick.reject !a.b.c.d
-
-  * allow only "user foo" from a.b.c.d i.e., only user foo is allowed from a.b.c.d, but anyone is allowed from ip addresses other than a.b.c.d
-    protocol/client:
-	option username foo
-	option password foo-password
-	option remote-subvolume foo-brick
-
-    protocol/server:
-	option auth.login.foo-brick.allow foo
-	option auth.login.foo.password foo-password
-	option auth.addr.foo-brick.allow !a.b.c.d
-
-  * reject only "user shoo from a.b.c.d"
-    protcol/client:
-	option remote-subvolume shoo-brick
-    
-    protocol/server:
-	# observe that no "option auth.login.shoo-brick.allow shoo" given
-	# Also other users from a.b.c.d have to be explicitly allowed using auth.login.shoo-brick.allow ...
-	option auth.addr.shoo-brick.allow !a.b.c.d
-
-  * reject only "user shoo" from a.b.c.d i.e., user shoo from a.b.c.d has to be rejected. 
-    * same as reject only "user shoo from a.b.c.d" above, but rules have to be added whether to allow ip addresses (and users from those ips) other than a.b.c.d
-
-****************************************************************************************************
-
-* ip or username/password based authentication
-  
-  * allow user foo or clients from a.b.c.d
-    protocol/client:
-	option remote-subvolume foo-brick
-
-    protocol/server:
-	option auth.login.foo-brick.allow foo
-	option auth.login.foo.password foo-password
-	option auth.addr.foo-brick.allow a.b.c.d
-
-  * reject user shoo or clients from a.b.c.d
-    protocol/client:
-	option remote-subvolume shoo-brick
-   
-    protocol/server:
-	option auth.login.shoo-brick.allow <usernames other than shoo>
-	#for each username mentioned in the above <usernames other than shoo> list, specify password as below
-	option auth.login.<username other than shoo>.password password
-	option auth.addr.shoo-brick.reject a.b.c.d
diff --git a/doc/booster.txt b/doc/booster.txt
deleted file mode 100644
index 684ac8965c4..00000000000
--- a/doc/booster.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-Introduction
-============
-* booster is a LD_PRELOADable library which boosts read/write performance by bypassing fuse for 
-  read() and write() calls.
-
-Requirements
-============
-* fetch volfile from glusterfs.
-* identify whether multiple files are from the same mount point. If so, use only one context.
-
-Design
-======
-* for a getxattr, along with other attributes, fuse returns following attributes.
-  * contents of client volume-file.
-  * mount point.
-
-* LD_PRELOADed booster.so maintains an hash table storing mount-points and libglusterfsclient handles 
-  so that handles are reused for files from same mount point. 
-
-* it also maintains a fdtable. fdtable maps the fd (integer) returned to application to fd (pointer to fd struct)
-  used by libglusterfsclient. application is returned the same fd as the one returned from libc apis.
-
-* During fork, these tables are overwritten to enable creation of fresh glusterfs context in child.
- 
-Working
-=======
-* application willing to use booster LD_PRELOADs booster.so which is a wrapper library implementing 
-  open, read and write.
-
-* application should specify the path to logfile through the environment variable GLFS_BOOSTER_LOGFILE. If 
-  not specified, logging is done to /dev/stderr.
-
-* open call does,
-  * real_open on the file.
-  * fgetxattr(fd).
-  * store the volume-file content got in the dictionary to a temparory file.
-  * look in the hashtable for the mount-point, if already present get the libglusterfsclient handle from the 
-    hashtable. Otherwise get a new handle from libglusterfsclient (be careful about mount point not present in 
-    the hashtable and multiple glusterfs_inits running simultaneously for the same mount-point there by using 
-    multiple handles for the same mount point).
-  * real_close (fd).
-  * delete temporary volume-volfile.
-  * glusterfs_open (handle, path, mode).
-  * store the fd returned by glusterfs_open in the fdtable at the same index as the fd returned by real_open.
-  * return the index as fd.
-
-* read/write calls do,
-  * get the libglusterfsclient fd from fdtable.
-  * if found use glusterfs_read/glusterfs_write, else use real_read/real_write.
-
-* close call does,
-  * remove the fd from the fdtable.
-
-* other calls use real_calls. 
diff --git a/doc/coding-standard.pdf b/doc/coding-standard.pdf
deleted file mode 100644
index bc9cb56202c..00000000000
--- a/doc/coding-standard.pdf
+++ /dev/null
diff --git a/doc/coding-standard.tex b/doc/coding-standard.tex
deleted file mode 100644
index 92f799c013f..00000000000
--- a/doc/coding-standard.tex
+++ /dev/null
@@ -1,385 +0,0 @@
-\documentclass{article}[12pt]
-\usepackage{color}
-
-\begin{document}
-
-
-\hrule
-\begin{center}\textbf{\Large{GlusterFS Coding Standards}}\end{center}
-\begin{center}\textbf{\large{\textcolor{red}{Z} Research}}\end{center}
-\begin{center}{July 14, 2008}\end{center}
-\hrule
-
-\vspace{8ex}
-
-\section*{$\bullet$ Structure definitions should have a comment per member}
-
-Every member in a structure definition must have a comment about its
-purpose. The comment should be descriptive without being overly verbose.
-
-\vspace{2ex}
-\textsl{Bad}:
-
-\begin{verbatim}
-        gf_lock_t   lock;           /* lock */
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        DBTYPE      access_mode;    /* access mode for accessing 
-                                     * the databases, can be
-                                     * DB_HASH, DB_BTREE
-                                     * (option access-mode <mode>)
-                                     */
-\end{verbatim}
-
-\section*{$\bullet$ Declare all variables at the beginning of the function}
-All local variables in a function must be declared immediately after the
-opening brace. This makes it easy to keep track of memory that needs to be freed
-during exit. It also helps debugging, since gdb cannot handle variables
-declared inside loops or other such blocks.
-
-\section*{$\bullet$ Always initialize local variables}
-Every local variable should be initialized to a sensible default value
-at the point of its declaration. All pointers should be initialized to NULL,
-and all integers should be zero or (if it makes sense) an error value.
-
-\vspace{2ex}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        int ret       = 0;
-        char *databuf = NULL;
-        int _fd       = -1;
-\end{verbatim}
-
-\section*{$\bullet$ Initialization should always be done with a constant value}
-Never use a non-constant expression as the initialization value for a variable.
-
-\vspace{2ex}
-
-\textsl{Bad}:
-
-\begin{verbatim}
-        pid_t pid     = frame->root->pid;
-        char *databuf = malloc (1024);
-\end{verbatim}
-
-\section*{$\bullet$ Validate all arguments to a function}
-All pointer arguments to a function must be checked for \texttt{NULL}.
-A macro named \texttt{VALIDATE} (in \texttt{common-utils.h}) 
-takes one argument, and if it is \texttt{NULL}, writes a log message and
-jumps to a label called \texttt{err} after setting op\_ret and op\_errno
-appropriately. It is recommended to use this template.
-
-\vspace{2ex}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        VALIDATE(frame);
-        VALIDATE(this);
-        VALIDATE(inode);
-\end{verbatim}
-
-\section*{$\bullet$ Never rely on precedence of operators}
-Never write code that relies on the precedence of operators to execute
-correctly.  Such code can be hard to read and someone else might not
-know the precedence of operators as accurately as you do.
-\vspace{2ex}
-
-\textsl{Bad}:
-
-\begin{verbatim}
-        if (op_ret == -1 && errno != ENOENT)
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        if ((op_ret == -1) && (errno != ENOENT))
-\end{verbatim}
-
-\section*{$\bullet$ Use exactly matching types}
-Use a variable of the exact type declared in the manual to hold the
-return value of a function. Do not use an ``equivalent'' type.
-
-\vspace{2ex}
-
-\textsl{Bad}:
-
-\begin{verbatim}
-        int len = strlen (path);
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        size_t len = strlen (path);
-\end{verbatim}
-
-\section*{$\bullet$ Never write code such as \texttt{foo->bar->baz}; check every pointer}
-Do not write code that blindly follows a chain of pointer
-references. Any pointer in the chain may be \texttt{NULL} and thus
-cause a crash. Verify that each pointer is non-null before following
-it.
-
-\section*{$\bullet$ Check return value of all functions and system calls}
-The return value of all system calls and API functions must be checked
-for success or failure.
-
-\vspace{2ex}
-\textsl{Bad}:
-
-\begin{verbatim}
-        close (fd);
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        op_ret = close (_fd);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, 
-                        "close on file %s failed (%s)", real_path, 
-                        strerror (errno));
-                op_errno = errno;
-                goto out;
-        }
-\end{verbatim}
-
-
-\section*{$\bullet$ Gracefully handle failure of malloc}
-GlusterFS should never crash or exit due to lack of memory. If a
-memory allocation fails, the call should be unwound and an error
-returned to the user.
-
-\section*{$\bullet$ Use result args and reserve the return value to indicate success or failure}
-The return value of every functions must indicate success or failure (unless 
-it is impossible for the function to fail --- e.g., boolean functions). If 
-the function needs to return additional data, it must be returned using a 
-result (pointer) argument.
-
-\vspace{2ex}
-\textsl{Bad}:
-
-\begin{verbatim}
-        int32_t dict_get_int32 (dict_t *this, char *key);
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        int dict_get_int32 (dict_t *this, char *key, int32_t *val);
-\end{verbatim}
-
-\section*{$\bullet$ Always use the `n' versions of string functions}
-Unless impossible, use the length-limited versions of the string functions.
-
-\vspace{2ex}
-\textsl{Bad}:
-
-\begin{verbatim}
-        strcpy (entry_path, real_path);
-\end{verbatim}
-
-\textsl{Good}:
-
-\begin{verbatim}
-        strncpy (entry_path, real_path, entry_path_len);
-\end{verbatim}
-
-\section*{$\bullet$ No dead or commented code}
-There must be no dead code (code to which control can never be passed) or 
-commented out code in the codebase.
-
-\section*{$\bullet$ Only one unwind and return per function}
-There must be only one exit out of a function. \texttt{UNWIND} and return 
-should happen at only point in the function.
-
-\section*{$\bullet$ Function length or Keep functions small}
-We live in the UNIX-world where modules do one thing and do it well.
-This rule should apply to our functions also. If a function is very long, try splitting it
-into many little helper functions. The question is, in a coding
-spree, how do we know a function is long and unreadable. One rule of
-thumb given by Linus Torvalds is that, a function should be broken-up
-if you have 4 or more levels of indentation going on for more than 3-4
-lines.
-
-\vspace{2ex}
-\textsl{Example for a helper function}:
-\begin{verbatim}
-        static int
-        same_owner (posix_lock_t *l1, posix_lock_t *l2)
-        {
-                return ((l1->client_pid == l2->client_pid) &&
-                       (l1->transport  == l2->transport));
-        }
-\end{verbatim}
-
-\section*{$\bullet$ Defining functions as static}
-Define internal functions as static only if you're
-very sure that there will not be a crash(..of any kind..) emanating in
-that function. If there is even a remote possibility, perhaps due to
-pointer derefering, etc, declare the function as non-static. This
-ensures that when a crash does happen, the function name shows up the
-in the back-trace generated by libc. However, doing so has potential
-for polluting the function namespace, so to avoid conflicts with other
-components in other parts, ensure that the function names are
-prepended with a prefix that identify the component to which it
-belongs. For eg. non-static functions in io-threads translator start
-with iot\_.
-
-\section*{$\bullet$ Ensure function calls wrap around after
-80-columns.}
-Place remaining arguments on the next line if needed.
-
-\section*{$\bullet$ Functions arguments and function definition}
-Place all the arguments of a function definition on the same line
-until the line goes beyond 80-cols. Arguments that extend beyind
-80-cols should be placed on the next line.
-
-\section*{Style issues}
-
-\subsection*{Brace placement}
-Use K\&R/Linux style of brace placement for blocks.
-
-\textsl{Example}:
-\begin{verbatim}
-        int some_function (...)
-        {
-                if (...) {
-                        /* ... */
-                } else if (...) {
-                        /* ... */
-                } else {
-                        /* ... */
-                }
-
-                do {
-                        /* ... */
-                } while (cond);
-        }
-\end{verbatim}
-
-\subsection*{Indentation}
-Use \textbf{eight} spaces for indenting blocks. Ensure that your
-file contains only spaces and not tab characters. You can do this
-in Emacs by selecting the entire file (\texttt{C-x h}) and
-running \texttt{M-x untabify}.
-
-To make Emacs indent lines automatically by eight spaces, add this
-line to your \texttt{.emacs}:
-
-\begin{verbatim}
-        (add-hook 'c-mode-hook (lambda () (c-set-style "linux")))
-\end{verbatim}
-
-\subsection*{Comments}
-Write a comment before every function describing its purpose (one-line),
-its arguments, and its return value. Mention whether it is an internal
-function or an exported function.
-
-Write a comment before every structure describing its purpose, and
-write comments about each of its members.
-
-Follow the style shown below for comments, since such comments
-can then be automatically extracted by doxygen to generate
-documentation.
-
-\textsl{Example}:
-\begin{verbatim}
-/**
- * hash_name -hash function for filenames
- * @par:  parent inode number
- * @name: basename of inode
- * @mod:  number of buckets in the hashtable
- *
- * @return: success: bucket number
- *          failure: -1
- *
- * Not for external use.
- */
-\end{verbatim}
-
-\subsection*{Indicating critical sections}
-To clearly show regions of code which execute with locks held, use 
-the following format:
-
-\begin{verbatim}
-        pthread_mutex_lock (&mutex);
-        {
-                /* code */
-        }
-        pthread_mutex_unlock (&mutex);
-\end{verbatim}
-
-\section*{A skeleton fop function}
-This is the recommended template for any fop. In the beginning come
-the initializations. After that, the `success' control flow should be
-linear.  Any error conditions should cause a \texttt{goto} to a single
-point, \texttt{out}.  At that point, the code should detect the error
-that has occured and do appropriate cleanup.
-
-\begin{verbatim}
-int32_t 
-sample_fop (call_frame_t *frame, xlator_t *this, ...)
-{
-        char *            var1     = NULL;
-        int32_t           op_ret   = -1;
-        int32_t           op_errno = 0;
-        DIR *             dir      = NULL;
-        struct posix_fd * pfd      = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-
-        /* other validations */
-        
-        dir = opendir (...);
-
-        if (dir == NULL) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, 
-                        "opendir failed on %s (%s)", loc->path, 
-                        strerror (op_errno));
-                goto out;
-        }
-
-        /* another system call */
-        if (...) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory :(");
-                goto out;
-        }
-
-        /* ... */
-
- out:
-        if (op_ret == -1) {
-
-          /* check for all the cleanup that needs to be
-             done */
-
-                if (dir) {
-                        closedir (dir);
-                        dir = NULL;
-                }     
-        
-                if (pfd) {
-                        if (pfd->path)
-                                FREE (pfd->path);
-                        FREE (pfd);
-                        pfd = NULL;
-                }
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, fd);
-        return 0;
-}
-\end{verbatim}
-
-\end{document}
diff --git a/doc/debugging/analyzing-regression-cores.md b/doc/debugging/analyzing-regression-cores.md
new file mode 100644
index 00000000000..5e10f41c6eb
--- /dev/null
+++ b/doc/debugging/analyzing-regression-cores.md
@@ -0,0 +1,54 @@
+# Analyzing Regression Cores
+This document explains how to analyze core-dumps obtained from regression machines, with examples.
+1. Download the core-tarball and extract it.
+2. `cd` into directory where the tarball is extracted.
+```
+[sh]# pwd
+/home/user/Downloads
+[sh]# ls
+build  build-install-20150625_05_42_39.tar.bz2  lib64  usr
+```
+3. Determine the core file you need to examine. There can be more than one core file. You can list them from './build/install/cores' directory.
+```
+[sh]# ls build/install/cores/
+core.9341  liblist.txt  liblist.txt.tmp
+```
+In case you are unsure which binary generated the core-file, executing 'file' command on it will help.
+```
+[sh]# file ./build/install/cores/core.9341 
+./build/install/cores/core.9341: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from '/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'
+```
+As seen, the core file was generated by glusterfsd binary, and path to it is provided (/build/install/sbin/glusterfsd).
+
+4. Now, run the following command on the core:
+```
+gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.xxx' <target, say ./build/install/sbin/glusterd>
+In this case,
+gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.9341' ./build/install/sbin/glusterfsd
+```
+5. You can cross check if all shared libraries are available and loaded by using 'info sharedlibrary' command from inside gdb.
+6. Once verified, usual gdb commands based on requirement can be used to debug the core.
+   `bt` or `backtrace` from gdb of core used in examples:
+```
+Core was generated by `/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'.
+Program terminated with signal SIGABRT, Aborted.
+#0  0x00007f512a54e625 in raise () from ./lib64/libc.so.6
+(gdb) bt
+#0  0x00007f512a54e625 in raise () from ./lib64/libc.so.6
+#1  0x00007f512a54fe05 in abort () from ./lib64/libc.so.6
+#2  0x00007f512a54774e in __assert_fail_base () from ./lib64/libc.so.6
+#3  0x00007f512a547810 in __assert_fail () from ./lib64/libc.so.6
+#4  0x00007f512b9fc434 in __gf_free (free_ptr=0x7f50f4000e50) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/mem-pool.c:304
+#5  0x00007f512b9b6657 in loc_wipe (loc=0x7f510c20d1a0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/xlator.c:685
+#6  0x00007f511cb8201d in mq_start_quota_txn_v2 (this=0x7f5118019b60, loc=0x7f510c20d2b8, ctx=0x7f50f4000bf0, contri=0x7f50f4000d60)
+    at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:2921
+#7  0x00007f511cb82c55 in mq_initiate_quota_task (opaque=0x7f510c20d2b0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3199
+#8  0x00007f511cb81820 in mq_synctask (this=0x7f5118019b60, task=0x7f511cb829fa <mq_initiate_quota_task>, spawn=_gf_false, loc=0x7f510c20d430, dict=0x0, buf=0x0, contri=0)
+    at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:2789
+#9  0x00007f511cb82f82 in mq_initiate_quota_blocking_txn (this=0x7f5118019b60, loc=0x7f510c20d430) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3230
+#10 0x00007f511cb82844 in mq_reduce_parent_size_task (opaque=0x7f510c000df0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3117
+#11 0x00007f512ba0f9dc in synctask_wrap (old_task=0x7f510c0053e0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/syncop.c:370
+#12 0x00007f512a55f8f0 in ?? () from ./lib64/libc.so.6
+#13 0x0000000000000000 in ?? ()
+(gdb) 
+```
diff --git a/doc/debugging/gfid-to-path.md b/doc/debugging/gfid-to-path.md
new file mode 100644
index 00000000000..1917bf2cca1
--- /dev/null
+++ b/doc/debugging/gfid-to-path.md
@@ -0,0 +1,68 @@
+# Convert GFID to Path
+
+GlusterFS internal file identifier (GFID) is a uuid that is unique to each
+file across the entire cluster. This is analogous to inode number in a
+normal filesystem. The GFID of a file is stored in its xattr named
+`trusted.gfid`.
+
+#### Special mount using [gfid-access translator][1]:
+```
+mount -t glusterfs -o aux-gfid-mount vm1:test /mnt/testvol
+```
+
+Assuming, you have `GFID` of a file from changelog (or somewhere else).
+For trying this out, you can get `GFID` of a file from mountpoint:
+```
+getfattr -n glusterfs.gfid.string /mnt/testvol/dir/file
+```
+
+
+---
+### Get file path from GFID (Method 1):
+**(Lists hardlinks delimited by `:`, returns path as seen from mountpoint)**
+
+#### Turn on build-pgfid option
+```
+gluster volume set test build-pgfid on
+```
+Read virtual xattr `glusterfs.ancestry.path` which contains the file path
+```
+getfattr -n glusterfs.ancestry.path -e text /mnt/testvol/.gfid/<GFID>
+```
+
+**Example:**
+```
+[root@vm1 glusterfs]# ls -il /mnt/testvol/dir/
+total 1
+10610563327990022372 -rw-r--r--. 2 root root 3 Jul 17 18:05 file
+10610563327990022372 -rw-r--r--. 2 root root 3 Jul 17 18:05 file3
+
+[root@vm1 glusterfs]# getfattr -n glusterfs.gfid.string /mnt/testvol/dir/file
+getfattr: Removing leading '/' from absolute path names
+# file: mnt/testvol/dir/file
+glusterfs.gfid.string="11118443-1894-4273-9340-4b212fa1c0e4"
+
+[root@vm1 glusterfs]# getfattr -n glusterfs.ancestry.path -e text /mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
+getfattr: Removing leading '/' from absolute path names
+# file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
+glusterfs.ancestry.path="/dir/file:/dir/file3"
+```
+
+---
+### Get file path from GFID (Method 2):
+**(Does not list all hardlinks, returns backend brick path)**
+```
+getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/<GFID>
+```
+
+**Example:**
+```
+[root@vm1 glusterfs]# getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
+getfattr: Removing leading '/' from absolute path names
+# file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4
+trusted.glusterfs.pathinfo="(<DISTRIBUTE:test-dht> <POSIX(/mnt/brick-test/b):vm1:/mnt/brick-test/b/dir//file3>)"
+```
+
+---
+#### References and links:
+[posix: placeholders for GFID to path conversion](http://review.gluster.org/5951)
diff --git a/doc/debugging/mem-alloc-list.md b/doc/debugging/mem-alloc-list.md
new file mode 100644
index 00000000000..1c68e65d323
--- /dev/null
+++ b/doc/debugging/mem-alloc-list.md
@@ -0,0 +1,19 @@
+## Viewing Memory Allocations
+
+While statedumps provide stats of the number of allocations, size etc for a
+particular mem type, there is no easy way to examine all the allocated objects of that type
+in memory.Being able to view this information could help with determining how an object is used,
+and if there are any memory leaks.
+
+The mem_acct_rec structures have been updated to include lists to which the allocated object is
+added. These can be examined in gdb using simple scripts.
+
+`gdb> plist xl->mem_acct.rec[$type]->obj_list`
+
+will print out the pointers of all allocations of $type.
+
+These changes are primarily targeted at developers and need to enabled
+at compile-time using `configure --enable-debug`.
+
+
+
diff --git a/doc/debugging/split-brain.md b/doc/debugging/split-brain.md
new file mode 100644
index 00000000000..6b122c40551
--- /dev/null
+++ b/doc/debugging/split-brain.md
@@ -0,0 +1,264 @@
+# Steps to recover from File split-brain
+This document contains steps to recover from a file split-brain.
+## Quick Start:
+### Step 1. Get the path of the file that is in split-brain:
+It can be obtained either by
+1. The command `gluster volume heal info split-brain`.
+2. Identify the files for which file operations performed from the client keep failing with Input/Output error.
+
+### Step 2. Close the applications that opened this file from the mount point.
+In case of VMs, they need to be powered-off.
+
+### Step 3. Decide on the correct copy:  
+This is done by observing the afr changelog extended attributes of the file on
+the bricks using the getfattr command; then identifying the type of split-brain 
+(data split-brain, metadata split-brain, entry split-brain or split-brain due to
+gfid-mismatch); and finally determining which of the bricks contains the 'good copy'
+of the file.  
+```
+getfattr -d -m . -e hex <file-path-on-brick>
+```
+  
+It is also possible that one brick might contain the correct data while the
+other might contain the correct metadata.
+
+### Step 4. Reset the relevant extended attribute on the brick(s) that contains the 'bad copy' of the file data/metadata using the setfattr command.  
+```
+setfattr -n <attribute-name> -v <attribute-value> <file-path-on-brick>
+```
+
+### Step 5. Trigger self-heal on the file by performing lookup from the client:  
+```
+ls -l <file-path-on-gluster-mount>
+```
+
+Detailed Instructions for steps 3 through 5:  
+===========================================
+To understand how to resolve split-brain we need to know how to interpret the
+afr changelog extended attributes.
+
+Execute `getfattr -d -m . -e hex <file-path-on-brick>`
+
+Example:  
+```
+[root@store3 ~]# getfattr -d -e hex -m. brick-a/file.txt  
+\#file: brick-a/file.txt  
+security.selinux=0x726f6f743a6f626a6563745f723a66696c655f743a733000  
+trusted.afr.vol-client-2=0x000000000000000000000000  
+trusted.afr.vol-client-3=0x000000000200000000000000  
+trusted.gfid=0x307a5c9efddd4e7c96e94fd4bcdcbd1b  
+```
+
+The extended attributes with `trusted.afr.<volname>-client-<subvolume-index>`
+are used by afr to maintain changelog of the file.The values of the
+`trusted.afr.<volname>-client-<subvolume-index>` are calculated by the glusterfs
+client (fuse or nfs-server) processes. When the glusterfs client modifies a file
+or directory, the client contacts each brick and updates the changelog extended 
+attribute according to the response of the brick.
+
+`subvolume-index` is nothing but (brick number - 1) in
+`gluster volume info <volname>` output.
+
+Example:
+```
+[root@pranithk-laptop ~]# gluster volume info vol  
+ Volume Name: vol  
+ Type: Distributed-Replicate  
+ Volume ID: 4f2d7849-fbd6-40a2-b346-d13420978a01  
+ Status: Created  
+ Number of Bricks: 4 x 2 = 8  
+ Transport-type: tcp  
+ Bricks:  
+ brick-a: pranithk-laptop:/gfs/brick-a  
+ brick-b: pranithk-laptop:/gfs/brick-b  
+ brick-c: pranithk-laptop:/gfs/brick-c  
+ brick-d: pranithk-laptop:/gfs/brick-d  
+ brick-e: pranithk-laptop:/gfs/brick-e  
+ brick-f: pranithk-laptop:/gfs/brick-f  
+ brick-g: pranithk-laptop:/gfs/brick-g  
+ brick-h: pranithk-laptop:/gfs/brick-h  
+```
+
+In the example above:  
+```
+Brick             |    Replica set        |    Brick subvolume index
+----------------------------------------------------------------------------
+-/gfs/brick-a     |       0               |       0
+-/gfs/brick-b     |       0               |       1
+-/gfs/brick-c     |       1               |       2
+-/gfs/brick-d     |       1               |       3
+-/gfs/brick-e     |       2               |       4
+-/gfs/brick-f     |       2               |       5
+-/gfs/brick-g     |       3               |       6
+-/gfs/brick-h     |       3               |       7
+```
+
+Each file in a brick maintains the changelog of itself and that of the files
+present in all the other bricks in it's replica set as seen by that brick.
+
+In the example volume given above, all files in brick-a will have 2 entries, 
+one for itself and the other for the file present in it's replica pair, i.e.brick-b:  
+```
+trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for itself (brick-a)  
+trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for brick-b as seen by brick-a  
+```
+Likewise, all files in brick-b will have:  
+```
+trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for brick-a as seen by brick-b  
+trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for itself (brick-b)  
+```
+
+The same can be extended for other replica pairs.  
+
+Interpreting Changelog (roughly pending operation count) Value:  
+Each extended attribute has a value which is 24 hexa decimal digits.  
+First 8 digits represent changelog of data. Second 8 digits represent changelog
+of metadata. Last 8 digits represent Changelog of directory entries.  
+
+Pictorially representing the same, we have:
+```
+0x 000003d7 00000001 00000000
+        |      |       |
+        |      |        \_ changelog of directory entries
+        |       \_ changelog of metadata
+         \ _ changelog of data
+```
+         
+
+For Directories metadata and entry changelogs are valid.
+For regular files data and metadata changelogs are valid.
+For special files like device files etc metadata changelog is valid.
+When a file split-brain happens it could be either data split-brain or
+meta-data split-brain or both. When a split-brain happens the changelog of the
+file would be something like this:  
+
+Example:(Lets consider both data, metadata split-brain on same file).  
+```
+[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
+getfattr: Removing leading '/' from absolute path names  
+\#file: gfs/brick-a/a  
+trusted.afr.vol-client-0=0x000000000000000000000000  
+trusted.afr.vol-client-1=0x000003d70000000100000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57   
+\#file: gfs/brick-b/a  
+trusted.afr.vol-client-0=0x000003b00000000100000000  
+trusted.afr.vol-client-1=0x000000000000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+```
+
+### Observations:
+
+#### According to changelog extended attributes on file /gfs/brick-a/a:  
+The first 8 digits of trusted.afr.vol-client-0 are all
+zeros (0x00000000................), and the first 8 digits of
+trusted.afr.vol-client-1 are not all zeros (0x000003d7................).
+So the changelog on /gfs/brick-a/a implies that some data operations succeeded
+on itself but failed on /gfs/brick-b/a.
+
+The second 8 digits of trusted.afr.vol-client-0 are
+all zeros (0x........00000000........), and the second 8 digits of
+trusted.afr.vol-client-1 are not all zeros (0x........00000001........).
+So the changelog on /gfs/brick-a/a implies that some metadata operations succeeded 
+on itself but failed on /gfs/brick-b/a.
+
+#### According to Changelog extended attributes on file /gfs/brick-b/a:  
+The first 8 digits of trusted.afr.vol-client-0 are not all
+zeros (0x000003b0................), and the first 8 digits of
+trusted.afr.vol-client-1 are all zeros (0x00000000................).
+So the changelog on /gfs/brick-b/a implies that some data operations succeeded
+on itself but failed on /gfs/brick-a/a.
+
+The second 8 digits of trusted.afr.vol-client-0 are not
+all zeros (0x........00000001........), and the second 8 digits of
+trusted.afr.vol-client-1 are all zeros (0x........00000000........).
+So the changelog on /gfs/brick-b/a implies that some metadata operations succeeded
+on itself but failed on /gfs/brick-a/a.
+
+Since both the copies have data, metadata changes that are not on the other
+file, it is in both data and metadata split-brain.
+
+Deciding on the correct copy:  
+-----------------------------
+The user may have to inspect stat,getfattr output of the files to decide which 
+metadata to retain and contents of the file to decide which data to retain.
+Continuing with the example above, lets say we want to retain the data
+of /gfs/brick-a/a and metadata of /gfs/brick-b/a.
+
+Resetting the relevant changelogs to resolve the split-brain:  
+-------------------------------------------------------------
+For resolving data-split-brain:  
+We need to change the changelog extended attributes on the files as if some data
+operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a. But
+/gfs/brick-b/a should NOT have any changelog which says some data operations
+succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a. We need to reset the
+data part of the changelog on trusted.afr.vol-client-0 of /gfs/brick-b/a.
+
+For resolving metadata-split-brain:  
+We need to change the changelog extended attributes on the files as if some
+metadata operations succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a.
+But /gfs/brick-a/a should NOT have any changelog which says some metadata
+operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a.
+We need to reset metadata part of the changelog on
+trusted.afr.vol-client-1 of /gfs/brick-a/a
+
+So, the intended changes are:  
+On /gfs/brick-b/a:  
+For trusted.afr.vol-client-0  
+0x000003b00000000100000000 to 0x000000000000000100000000  
+(Note that the metadata part is still not all zeros)  
+Hence execute
+`setfattr -n trusted.afr.vol-client-0 -v 0x000000000000000100000000 /gfs/brick-b/a`
+
+On /gfs/brick-a/a:  
+For trusted.afr.vol-client-1  
+0x0000000000000000ffffffff to 0x000003d70000000000000000  
+(Note that the data part is still not all zeros)  
+Hence execute  
+`setfattr -n trusted.afr.vol-client-1 -v 0x000003d70000000000000000 /gfs/brick-a/a`
+
+Thus after the above operations are done, the changelogs look like this:  
+```
+[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
+getfattr: Removing leading '/' from absolute path names  
+\#file: gfs/brick-a/a  
+trusted.afr.vol-client-0=0x000000000000000000000000  
+trusted.afr.vol-client-1=0x000003d70000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+
+\#file: gfs/brick-b/a  
+trusted.afr.vol-client-0=0x000000000000000100000000  
+trusted.afr.vol-client-1=0x000000000000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+```
+
+Triggering Self-heal:
+---------------------
+Perform `ls -l <file-path-on-gluster-mount>` to trigger healing.
+
+Fixing Directory entry split-brain:
+----------------------------------
+Afr has the ability to conservatively merge different entries in the directories
+when there is a split-brain on directory.
+If on one brick directory 'd' has entries '1', '2' and has entries '3', '4' on
+the other brick then afr will merge all of the entries in the directory to have
+'1', '2', '3', '4' entries in the same directory.
+(Note: this may result in deleted files to re-appear in case the split-brain
+happens because of deletion of files on the directory)
+Split-brain resolution needs human intervention when there is at least one entry
+which has same file name but different gfid in that directory.
+Example:  
+On brick-a the directory has entries '1' (with gfid g1), '2' and on brick-b
+directory has entries '1' (with gfid g2) and '3'.
+These kinds of directory split-brains need human intervention to resolve.
+The user needs to remove either file '1' on brick-a or the file '1' on brick-b
+to resolve the split-brain. In addition, the corresponding gfid-link file also
+needs to be removed.The gfid-link files are present in the .glusterfs folder
+in the top-level directory of the brick. If the gfid of the file is
+0x307a5c9efddd4e7c96e94fd4bcdcbd1b (the trusted.gfid extended attribute got
+from the getfattr command earlier),the gfid-link file can be found at
+`/gfs/brick-a/.glusterfs/30/7a/307a5c9efddd4e7c96e94fd4bcdcbd1b`
+
+#### Word of caution:
+Before deleting the gfid-link, we have to ensure that there are no hard links
+to the file present on that brick. If hard-links exist,they must be deleted as
+well.
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
new file mode 100644
index 00000000000..9dfdce15fad
--- /dev/null
+++ b/doc/debugging/statedump.md
@@ -0,0 +1,414 @@
+# Statedump
+Statedump is a file generated by glusterfs process with different data structure state which may contain the active inodes, fds, mempools, iobufs, memory allocation stats of different types of datastructures per xlator etc.
+
+## How to generate statedump
+We can find the directory where statedump files are created using `gluster --print-statedumpdir` command.
+Create that directory if not already present based on the type of installation.
+Lets call this directory `statedump-directory`.
+
+We can generate statedump using `kill -USR1 <pid-of-gluster-process>`.
+gluster-process is nothing but glusterd/glusterfs/glusterfsd process.
+
+There are also commands to generate statedumps for brick processes/nfs server/quotad
+
+For bricks: 
+```
+gluster volume statedump <volname>
+```
+
+For nfs server:
+```
+gluster volume statedump <volname> nfs
+```
+
+For quotad:
+```
+gluster volume statedump <volname> quotad
+```
+
+For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
+
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+```
+gluster volume statedump <volname> client <hostname>:<process id>
+```
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
+## How to read statedump
+We shall see snippets of each type of statedump.
+
+First and last lines of the file have starting and ending time of writing the statedump file. Times will be in UTC timezone.
+
+mallinfo return status is printed in the following format. Please read man mallinfo for more information about what each field means.
+### Mallinfo
+```
+[mallinfo]
+mallinfo_arena=100020224 /* Non-mmapped space allocated (bytes) */
+mallinfo_ordblks=69467 /* Number of free chunks */
+mallinfo_smblks=449 /* Number of free fastbin blocks */
+mallinfo_hblks=13 /* Number of mmapped regions */
+mallinfo_hblkhd=20144128 /* Space allocated in mmapped regions (bytes) */
+mallinfo_usmblks=0 /* Maximum total allocated space (bytes) */
+mallinfo_fsmblks=39264 /* Space in freed fastbin blocks (bytes) */
+mallinfo_uordblks=96710112 /* Total allocated space (bytes) */
+mallinfo_fordblks=3310112 /* Total free space (bytes) */
+mallinfo_keepcost=133712 /* Top-most, releasable space (bytes) */
+```
+
+### Data structure allocation stats
+For every xlator data structure memory per translator loaded in the call-graph is displayed in the following format:
+
+For xlator with name: glusterfs
+```
+[global.glusterfs - Memory usage] #[global.xlator-name - Memory usage]
+num_types=119 #It shows the number of data types it is using
+```
+
+Now for each data-type it prints the memory usage.
+
+```
+[global.glusterfs - usage-type gf_common_mt_gf_timer_t memusage]
+#[global.xlator-name - usage-type <tag associated with the data-type> memusage]
+size=112 #num_allocs times the sizeof(data-type) i.e. num_allocs * sizeof (data-type)
+num_allocs=2 #Number of allocations of the data-type which are active at the time of taking statedump.
+max_size=168 #max_num_allocs times the sizeof(data-type) i.e. max_num_allocs * sizeof (data-type)
+max_num_allocs=3 #Maximum number of active allocations at any point in the life of the process.
+total_allocs=7 #Number of times this data is allocated in the life of the process.
+```
+
+### Mempools
+
+Mempools are optimization to reduce the number of allocations of a data type. If we create a mem-pool of lets say 1024 elements for a data-type, new elements will be allocated from heap using syscalls like calloc, only if all the 1024 elements in the pool are in active use.
+
+Memory pool allocated by each xlator is displayed in the following format:
+
+```
+[mempool] #Section name
+-----=-----
+pool-name=fuse:fd_t #pool-name=<xlator-name>:<data-type>
+hot-count=1 #number of mempool elements that are in active use. i.e. for this pool it is the number of 'fd_t' s in active use.
+cold-count=1023 #number of mempool elements that are not in use. If a new allocation is required it will be served from here until all the elements in the pool are in use i.e. cold-count becomes 0.
+padded_sizeof=108 #Each mempool element is padded with a doubly-linked-list + ptr of mempool + is-in-use info to operate the pool of elements, this size is the element-size after padding
+pool-misses=0 #Number of times the element had to be allocated from heap because all elements from the pool are in active use.
+alloc-count=314 #Number of times this type of data is allocated through out the life of this process. This may include pool-misses as well.
+max-alloc=3 #Maximum number of elements from the pool in active use at any point in the life of the process. This does *not* include pool-misses.
+cur-stdalloc=0 #Denotes the number of allocations made from heap once cold-count reaches 0, that are yet to be released via mem_put().
+max-stdalloc=0 #Maximum number of allocations from heap that are in active use at any point in the life of the process.
+```
+
+### Iobufs
+```
+[iobuf.global]
+iobuf_pool=0x1f0d970 #The memory pool for iobufs
+iobuf_pool.default_page_size=131072 #The default size of iobuf (if no iobuf size is specified the default size is allocated)
+#iobuf_arena: One arena represents a group of iobufs of a particular size
+iobuf_pool.arena_size=12976128 # The initial size of the iobuf pool (doesn't include the stdalloc'd memory or the newly added arenas)
+iobuf_pool.arena_cnt=8 #Total number of arenas in the pool
+iobuf_pool.request_misses=0 #The number of iobufs that were stdalloc'd (as they exceeded the default max page size provided by iobuf_pool).
+```
+
+There are 3 lists of arenas:
+
+1. Arena list: arenas allocated during iobuf pool creation and the arenas that are in use(active_cnt != 0) will be part of this list.
+2. Purge list: arenas that can be purged(no active iobufs, active_cnt == 0).
+3. Filled list: arenas without free iobufs.
+
+```
+[purge.1] #purge.<S.No.>
+purge.1.mem_base=0x7fc47b35f000 #The address of the arena structure
+purge.1.active_cnt=0 #The number of iobufs active in that arena
+purge.1.passive_cnt=1024 #The number of unused iobufs in the arena
+purge.1.alloc_cnt=22853 #Total allocs in this pool(number of times the iobuf was allocated from this arena)
+purge.1.max_active=7 #Max active iobufs from this arena, at any point in the life of this process.
+purge.1.page_size=128 #Size of all the iobufs in this arena.
+
+[arena.5] #arena.<S.No.>
+arena.5.mem_base=0x7fc47af1f000
+arena.5.active_cnt=0
+arena.5.passive_cnt=64
+arena.5.alloc_cnt=0
+arena.5.max_active=0
+arena.5.page_size=32768
+```
+
+If the active_cnt of any arena is non zero, then the statedump will also have the iobuf list.
+```
+[arena.6.active_iobuf.1] #arena.<S.No>.active_iobuf.<iobuf.S.No.>
+arena.6.active_iobuf.1.ref=1 #refcount of the iobuf
+arena.6.active_iobuf.1.ptr=0x7fdb921a9000 #address of the iobuf
+
+[arena.6.active_iobuf.2]
+arena.6.active_iobuf.2.ref=1
+arena.6.active_iobuf.2.ptr=0x7fdb92189000
+```
+
+At any given point in time if there are lots of filled arenas then that could be a sign of iobuf leaks.
+
+### Call stack
+All the fops received by gluster are handled using call-stacks. Call stack contains the information about uid/gid/pid etc of the process that is executing the fop. Each call-stack contains different call-frames per xlator which handles that fop.
+
+```
+[global.callpool.stack.3] #global.callpool.stack.<Serial-Number>
+stack=0x7fc47a44bbe0 #Stack address
+uid=0 #Uid of the process which is executing the fop
+gid=0 #Gid of the process which is executing the fop
+pid=6223 #Pid of the process which is executing the fop
+unique=2778 #Xlators like afr do copy_frame and perform the operation in a different stack, this id is useful to find out the stacks that are inter-related because of copy-frame
+lk-owner=0000000000000000 #Some of the fuse fops have lk-owner.
+op=LOOKUP #Fop
+type=1 #Type of the op i.e. FOP/MGMT-OP
+cnt=9 #Number of frames in this stack.
+```
+### Call-frame
+Each frame will have information about which xlator the frame belongs to, what is the function it wound to/from and will be unwind to. It also mentions if the unwind happened or not. If we observe hangs in the system and want to find out which xlator is causing it. Take a statedump and see what is the final xlator which is yet to be unwound.
+
+```
+[global.callpool.stack.3.frame.2]#global.callpool.stack.<stack-serial-number>.frame.<frame-serial-number>
+frame=0x7fc47a611dbc #Frame address
+ref_count=0 #Incremented at the time of wind and decremented at the time of unwind.
+translator=r2-client-1 #Xlator this frame belongs to
+complete=0 #if this value is 1 that means this frame is already unwound. 0 if it is yet to unwind.
+parent=r2-replicate-0 #Parent xlator of this frame
+wind_from=afr_lookup #Parent xlator function from which the wind happened
+wind_to=priv->children[i]->fops->lookup
+unwind_to=afr_lookup_cbk #Parent xlator function to which unwind happened
+```
+
+### History of operations in Fuse
+
+Fuse maintains history of operations that happened in fuse.
+
+```
+[xlator.mount.fuse.history]
+TIME=2014-07-09 16:44:57.523364
+message=[0] fuse_release: RELEASE(): 4590:, fd: 0x1fef0d8, gfid: 3afb4968-5100-478d-91e9-76264e634c9f
+
+TIME=2014-07-09 16:44:57.523373
+message=[0] send_fuse_err: Sending Success for operation 18 on inode 3afb4968-5100-478d-91e9-76264e634c9f
+
+TIME=2014-07-09 16:44:57.523394
+message=[0] fuse_getattr_resume: 4591, STAT, path: (/iozone.tmp), gfid: (3afb4968-5100-478d-91e9-76264e634c9f)
+```
+
+### Xlator configuration
+```
+[cluster/replicate.r2-replicate-0] #Xlator type, name information
+child_count=2 #Number of children to the xlator
+#Xlator specific configuration below
+child_up[0]=1
+pending_key[0]=trusted.afr.r2-client-0
+child_up[1]=1
+pending_key[1]=trusted.afr.r2-client-1
+data_self_heal=on
+metadata_self_heal=1
+entry_self_heal=1
+data_change_log=1
+metadata_change_log=1
+entry-change_log=1
+read_child=1
+favorite_child=-1
+wait_count=1
+```
+
+### Graph/inode table
+```
+[active graph - 1]
+
+conn.1.bound_xl./data/brick01a/homegfs.hashsize=14057
+conn.1.bound_xl./data/brick01a/homegfs.name=/data/brick01a/homegfs/inode
+conn.1.bound_xl./data/brick01a/homegfs.lru_limit=16384 #Least recently used size limit
+conn.1.bound_xl./data/brick01a/homegfs.active_size=690 #Number of inodes undergoing some kind of fop to be precise on which there is at least one ref.
+conn.1.bound_xl./data/brick01a/homegfs.lru_size=183 #Number of inodes present in lru list
+conn.1.bound_xl./data/brick01a/homegfs.purge_size=0 #Number of inodes present in purge list
+```
+
+### Inode
+```
+[conn.1.bound_xl./data/brick01a/homegfs.active.324] #324th inode in active inode list
+gfid=e6d337cf-97eb-44b3-9492-379ba3f6ad42 #Gfid of the inode
+nlookup=13 #Number of times lookups happened from the client or from fuse kernel
+fd-count=4 #Number of fds opened on the inode
+ref=11 #Number of refs taken on the inode
+ia_type=1 #Type of the inode. This should be changed to some string :-(
+Ref by xl:.patchy-md-cache=11 #Further this there will be a list of xlators, and the ref count taken by each of them on this inode at the time of statedump
+
+[conn.1.bound_xl./data/brick01a/homegfs.lru.1] #1st inode in lru list. Note that ref count is zero for these inodes.
+gfid=5114574e-69bc-412b-9e52-f13ff087c6fc
+nlookup=5
+fd-count=0
+ref=0
+ia_type=2
+Ref by xl:.fuse=1
+Ref by xl:.patchy-client-0=-1
+```
+### Inode context
+For each inode per xlator some context could be stored. This context can also be printed in the statedump. Here is the inode ctx of locks xlator
+```
+[xlator.features.locks.homegfs-locks.inode]
+path=/homegfs/users/dfrobins/gfstest/r4/SCRATCH/fort.5102 - path of the file
+mandatory=0
+inodelk-count=5 #Number of inode locks
+lock-dump.domain.domain=homegfs-replicate-0:self-heal #Domain name where self-heals take locks to prevent more than one heal on the same file
+inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=0, len=0, pid = 18446744073709551615, owner=080b1ada117f0000, client=0xb7fc30, connection-id=compute-30-029.com-3505-2014/06/29-14:46:12:477358-homegfs-client-0-0-1, granted at Sun Jun 29 11:01:00 2014 #Active lock information
+
+inodelk.inodelk[1](BLOCKED)=type=WRITE, whence=0, start=0, len=0, pid = 18446744073709551615, owner=c0cb091a277f0000, client=0xad4f10, connection-id=gfs01a.com-4080-2014/06/29-14:41:36:917768-homegfs-client-0-0-0, blocked at Sun Jun 29 11:04:44 2014 #Blocked lock information
+
+lock-dump.domain.domain=homegfs-replicate-0:metadata #Domain name where metadata operations take locks to maintain replication consistency
+lock-dump.domain.domain=homegfs-replicate-0 #Domain name where entry/data operations take locks to maintain replication consistency
+inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=11141120, len=131072, pid = 18446744073709551615, owner=080b1ada117f0000, client=0xb7fc30, connection-id=compute-30-029.com-3505-2014/06/29-14:46:12:477358-homegfs-client-0-0-1, granted at Sun Jun 29 11:10:36 2014 #Active lock information
+```
+
+## FAQ
+### How to debug Memory leaks using statedump?
+
+#### Using memory accounting feature:
+
+[Bug 1120151](https://bugzilla.redhat.com/show_bug.cgi?id=1120151) is one of the bugs which was debugged using statedump to see which data-structure is leaking. Here is the process used to find what the leak is using statedump. According to the bug the observation is that the process memory usage is increasing whenever one of the bricks is wiped in a replicate volume and a `full` self-heal is invoked to heal the contents. Statedump of the process is taken using `kill -USR1 <pid-of-gluster-self-heal-daemon>`.
+```
+grep -w num_allocs glusterdump.5225.dump.1405493251
+num_allocs=77078
+num_allocs=87070
+num_allocs=117376
+....
+
+grep hot-count glusterdump.5225.dump.1405493251
+hot-count=16384
+hot-count=16384
+hot-count=4095
+....
+
+Find the occurrences in the statedump file to figure out the tags.
+```
+grep of the statedump revealed too many allocations for the following data-types under replicate,
+
+1. gf_common_mt_asprintf
+2. gf_common_mt_char
+3. gf_common_mt_mem_pool.
+
+After checking afr-code for allocations with tag `gf_common_mt_char` found `data-self-heal` code path does not free one such allocated memory. `gf_common_mt_mem_pool` suggests that there is a leak in pool memory. `replicate-0:dict_t`, `glusterfs:data_t` and `glusterfs:data_pair_t` pools are using lot of memory, i.e. cold_count is `0` and too many allocations. Checking source code of dict.c revealed that `key` in `dict` is allocated with `gf_common_mt_char` i.e. `2.` tag and value is created using gf_asprintf which in-turn uses `gf_common_mt_asprintf` i.e. `1.`. Browsing the code for leak in self-heal code paths lead to a line which over-writes a variable with new dictionary even when it was already holding a reference to another dictionary. After fixing these leaks, ran the same test to verify that none of the `num_allocs` are increasing even after healing 10,000 files directory hierarchy in statedump of self-heal daemon.
+Please check this [patch](http://review.gluster.org/8316) for more info about the fix.
+
+#### Debugging leaks in memory pools:
+Statedump output of memory pools was used to test and verify the fixes to [Bug 1134221](https://bugzilla.redhat.com/show_bug.cgi?id=1134221). On code analysis, dict_t objects were found to be leaking (in terms of not being unref'd enough number of times, during name self-heal. The test involved creating 100 files on plain replicate volume, removing them from one of the brick's backend, and then triggering lookup on them from the mount point. Statedump of the mount process was taken before executing the test case and after it, after compiling glusterfs with -DDEBUG flags (to have cold count set to 0 by default).
+
+Statedump output of the fuse mount process before the test case was executed:
+
+```
+
+pool-name=glusterfs:dict_t
+hot-count=0
+cold-count=0
+padded_sizeof=140
+alloc-count=33
+max-alloc=0
+pool-misses=33
+cur-stdalloc=14
+max-stdalloc=18
+
+```
+Statedump output of the fuse mount process after the test case was executed:
+
+```
+
+pool-name=glusterfs:dict_t
+hot-count=0
+cold-count=0
+padded_sizeof=140
+alloc-count=2841
+max-alloc=0
+pool-misses=2841
+cur-stdalloc=214
+max-stdalloc=220
+
+```
+Here, with cold count being 0 by default, `cur-stdalloc` indicated the number of `dict_t` objects that were allocated in heap using `mem_get()`, and yet to be freed using `mem_put()` (refer to this [page](../developer-guide/datastructure-mem-pool.md) for more details on how mempool works). After the test case (name selfheal of 100 files), there was a rise in the cur-stdalloc value (from 14 to 214) for `dict_t`.
+
+After these leaks were fixed, glusterfs was again compiled with -DDEBUG flags, and the same steps were performed again and statedump was taken before and after executing the test case, of the mount. This was done to ascertain the validity of the fix. And the following are the results:
+
+Statedump output of the fuse mount process before executing the test case:
+
+```
+pool-name=glusterfs:dict_t
+hot-count=0
+cold-count=0
+padded_sizeof=140
+alloc-count=33
+max-alloc=0
+pool-misses=33
+cur-stdalloc=14
+max-stdalloc=18
+
+```
+Statedump output of the fuse mount process after executing the test case:
+
+```
+pool-name=glusterfs:dict_t
+hot-count=0
+cold-count=0
+padded_sizeof=140
+alloc-count=2837
+max-alloc=0
+pool-misses=2837
+cur-stdalloc=14
+max-stdalloc=119
+
+```
+The value of cur-stdalloc remained 14 before and after the test, indicating that the fix indeed does what it's supposed to do.
+
+### How to debug hangs because of frame-loss?
+[Bug 994959](https://bugzilla.redhat.com/show_bug.cgi?id=994959) is one of the bugs where statedump was helpful in finding where the frame was lost. Here is the process used to find where the hang is using statedump.
+When the hang was observed, statedumps are taken for all the processes. On mount's statedump the following stack is shown:
+```
+[global.callpool.stack.1.frame.1]
+ref_count=1
+translator=fuse
+complete=0
+
+[global.callpool.stack.1.frame.2]
+ref_count=0
+translator=r2-client-1
+complete=1 <<----- Client xlator completed the readdirp call and unwound to afr
+parent=r2-replicate-0
+wind_from=afr_do_readdir
+wind_to=children[call_child]->fops->readdirp
+unwind_from=client3_3_readdirp_cbk
+unwind_to=afr_readdirp_cbk
+
+[global.callpool.stack.1.frame.3]
+ref_count=0
+translator=r2-replicate-0
+complete=0 <<---- Afr xlator is not unwinding for some reason.
+parent=r2-dht
+wind_from=dht_do_readdir
+wind_to=xvol->fops->readdirp
+unwind_to=dht_readdirp_cbk
+
+[global.callpool.stack.1.frame.4]
+ref_count=1
+translator=r2-dht
+complete=0
+parent=r2-io-cache
+wind_from=ioc_readdirp
+wind_to=FIRST_CHILD(this)->fops->readdirp
+unwind_to=ioc_readdirp_cbk
+
+[global.callpool.stack.1.frame.5]
+ref_count=1
+translator=r2-io-cache
+complete=0
+parent=r2-quick-read
+wind_from=qr_readdirp
+wind_to=FIRST_CHILD (this)->fops->readdirp
+unwind_to=qr_readdirp_cbk
+
+```
+`unwind_to` shows that call was unwound to `afr_readdirp_cbk` from client xlator.
+Inspecting that function revealed that afr is not unwinding the stack when fop failed.
+Check this [patch](http://review.gluster.org/5531) for more info about the fix.
diff --git a/doc/developer-guide/Language-Bindings.md b/doc/developer-guide/Language-Bindings.md
new file mode 100644
index 00000000000..951f5fae2f6
--- /dev/null
+++ b/doc/developer-guide/Language-Bindings.md
@@ -0,0 +1,45 @@
+# Language Bindings
+GlusterFS 3.4 introduced the libgfapi client API for C programs. This
+page lists bindings to the libgfapi C library from other languages.
+
+Go
+--
+
+-   [gogfapi](https://github.com/gluster/gogfapi) - Go language bindings
+    for libgfapi, aiming to provide an api consistent with the default
+    Go file apis.
+
+Java
+----
+
+-   [libgfapi-jni](https://github.com/semiosis/libgfapi-jni/) - Low
+    level JNI binding for libgfapi
+-   [glusterfs-java-filesystem](https://github.com/semiosis/glusterfs-java-filesystem)
+    - High level NIO.2 FileSystem Provider implementation for the Java
+    platform
+-   [libgfapi-java-io](https://github.com/gluster/libgfapi-java-io) -
+    Java bindings for libgfapi, similar to java.io
+
+Python
+------
+
+-   [libgfapi-python](https://github.com/gluster/libgfapi-python) -
+    Libgfapi bindings for Python
+
+Ruby
+----
+
+-   [libgfapi-ruby](https://github.com/spajus/libgfapi-ruby) - Libgfapi
+    bindings for Ruby using FFI
+
+Rust
+----
+
+-   [gfapi-sys](https://github.com/cholcombe973/Gfapi-sys) - Libgfapi
+    bindings for Rust using FFI
+
+Perl
+----
+
+-   [libgfapi-perl](https://github.com/gluster/libgfapi-perl) - Libgfapi
+    bindings for Perl using FFI
diff --git a/doc/developer-guide/README.md b/doc/developer-guide/README.md
new file mode 100644
index 00000000000..aaf9c7476b0
--- /dev/null
+++ b/doc/developer-guide/README.md
@@ -0,0 +1,81 @@
+Developers
+==========
+
+### From GlusterDocumentation
+
+Contributing to the Gluster community
+-------------------------------------
+
+Are you itching to send in patches and participate as a developer in the
+Gluster community? Here are a number of starting points for getting
+involved. We don't require a signed contributor license agreement or
+copyright assignment, but we do require a "signed-off-by" line on each
+code check-in.
+
+-   [License
+    Change](http://www.gluster.org/2012/05/glusterfs-license-change/) -
+    we recently changed the client library code to a dual license under
+    the GPL v2 and the LGPL v3 or later
+-   [GlusterFS Coding Standards](./coding-standard.md)
+
+-   If you are not sure of where to start, and what to do, we have a small
+    write-up on what you can pick. [Check it out](./options-to-contribute.md)
+
+
+Adding File operations
+----------------------
+
+-   [Steps to be followed when adding a new FOP to GlusterFS ](./adding-fops.md)
+
+Automatic File Replication
+--------------------------
+
+-   [Cluster/afr translator](./afr.md)
+-   [History of Locking in AFR](./afr-locks-evolution.md)
+-   [Self heal Daemon](./afr-self-heal-daemon.md)
+
+Data Structures
+---------------
+
+-   [inode data structure](./datastructure-inode.md)
+-   [iobuf data structure](./datastructure-iobuf.md)
+-   [mem-pool data structure](./datastructure-mem-pool.md)
+
+Find the gfapi symbol versions [here](./gfapi-symbol-versions.md)
+
+Daemon Management Framework
+---------------------------
+
+-   [How to introduce new daemons using daemon management framework](./daemon-management-framework.md)
+
+Translators
+-----------
+
+-   [Performance/write-Behind Translator](./write-behind.md)
+-   [Translator Development](./translator-development.md)
+-   [Storage/posix Translator](./posix.md)
+
+
+Brick multiplex
+---------------
+
+-   [Brick mux resource reduction](./brickmux-thread-reduction.md)
+
+Fuse
+----
+
+-   [Interrupt handling](./fuse-interrupt.md)
+
+Testing/Debugging
+-----------------
+
+-   [Unit Tests in GlusterFS](./unittest.md)
+-   [Using the Gluster Test
+    Framework](./Using-Gluster-Test-Framework.md) - Step by
+    step instructions for running the Gluster Test Framework
+-   [Coredump Analysis](../debugging/analyzing-regression-cores.md) - Steps to analize coredumps generated by regression machines.
+-   [Identifying Resource Leaks](./identifying-resource-leaks.md)
+
+Release Process
+---------------
+-   [Versioning](./versioning.md)
diff --git a/doc/developer-guide/Using-Gluster-Test-Framework.md b/doc/developer-guide/Using-Gluster-Test-Framework.md
new file mode 100644
index 00000000000..d2bb1c391da
--- /dev/null
+++ b/doc/developer-guide/Using-Gluster-Test-Framework.md
@@ -0,0 +1,271 @@
+# USing Gluster Test Framwork
+Description
+-----------
+
+The Gluster Test Framework, is a suite of scripts used for regression
+testing of Gluster.
+
+It runs well on RHEL and CentOS (possibly Fedora too, presently being
+tested), and is automatically run against every patch submitted to
+Gluster [for review](http://review.gluster.org).
+
+The Gluster Test Framework is part of the main Gluster code base, living
+under the "tests" subdirectory:
+
+		http://git.gluster.org/?p=glusterfs.git;a=summary
+
+WARNING
+-------
+
+Running the Gluster Test Framework deletes “/var/lib/glusterd/\*”.
+
+**DO NOT run it on a server with any data.**
+
+Preparation steps for Ubuntu 14.04 LTS
+--------------------------------------
+
+1. \# apt-get install dbench git libacl1-dev mock nfs-common
+nfs-kernel-server libtest-harness-perl libyajl-dev xfsprogs psmisc attr
+acl lvm2 rpm
+
+2. \# apt-get install python-webob python-paste python-sphinx
+
+3. \# apt-get install autoconf automake bison dos2unix flex libfuse-dev
+libaio-dev libibverbs-dev librdmacm-dev libtool libxml2-dev
+libxml2-utils liblvm2-dev make libssl-dev pkg-config libpython-dev
+python-eventlet python-netifaces python-simplejson python-pyxattr
+libreadline-dev tar
+
+4) Install cmockery2 from github (https://github.com/lpabon/cmockery2)
+and compile and make install as in Readme
+
+5)
+
+		sudo groupadd mock
+		sudo useradd -g mock mock
+
+6) mkdir /var/run/gluster
+
+**Note**: redhat-rpm-config package is not found in ubuntu
+
+Preparation steps for CentOS 7 (only)
+-------------------------------------
+
+1. Install EPEL:
+
+		$ sudo yum install -y http://epel.mirror.net.in/epel/7/x86_64/e/epel-release-7-1.noarch.rpm
+
+2. Install the CentOS 7.x dependencies:
+
+		$ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils perl-Test-Harness yajl xfsprogs psmisc
+
+		$ sudo yum install -y --enablerepo=epel python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config
+
+==\> Despite below missing packages it worked for me
+
+		No package python-webob1.0 available.
+		No package python-paste-deploy1.5 available.
+		No package python-sphinx10 available.
+
+		$ sudo yum install -y --enablerepo=epel autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \
+		 librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \
+		 python-devel python-eventlet python-netifaces python-paste-deploy \
+		 python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \
+		 tar
+
+3. Create the mock user
+
+		$ sudo useradd -g mock mock
+
+Preparation steps for CentOS 6.3+ (only)
+----------------------------------------
+
+1. Install EPEL:
+
+		$ sudo yum install -y http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+2. Install the CentOS 6.x dependencies:
+
+		$ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils perl-Test-Harness yajl xfsprogs
+		$ sudo yum install -y --enablerepo=epel python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config
+		$ sudo yum install -y --enablerepo=epel autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \
+		  librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \
+		  python-devel python-eventlet python-netifaces python-paste-deploy \
+		  python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \
+		  tar
+
+3. Create the mock user
+
+		$ sudo useradd -g mock mock
+
+Preparation steps for RHEL 6.3+ (only)
+--------------------------------------
+
+1. Ensure you have the "Scalable Filesystem Support" group installed
+
+This provides the xfsprogs package, which is required by the test
+framework.
+
+2. Install EPEL:
+
+		$ sudo yum install -y http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+3. Install the CentOS 6.x dependencies:
+
+		$ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils yajl perl-Test-Harness
+		$ sudo yum install -y --enablerepo=rhel-6-server-optional-rpms python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config
+		$ sudo yum install -y --disablerepo=rhs* --enablerepo=*optional-rpms autoconf \
+		  automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \
+		  librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \
+		  python-devel python-eventlet python-netifaces python-paste-deploy \
+		  python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \
+		  tar
+
+4. Create the mock user
+
+		$ sudo useradd -g mock mock
+
+Preparation steps for Fedora 16-19 (only)
+-----------------------------------------
+
+**Still in development**
+
+1. Install the Fedora dependencies:
+
+		$ sudo yum install -y attr cmockery2-devel dbench git mock nfs-utils perl-Test-Harness psmisc xfsprogs
+		$ sudo yum install -y python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config
+		$ sudo yum install -y autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \
+		  librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \
+		  python-devel python-eventlet python-netifaces python-paste-deploy \
+		  python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \
+		  tar
+
+3. Create the mock user
+
+		$ sudo useradd -g mock mock
+
+Common steps
+------------
+
+1. Ensure DNS for your server is working
+
+The Gluster Test Framework fails miserably if the full domain name for
+your server doesn't resolve back to itself.
+
+If you don't have a working DNS infrastructure in place, adding an entry
+for your server to its /etc/hosts file will work.
+
+2. Install the version of Gluster you are testing
+
+Either install an existing set of rpms:
+
+		$ sudo yum install [your gluster rpms here]
+
+Or compile your own ones (fairly easy):
+
+	http://www.gluster.org/community/documentation/index.php/CompilingRPMS
+
+3. Clone the GlusterFS git repository
+
+		$ git clone git://git.gluster.org/glusterfs
+		$ cd glusterfs
+
+Ensure mock can access the directory
+------------------------------------
+
+Some tests run as the user "mock". If the mock user can't access the
+tests subdirectory directory, these tests fail. (rpm.t is one such test)
+
+This is a known gotcha when the git repo is cloned to your home
+directory. Home directories generally don't have world readable
+permissions. You can fix this by adjusting your home directory
+permissions, or placing the git repo somewhere else (with access for the
+mock user).
+
+Running the tests
+-----------------
+
+The tests need to run as root, so they can mount volumes and manage
+gluster processes as needed.
+
+It's also best to run them directly as the root user, instead of through
+sudo. Strange things sporadicly happen (for me) when using the full test
+framework through sudo, that haven't happened (yet) when running
+directly as root. Hangs in dbench particularly, which are part of at
+least one test.
+
+		# ./run-tests.sh
+
+The test framework takes just over 45 minutes to run in a VM here (4
+cpu's assigned, 8GB ram, SSD storage). It may take significantly more or
+less time for you, depending on the hardware and software you're using.
+
+Showing debug information
+-------------------------
+
+To display verbose information while the tests are running, set the
+DEBUG environment variable to 1 prior to running the tests.
+
+		# DEBUG=1 ./run-tests.sh
+
+Log files
+---------
+
+Verbose output from the rpm.t test goes into "rpmbuild-mock.log",
+located in the same directory the test is run from.
+
+Reporting bugs
+--------------
+
+If you hit a bug when running the test framework, **please** create a
+bug report for it on Bugzilla so it gets fixed:
+
+	https://bugzilla.redhat.com/enter_bug.cgi?product=GlusterFS&component=tests
+
+Creating your own tests
+-----------------------
+
+The test scripts are written in bash, with their filenames ending in .t
+instead of .sh.
+
+When creating your own test scripts, create them in an appropriate
+subdirectory under "tests" (eg "bugs" or "features") and use descriptive
+names like "bug-XXXXXXX-checking-feature-X.t"
+
+Also include the "include.rc" file, which defines the test types and
+host/brick/volume defaults:
+
+		. $(dirname $0)/../include.rc
+
+There are 5 test types available at present, but feel free to add more
+if you need something that doesn't yet exist. The test types are
+explained in more detail below.
+
+Also essential is the "cleanup" command, which removes any existing
+Gluster configuration (**without backing it up**), and also kills any
+running gluster processes.
+
+There is a basic test template you can copy, named bug-000000.t in the
+bugs subdirectory:
+
+		$ cp bugs/bug-000000.t somedir/descriptive-name.t
+
+### TEST
+
+-   Example of usage in basic/volume.t
+
+### TEST\_IN\_LOOP
+
+-   Example of usage in basic/rpm.t
+
+### EXPECT
+
+-   Example of usage in basic/volume.t
+
+### EXPECT\_WITHIN
+
+-   Example of usage in basic/volume-status.t
+
+### EXPECT\_KEYWORD
+
+-   Defined in include.rc, but seems to be unused?
diff --git a/doc/developer-guide/adding-fops.md b/doc/developer-guide/adding-fops.md
new file mode 100644
index 00000000000..3f72ed3e23a
--- /dev/null
+++ b/doc/developer-guide/adding-fops.md
@@ -0,0 +1,18 @@
+Adding a new FOP
+================
+
+Steps to be followed when adding a new FOP to GlusterFS:
+
+1. Edit `glusterfs.h` and add a `GF_FOP_*` constant.
+2. Edit `xlator.[ch]` and:
+    * add the new prototype for fop and callback.
+    * edit `xlator_fops` structure.
+3. Edit `xlator.c` and add to fill_defaults.
+4. Edit `protocol.h` and add struct necessary for the new FOP.
+5. Edit `defaults.[ch]` and provide default implementation.
+6. Edit `call-stub.[ch]` and provide stub implementation.
+7. Edit `common-utils.c` and add to gf_global_variable_init().
+8. Edit client-protocol and add your FOP.
+9. Edit server-protocol and add your FOP.
+10. Implement your FOP in any translator for which the default implementation
+    is not sufficient.
diff --git a/doc/developer-guide/afr-locks-evolution.md b/doc/developer-guide/afr-locks-evolution.md
new file mode 100644
index 00000000000..2dabbcfeb13
--- /dev/null
+++ b/doc/developer-guide/afr-locks-evolution.md
@@ -0,0 +1,91 @@
+History of locking in AFR
+--------------------------
+
+GlusterFS has **locks** translator which provides the following internal locking operations called `inodelk`, `entrylk` which are used by afr to achieve synchronization of operations on files or directories that conflict with each other.
+
+`Inodelk` gives the facility for translators in GlusterFS to obtain range (denoted by tuple with **offset**, **length**) locks in a given **domain** for an inode.
+Full file lock is denoted by the tuple (offset: `0`, length: `0`) i.e. length `0` is considered as infinity.
+
+`Entrylk` enables translators of GlusterFS to obtain locks on `name` in a given **domain** for an inode, typically a directory.
+
+The **locks** translator provides both *blocking* and *nonblocking* variants and of these locks.
+
+
+AFR makes use of locks xlator extensively:
+
+1)For FOPS (from clients)
+-----------------------
+* Data transactions take inode locks on data domain, Let's refer to this domain name as DATA_DOMAIN.
+
+  So locking for writes would be something like this:`inodelk(offset,length, DATA_DOMAIN)`
+  For truncating a file to zero, it would be `inodelk(0,0,DATA_DOMAIN)`
+
+* Metadata transactions (chown/chmod) also take inode locks but on a special range on metadata domain,
+  i.e.`(LLONG_MAX-1 , 0, METADATA_DOMAIN).`
+
+* Entry transactions (create, mkdir, rmdir,unlink, symlink, link,rename) take entrylk on `(name, parent inode)`.
+
+
+2)For self heal:
+-------------
+* For Metadata self-heal, it is the same.  i.e.`inodelk(LLONG_MAX-1 , 0, METADATA_DOMAIN)`.
+* For Entry self-heal, it is `entrylk(NULL name, parent inode)`. Specifying NULL for the name takes full lock on the directory referred to by the inode.
+* For data self-heal, there is a bit of history as to how locks evolved:
+
+### Initial version (say version 1) :
+There was no concept of selfheal daemon (shd). Only client lookups triggered heals. so AFR always took `inodelk(0,0,DATA_DOMAIN)` for healing. The issue with this approach was that when heal was in progress, I/O from clients was blocked .
+
+### version 2:
+shd was introduced. We needed to allow I/O to go through when heal was going,provided the ranges did not overlap. To that extent, the following approach was adopted:
+
++ 1.shd takes (full inodelk in DATA_DOMAIN). Thus client FOPS are blocked and cannot modify changelog-xattrs
++ 2.shd inspects xattrs to determine source/sink
++ 3.shd takes a chunk inodelk(0-128kb) again in DATA_DOMAIN (locks xlator allows overlapping locks if lock owner is the same).
++ 4.unlock full lock
++ 5.heal
++ 6.take next chunk lock(129-256kb)
++ 7.unlock 1st chunk lock, heal the second chunk and so on.
+
+
+Thus after 4, any client FOP could write to regions that was not currently under heal. The exception was truncate (to size 0) because it needs full file lock and will always block because some chunk is always under lock by the shd until heal completes.
+
+Another issue was that 2 shds could run in parallel. Say SHD1 and SHD2 compete for step 1. Let SHD1 win. It proceeds and completes step 4. Now SHD2 also succeeds in step 1, continues all steps. Thus at the end both shds will decrement the changelog leading to negative values in it)  
+
+### version 3
+To prevent parallel self heals, another domain was introduced, let us call it SELF_HEAL_DOMAIN. With this domain, the following approach was adopted and is **the approach currently in use**:
+
++ 1.shd takes (full inodelk on SELF_HEAL_DOMAIN)
++ 2.shd takes (full inodelk on DATA_DOMAIN)
++ 3.shd inspects xattrs  to determine source/sink
++ 4.unlock full lock on DATA_DOMAIN
++ 5.take chunk lock(0-128kb) on DATA_DOMAIN
++ 6.heal
++ 7.take next chunk lock(129-256kb) on DATA_DOMAIN
++ 8.unlock 1st chunk lock, heal and so on.
++ 9.Finally release full lock on SELF_HEAL_DOMAIN
+
+Thus until one shd completes step 9, another shd cannot start step 1, solving the problem of simultaneous heals.
+Note that the issue of truncate (to zero) FOP hanging still remains.
+Also there are multiple network calls involved in this scheme. (lock,heal(ie read+write), unlock) per chunk. i.e 4 calls per chunk.
+
+### version 4 (ToDo)
+Some improvements that need to be made in version 3:
+* Reduce network calls using piggy backing.
+* After taking chunk lock and healing, we need to unlock the lock before locking the next chunk. This gives a window for any pending truncate FOPs to succeed. If truncate succeeds, the heal of next chunk will fail (read returns zero)
+and heal is stopped. *BUT* there is **yet another** issue:
+
+* shd does steps 1 to 4. Let's assume source is brick b1, sink is brick b2 . i.e xattrs are (0,1) and (0,0) on b1 and b2 respectively. Now before shd takes (0-128kb) lock, a client FOP takes it.
+It modifies data but the FOP succeeds only on brick 2. writev returns success, and the attrs now read (0,1) (1,0). SHD takes over and heals. It had observed (0,1),(0,0) earlier
+and thus goes ahead and copies stale 128Kb from brick 1 to brick2. Thus as far as application is concerned, `writev` returned success but bricks have stale data.
+What needs to be done is `writev` must return success only if it succeeded on atleast one source brick (brick b1 in this case). Otherwise  The heal still happens in reverse direction but as far as the application is concerned, it received an error.  
+
+### Note on lock **domains**
+We have used conceptual names in this document like DATA_DOMAIN/ METADATA_DOMAIN/ SELF_HEAL_DOMAIN. In the code, these are mapped to strings that are based on the AFR xlator name like so:
+
+DATA_DOMAIN     --->"vol_name-replicate-n"
+
+METADATA_DOMAIN  --->"vol_name-replicate-n:metadata"
+
+SELF_HEAL_DOMAIN -->"vol_name-replicate-n:self-heal"
+
+where vol_name is the name of the volume and 'n' is the replica subvolume index (starting from 0).
diff --git a/doc/developer-guide/afr-self-heal-daemon.md b/doc/developer-guide/afr-self-heal-daemon.md
new file mode 100644
index 00000000000..65940d420b7
--- /dev/null
+++ b/doc/developer-guide/afr-self-heal-daemon.md
@@ -0,0 +1,92 @@
+Self-Heal Daemon
+================
+The self-heal daemon (shd) is a glusterfs process that is responsible for healing files in a replicate/ disperse gluster volume.
+Every server (brick) node of the volume runs one instance of the shd. So even if one node contains replicate/ disperse bricks of
+multiple volumes, it would be healed by the same shd.
+
+This document only describes how the shd works for replicate (AFR) volumes.
+
+The shd is launched by glusterd when the volume starts (only if the volume includes a replicate configuration). The graph
+of the shd process in every node contains the following: The io-stats which is the top most xlator, its children being the
+replicate xlators (subvolumes) of *only* the bricks present in that particular node, and finally *all* the client xlators that are the children of the replicate xlators.
+
+The shd does two types of self-heal crawls: Index heal and Full heal. For both these types of crawls, the basic idea is the same:  
+For each file encountered while crawling, perform metadata, data and entry heals under appropriate locks.  
+* An overview of how each of these heals is performed is detailed in the 'Self-healing' section of *doc/features/afr-v1.md*
+* The different file locks which the shd takes for each of these heals is detailed in *doc/developer
+-guide/afr-locks-evolution.md*
+
+Metadata heal refers to healing extended attributes, mode and permissions of a file or directory.
+Data heal refers to healing the file contents.
+Entry self-heal refers to healing entries inside a directory.
+
+Index heal
+==========
+The index heal is done:  
+  a) Every 600 seconds (can be changed via the `cluster.heal-timeout` volume option)  
+  b) When it is explicitly triggered via the `gluster vol heal <VOLNAME>` command  
+  c) Whenever a replica brick that was down comes back up.  
+  
+Only one heal can be in progress at one time, irrespective of reason why it was triggered. If another heal is triggered before the first one completes, it will be queued.
+Only one heal can be queued while the first one is running. If an Index heal is queued, it can be overridden by queuing a Full heal and not vice-versa.  Also, before processing
+each entry in index heal, a check is made if a full heal is queued. If it is, then the index heal is aborted so that the full heal can proceed. 
+
+In index heal, each shd reads the entries present inside .glusterfs/indices/xattrop/ folder and triggers heal on each entry with appropriate locks.
+The .glusterfs/indices/xattrop/ directory contains a base entry of the name "xattrop-<virtual-gfid-string>". All other entries are hardlinks to the base entry. The
+*names* of the hardlinks are the gfid strings of the files that may need heal. 
+
+When a client (mount) performs an operation on the file, the index xlator present in each brick process adds the hardlinks in the pre-op phase of the FOP's transaction
+and removes it in post-op phase if the operation is successful. Thus if an entry is present inside the .glusterfs/indices/xattrop/ directory when there is no I/O 
+happening on the file, it means the file needs healing (or atleast an examination if the brick crashed after the post-op completed but just before the removal of the hardlink).
+
+#### Index heal steps:
+<pre><code>
+In shd process of *each node* {
+        opendir +readdir (.glusterfs/indices/xattrop/)
+        for each entry inside it {
+                self_heal_entry() //Explained below.
+        }
+}
+</code></pre>
+
+<pre><code>
+self_heal_entry() {
+        Call syncop_lookup(replicae subvolume) which eventually does {
+                take appropriate locks
+                determine source and sinks from AFR changelog xattrs	
+                perform whatever heal is needed (any of metadata, data and entry heal in that order)
+                clear changelog xattrs and hardlink inside .glusterfs/indices/xattrop/
+        }
+}
+</code></pre>
+
+Note:
+* If the gfid hardlink is present in the .glusterfs/indices/xattrop/ of both replica bricks, then each shd will try to heal the file but only one of them will be able to proceed due to the self-heal domain lock.
+
+* While processing entries inside .glusterfs/indices/xattrop/, if shd encounters an entry whose parent is yet to be healed, it will skip it and it will be picked up in the next crawl.
+
+* If a file is in data/ metadata split-brain, it will not be healed.
+
+* If a directory is in entry split-brain, a conservative merge will be performed, wherein after the merge, the entries of the directory will be a union of the entries in the replica pairs.
+
+Full heal
+=========
+A full heal is triggered by running `gluster vol heal <VOLNAME> full`. This command is usually run in disk replacement scenarios where the entire data is to be copied from one of the healthy bricks of the replica to the brick that was just replaced.
+
+Unlike the index heal which runs on the shd of every node in a replicate subvolume, the full heal is run only on the shd of one node per replicate subvolume: the node having the highest UUID.
+i.e In a 2x2 volume made of 4 nodes N1, N2, N3 and N4, If UUID of N1>N2 and UUID N4 >N3, then the full crawl is carried out by the shds of N1 and N4.(Node UUID can be found in `/var/lib/glusterd/glusterd.info`)
+
+The full heal steps are almost identical to the index heal, except the heal is performed on each replica starting from the root of the volume:
+<pre><code>
+In shd process of *highest UUID node per replica* {
+        opendir +readdir ("/")
+        for each entry inside it {
+                self_heal_entry()
+                if (entry == directory) {
+                        /* Recurse*/
+                        again opendir+readdir (directory) followed by self_heal_entry() of each entry.
+                }
+                
+        }
+}
+</code></pre>
diff --git a/doc/developer-guide/afr.md b/doc/developer-guide/afr.md
new file mode 100644
index 00000000000..566573a4e26
--- /dev/null
+++ b/doc/developer-guide/afr.md
@@ -0,0 +1,191 @@
+cluster/afr translator
+======================
+
+Locking
+-------
+
+Before understanding replicate, one must understand two internal FOPs:
+
+### `GF_FILE_LK`
+
+This is exactly like `fcntl(2)` locking, except the locks are in a
+separate domain from locks held by applications.
+
+### `GF_DIR_LK (loc_t *loc, char *basename)`
+
+This allows one to lock a name under a directory. For example,
+to lock /mnt/glusterfs/foo, one would use the call:
+
+```
+GF_DIR_LK ({loc_t for "/mnt/glusterfs"}, "foo")
+```
+
+If one wishes to lock *all* the names under a particular directory,
+supply the basename argument as `NULL`.
+
+The locks can either be read locks or write locks; consult the
+function prototype for more details.
+
+Both these operations are implemented by the features/locks (earlier
+known as posix-locks) translator.
+
+Basic design
+------------
+
+All FOPs can be classified into four major groups:
+
+### inode-read
+
+Operations that read an inode's data (file contents) or metadata (perms, etc.).
+
+access, getxattr, fstat, readlink, readv, stat.
+
+### inode-write
+
+Operations that modify an inode's data or metadata.
+
+chmod, chown, truncate, writev, utimens.
+
+### dir-read
+
+Operations that read a directory's contents or metadata.
+
+readdir, getdents, checksum.
+
+### dir-write
+
+Operations that modify a directory's contents or metadata.
+
+create, link, mkdir, mknod, rename, rmdir, symlink, unlink.
+
+Some of these make a subgroup in that they modify *two* different entries:
+link, rename, symlink.
+
+### Others
+
+Other operations.
+
+flush, lookup, open, opendir, statfs.
+
+Algorithms
+----------
+
+Each of the four major groups has its own algorithm:
+
+### inode-read, dir-read
+
+1. Send a request to the first child that is up:
+    * if it fails:
+        * try the next available child
+    * if we have exhausted all children:
+        * return failure
+
+### inode-write
+
+ All operations are done in parallel unless specified otherwise.
+
+1. Send a ``GF_FILE_LK`` request on all children for a write lock on the 
+   appropriate region
+   (for metadata operations: entire file (0, 0) for writev:
+   (offset, offset+size of buffer))
+    * If a lock request fails on a child:
+        * unlock all children
+        * try to acquire a blocking lock (`F_SETLKW`) on each child, serially.
+	  If this fails (due to `ENOTCONN` or `EINVAL`):
+          Consider this child as dead for rest of transaction.
+2. Mark all children as "pending" on all (alive) children (see below for 
+meaning of "pending").
+    * If it fails on any child:
+        * mark it as dead (in transaction local state).
+3. Perform operation on all (alive) children.
+    * If it fails on any child:
+        * mark it as dead (in transaction local state).
+4. Unmark all successful children as not "pending" on all nodes.
+5. Unlock region on all (alive) children.
+
+### dir-write
+
+ The algorithm for dir-write is same as above except instead of holding
+ `GF_FILE_LK` locks we hold a GF_DIR_LK lock on the name being operated upon.
+ In case of link-type calls, we hold locks on both the operand names.
+
+"pending"
+---------
+
+The "pending" number is like a journal entry. A pending entry is an
+array of 32-bit integers stored in network byte-order as the extended
+attribute of an inode (which can be a directory as well).
+
+There are three keys corresponding to three types of pending operations:
+
+### `AFR_METADATA_PENDING`
+
+There are some metadata operations pending on this inode (perms, ctime/mtime,
+xattr, etc.).
+
+### `AFR_DATA_PENDING`
+
+There is some data pending on this inode (writev).
+
+### `AFR_ENTRY_PENDING`
+
+There are some directory operations pending on this directory
+(create, unlink, etc.).
+
+Self heal
+---------
+
+* On lookup, gather extended attribute data:
+    * If entry is a regular file:
+        * If an entry is present on one child and not on others:
+            * create entry on others.
+        * If entries exist but have different metadata (perms, etc.):
+            * consider the entry with the highest `AFR_METADATA_PENDING` number as
+              definitive and replicate its attributes on children.
+    * If entry is a directory:
+        * Consider the entry with the highest `AFR_ENTRY_PENDING` number as
+          definitive and replicate its contents on all children.
+    * If any two entries have non-matching types (i.e., one is file and
+      other is directory):
+        * Announce to the user via log that a split-brain situation has been
+          detected, and do nothing.
+* On open, gather extended attribute data:
+    * Consider the file with the highest `AFR_DATA_PENDING` number as
+      the definitive one and replicate its contents on all other
+      children.
+
+During all self heal operations, appropriate locks must be held on all
+regions/entries being affected.
+
+Inode scaling
+-------------
+
+Inode scaling is necessary because if a situation arises where an inode number 
+is returned for a directory (by lookup) which was previously the inode number 
+of a file (as per FUSE's table), then FUSE gets horribly confused (consult a 
+FUSE expert for more details).
+
+To avoid such a situation, we distribute the 64-bit inode space equally
+among all children of replicate.
+
+To illustrate:
+
+If c1, c2, c3 are children of replicate, they each get 1/3 of the available
+inode space:
+
+-------------  --  --  --  --  --  --  --  --  --  --  --  ---
+Child:         c1  c2  c3  c1  c2  c3  c1  c2  c3  c1  c2  ...
+Inode number:  1    2   3   4   5   6   7   8   9  10  11  ...
+-------------  --  --  --  --  --  --  --  --  --  --  --  ---
+
+Thus, if lookup on c1 returns an inode number "2", it is scaled to "4"
+(which is the second inode number in c1's space).
+
+This way we ensure that there is never a collision of inode numbers from
+two different children.
+
+This reduction of inode space doesn't really reduce the usability of
+replicate since even if we assume replicate has 1024 children (which would be a
+highly unusual scenario), each child still has a 54-bit inode space:
+$2^{54} \sim 1.8 \times 10^{16}$, which is much larger than any real
+world requirement.
diff --git a/doc/developer-guide/brickmux-thread-reduction.md b/doc/developer-guide/brickmux-thread-reduction.md
new file mode 100644
index 00000000000..7d76e8ff579
--- /dev/null
+++ b/doc/developer-guide/brickmux-thread-reduction.md
@@ -0,0 +1,64 @@
+# Resource usage reduction in brick multiplexing
+
+Each brick is regresented with a graph of translators in a brick process.
+Each translator in the graph has its own set of threads and mem pools
+and other system resources allocations. Most of the times all these
+resources are not put to full use. Reducing the resource consumption
+of each brick is a problem in itself that needs to be addressed. The other
+aspect to it is, sharing of resources across brick graph, this becomes
+critical in brick multiplexing scenario. In this document we will be discussing
+only about the threads.
+
+If a brick mux process hosts 50 bricks there are atleast 600+ threads created
+in that process. Some of these are global threads that are shared by all the
+brick graphs, and others are per translator threads. The global threads like
+synctask threads, timer threads, sigwaiter, poller etc. are configurable and
+do not needs to be reduced. The per translator threads keeps growing as the
+number of bricks in the process increases. Each brick spawns atleast 10+
+threads:
+- io-threads
+- posix threads:
+     1. Janitor
+     2. Fsyncer
+     3. Helper
+     4. aio-thread
+- changelog and bitrot threads(even when the features are not enabled)
+
+## io-threads
+
+io-threads should be made global to the process, having 16+ threads for
+each brick does not make sense. But io-thread translator is loaded in
+the graph, and the position of io-thread translator decides from when
+the fops will be parallelised across threads. We cannot entirely move
+the io-threads to libglusterfs and say the multiplexing happens from
+the master translator or so. Hence, the io-thread orchestrator code
+is moved to libglusterfs, which ensures there is only one set of
+io-threads that is shared among the io-threads translator in each brick.
+This poses performance issues due to lock-contention in the io-threds
+layer. This also shall be addressed by having multiple locks instead of
+one global lock for io-threads.
+
+## Posix threads
+Most of the posix threads execute tasks in a timely manner, hence it can be
+replaced with a timer whose handler register a task to synctask framework, once
+the task is complete, the timer is registered again. With this we can eliminate
+the need of one thread for each task. The problem with using synctasks is
+the performance impact it will have due to make/swapcontext. For task that
+does not involve network wait, we need not do makecontext, instead the task
+function with arg can be stored and executed when a synctask thread is free.
+We need to implement an api in synctask to execute atomic tasks(no network wait)
+without the overhead of make/swapcontext. This will solve the performance
+impact associated with using synctask framework.
+
+And the other challenge, is to cancel all the tasks pending from a translator.
+This is important to cleanly detach brick. For this, we need to implement an
+api in synctask that can cancel all the tasks from a given translator.
+
+For future, this will be replced to use global thread-pool(once implemented).
+
+## Changelog and bitrot threads
+
+In the initial implementation, the threads are not created if the feature is
+not enabled. We need to share threads across changelog instances if we plan
+to enable these features in brick mux scenario.
+
diff --git a/doc/developer-guide/coding-standard.md b/doc/developer-guide/coding-standard.md
new file mode 100644
index 00000000000..031c6c0da99
--- /dev/null
+++ b/doc/developer-guide/coding-standard.md
@@ -0,0 +1,697 @@
+GlusterFS Coding Standards
+==========================
+
+Before you get started
+----------------------
+Before starting with other part of coding standard, install `clang-format`
+
+On Fedora:
+```
+$ dnf install clang
+```
+On debian/Ubuntu:
+```
+$ apt-get install clang
+```
+Once you are done with all the local changes, you need to run below set of commands,
+before submitting the patch for review.
+```
+$ git add $file # if any
+$ git commit -a -s -m "commit message"
+$ git show --pretty="format:" --name-only | grep -v "contrib/" | egrep "*\.[ch]$" | xargs clang-format -i
+$ git diff # see if there are any changes
+$ git commit -a --amend # get the format changes done
+$ ./submit-for-review.sh
+```
+
+
+Structure definitions should have a comment per member
+------------------------------------------------------
+
+Every member in a structure definition must have a comment about its purpose.
+The comment should be descriptive without being overly verbose.  For pointer
+members, lifecycle concerns for the pointed-to object should be noted.  For lock
+members, the relationship between the lock member and the other members it
+protects should be explicit.
+
+*Bad:*
+
+```
+gf_lock_t   lock;           /* lock */
+```
+
+*Good:*
+
+```
+DBTYPE      access_mode;    /* access mode for accessing
+                             * the databases, can be
+                             * DB_HASH, DB_BTREE
+                             * (option access-mode <mode>)
+                             */
+```
+
+Structure members should be aligned based on the padding requirements
+---------------------------------------------------------------------
+
+The compiler will make sure that structure members have optimum alignment,
+but at the expense of suboptimal padding. More important is to optimize the
+padding. The compiler won't do that for you.
+
+This also will help utilize the memory better
+
+*Bad:*
+```
+struct bad {
+    bool  b; /* 0 */
+             /* 1..7 pad */
+    void *p; /* 8..15 */
+    char  c; /* 16 */
+    char  a[16]; /* 17..33 */
+             /* 34..39 pad */
+    int64_t ii; /* 40..47 */
+    int32_t i; /* 48..51 */
+             /* 52..55 pad */
+    int64_t iii; /* 56..63 */
+};
+```
+
+*Good:*
+```
+struct good {
+   int64_t ii; /* explicit 64-bit types */
+   void *p; /* may be 64- or 32-bit */
+   long  l;   /* may be 64- or 32-bit */
+   int   i;   /* 32-bit */
+   short s;   /* 16-bit */
+   char  c;   /* 8-bit */
+   bool  b;   /* 8-bit */
+   char  a[1024];
+);
+```
+Make sure the items with the most stringent alignment requirements will need
+to come earliest (ie, pointers and  perhaps uint64_t etc), and those with less
+stringent alignment requirements at the end (uint16/uint8 and char). Also note
+that the long array (if any) should be at the end of the structure, regardless
+of the type.
+
+Also note, if your structure's overall size is crossing 1k-4k limit, it is
+recommended to mention the reason why the particular structure needs so much
+memory as a comment at the top.
+
+Use \_typename for struct tags and typename\_t for typedefs
+---------------------------------------------------------
+
+Being consistent here makes it possible to automate navigation from use of a
+type to its true definition (not just the typedef).
+
+*Bad:*
+
+```
+struct thing {...};
+struct thing_t {...};
+typedef struct _thing thing;
+```
+
+*Good:*
+
+```
+typedef struct _thing {...} thing_t;
+```
+
+No double underscores
+---------------------
+
+Identifiers beginning with double underscores are supposed to reserved for the
+compiler.
+
+http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf
+
+When you need to define inner/outer functions, use a different prefix/suffix.
+
+*Bad:*
+
+```
+void __do_something (void);
+
+void
+do_something (void)
+{
+        LOCK ();
+        __do_something ();
+        UNLOCK ();
+}
+```
+
+*Good:*
+
+```
+void do_something_locked (void);
+```
+
+Only use safe pointers in initializers
+----------------------------------------------------------
+
+Some pointers, such as `this` in a fop function, can be assumed to be non-NULL.
+However, other parameters and further-derived values might be NULL.
+
+*Good:*
+
+```
+pid_t pid     = frame->root->pid;
+```
+
+
+*Bad:*
+
+```
+data_t *my_data = dict_get (xdata, "fubar");
+```
+
+No giant stack allocations
+--------------------------
+
+Synctasks have small finite stacks.  To avoid overflowing these stacks, avoid
+allocating any large data structures on the stack.  Use dynamic allocation
+instead.
+
+*Bad:*
+
+```
+gf_boolean_t port_inuse[65536]; /* 256KB, this actually happened */
+```
+
+NOTE: Ideal is to limit the stack array to less than 256 bytes.
+
+
+Character array initializing
+----------------------------
+
+It is recommended to keep the character array initializing to empty string.
+
+*Good:*
+```
+char msg[1024] = "";
+```
+
+Not so much recommended, even though it means the same.
+
+```
+char msg[1024] = {0,};
+```
+
+We recommend above to structure initialization.
+
+
+
+Validate all arguments to a function
+------------------------------------
+
+All pointer arguments to a function must be checked for `NULL`.
+A macro named `GF_VALIDATE_OR_GOTO` (in `common-utils.h`)
+takes two arguments; if the first is `NULL`, it writes a log message and
+jumps to a label specified by the second aergument after setting errno
+appropriately. There are several variants of this function for more
+specific purposes, and their use is recommended.
+
+*Bad:*
+
+```
+/* top of function */
+ret = dict_get (xdata, ...)
+```
+
+*Good:*
+
+```
+/* top of function */
+GF_VALIDATE_OR_GOTO(xdata,out);
+ret = dict_get (xdata, ...)
+```
+
+Never rely on precedence of operators
+-------------------------------------
+
+Never write code that relies on the precedence of operators to execute
+correctly.  Such code can be hard to read and someone else might not
+know the precedence of operators as accurately as you do.  This includes
+precedence of increment/decrement vs. field/subscript.  The only exceptions are
+arithmetic operators (which have had defined precedence since before computers
+even existed) and boolean negation.
+
+*Bad:*
+
+```
+if (op_ret == -1 && errno != ENOENT)
+++foo->bar      /* incrementing foo, or incrementing foo->bar? */
+a && b || !c
+```
+
+*Good:*
+
+```
+if ((op_ret == -1) && (errno != ENOENT))
+(++foo)->bar
+++(foo->bar)
+(a && b) || !c
+a && (b || !c)
+```
+
+Use exactly matching types
+--------------------------
+
+Use a variable of the exact type declared in the manual to hold the
+return value of a function. Do not use an 'equivalent' type.
+
+
+*Bad:*
+
+```
+int len = strlen (path);
+```
+
+*Good:*
+
+```
+size_t len = strlen (path);
+```
+
+Avoid code such as `foo->bar->baz`; check every pointer
+-------------------------------------------------------------
+
+Do not write code that blindly follows a chain of pointer references. Any
+pointer in the chain may be `NULL` and thus cause a crash. Verify that each
+pointer is non-null before following it.  Even if `foo->bar` has been checked
+and is known safe, repeating it can make code more verbose and less clear.
+
+This rule includes `[]` as well as `->` because both dereference pointers.
+
+*Bad:*
+
+```
+foo->bar->field1 = value1;
+xyz = foo->bar->field2 + foo->bar->field3 * foo->bar->field4;
+foo->bar[5].baz
+```
+
+*Good:*
+
+```
+my_bar = foo->bar;
+if (!my_bar) ... return;
+my_bar->field1 = value1;
+xyz = my_bar->field2 + my_bar->field3 * my_bar->field4;
+```
+
+Document unchecked return values
+----------------------------------------------------
+
+In general, return values should be checked.  If a function is being called
+for its side effects and the return value really doesn't matter, an explicit
+cast to void is required (to keep static analyzers happy) and a comment is
+recommended.
+
+*Bad:*
+
+```
+close (fd);
+do_important_thing ();
+```
+
+*Good (or at least OK):*
+
+```
+(void) sleep (1);
+```
+
+Gracefully handle failure of malloc (and other allocation functions)
+--------------------------------------------------------------------
+
+GlusterFS should never crash or exit due to lack of memory. If a
+memory allocation fails, the call should be unwound and an error
+returned to the user.
+
+*Use result args and reserve the return value to indicate success or failure:*
+
+The return value of every functions must indicate success or failure (unless
+it is impossible for the function to fail --- e.g., boolean functions). If
+the function needs to return additional data, it must be returned using a
+result (pointer) argument.
+
+*Bad:*
+
+```
+int32_t dict_get_int32 (dict_t *this, char *key);
+```
+
+*Good:*
+
+```
+int dict_get_int32 (dict_t *this, char *key, int32_t *val);
+```
+
+Always use the 'n' versions of string functions
+-----------------------------------------------
+
+Unless impossible, use the length-limited versions of the string functions.
+
+*Bad:*
+
+```
+strcpy (entry_path, real_path);
+```
+
+*Good:*
+
+```
+strncpy (entry_path, real_path, entry_path_len);
+```
+
+Do not use memset prior to sprintf/snprintf/vsnprintf etc...
+------------------------------------------------------------
+snprintf(and other similar string functions) terminates the buffer with a
+'\0'(null character). Hence, there is no need to do a memset before using
+snprintf. (Of course you need to account one extra byte for the null character
+in your allocation).
+
+Note: Similarly if you are doing pre-memory allocation for the buffer, use
+GF_MALLOC instead of GF_CALLOC, since the later is bit costlier.
+
+*Bad:*
+
+```
+char buffer[x];
+memset (buffer, 0, x);
+bytes_read = snprintf (buffer, sizeof buffer, "bad standard");
+```
+
+*Good:*
+```
+char buffer[x];
+bytes_read = snprintf (buffer, sizeof (buffer), "good standard");
+```
+
+And it is always to good initialize the char array if the string is static.
+
+E.g.
+```
+char buffer[] = "good standard";
+```
+
+No dead or commented code
+-------------------------
+
+There must be no dead code (code to which control can never be passed) or
+commented out code in the codebase.
+
+Function length or Keep functions small
+---------------------------------------
+
+We live in the UNIX-world where modules do one thing and do it well.
+This rule should apply to our functions also. If a function is very long, try splitting it
+into many little helper functions. The question is, in a coding
+spree, how do we know a function is long and unreadable. One rule of
+thumb given by Linus Torvalds is that, a function should be broken-up
+if you have 4 or more levels of indentation going on for more than 3-4
+lines.
+
+*Example for a helper function:*
+```
+static int
+same_owner (posix_lock_t *l1, posix_lock_t *l2)
+{
+        return ((l1->client_pid == l2->client_pid) &&
+               (l1->transport  == l2->transport));
+}
+```
+
+Define functions as static
+--------------------------
+
+Declare functions as static unless they're exposed via a module-level API for
+use from other modules.
+
+No nested functions
+-------------------
+
+Nested functions have proven unreliable, e.g. as callbacks in code that uses
+ucontext (green) threads,
+
+Use inline functions instead of macros whenever possible
+--------------------------------------------------------
+
+Inline functions enforce type safety; macros do not.  Use macros only for things
+that explicitly need to be type-agnostic (e.g. cases where one might use
+generics or templates in other languages), or that use other preprocessor
+features such as `#` for stringification or `##` for token pasting.  In general,
+"static inline" is the preferred form.
+
+Avoid copypasta
+---------------
+
+Code that is copied and then pasted into multiple functions often creates
+maintenance problems later, e.g. updating all but one instance for a subsequent
+change.  If you find yourself copying the same "boilerplate" many places,
+consider refactoring to use helper functions (including inline) or macros, or
+code generation.
+
+Ensure function calls wrap around after 80-columns
+--------------------------------------------------
+
+Place remaining arguments on the next line if needed.
+
+Functions arguments and function definition
+-------------------------------------------
+
+Place all the arguments of a function definition on the same line
+until the line goes beyond 80-cols. Arguments that extend beyind
+80-cols should be placed on the next line.
+
+Style issues
+------------
+
+### Brace placement
+
+Use K&R/Linux style of brace placement for blocks.
+
+*Good:*
+
+```
+int some_function (...)
+{
+        if (...) {
+                /* ... */
+        } else if (...) {
+                /* ... */
+        } else {
+                /* ... */
+        }
+
+        do {
+                /* ... */
+        } while (cond);
+}
+```
+
+### Indentation
+
+Use *eight* spaces for indenting blocks. Ensure that your
+file contains only spaces and not tab characters. You can do this
+in Emacs by selecting the entire file (`C-x h`) and
+running `M-x untabify`.
+
+To make Emacs indent lines automatically by eight spaces, add this
+line to your `.emacs`:
+
+```
+(add-hook 'c-mode-hook (lambda () (c-set-style "linux")))
+```
+
+### Comments
+
+Write a comment before every function describing its purpose (one-line),
+its arguments, and its return value. Mention whether it is an internal
+function or an exported function.
+
+Write a comment before every structure describing its purpose, and
+write comments about each of its members.
+
+Follow the style shown below for comments, since such comments
+can then be automatically extracted by doxygen to generate
+documentation.
+
+*Good:*
+
+```
+/**
+* hash_name -hash function for filenames
+* @par:  parent inode number
+* @name: basename of inode
+* @mod:  number of buckets in the hashtable
+*
+* @return: success: bucket number
+*          failure: -1
+*
+* Not for external use.
+*/
+```
+
+### Indicating critical sections
+
+To clearly show regions of code which execute with locks held, use
+the following format:
+
+```
+pthread_mutex_lock (&mutex);
+{
+        /* code */
+}
+pthread_mutex_unlock (&mutex);
+```
+
+### Always use braces
+
+Even around single statements.
+
+*Bad:*
+
+```
+if (condition) action ();
+
+if (condition)
+        action ();
+```
+
+*Good:*
+
+```
+if (condition) {
+        action ();
+}
+```
+
+### Avoid multi-line conditionals
+
+These can be hard to read and even harder to modify later.  Predicate functions
+and helper variables are always better for maintainability.
+
+*Bad:*
+
+```
+if ((thing1 && other_complex_condition (thing1, lots, of, args))
+    || (!thing2 || even_more_complex_condition (thing2))
+    || all_sorts_of_stuff_with_thing3) {
+    return;
+}
+
+```
+
+*Better:*
+
+```
+thing1_ok = predicate1 (thing1, lots, of, args
+thing2_ok = predicate2 (thing2);
+thing3_ok = predicate3 (thing3);
+
+if (!thing1_ok || !thing2_ok || !thing3_ok) {
+        return;
+}
+```
+
+*Best:*
+
+```
+if (thing1 && other_complex_condition (thing1, lots, of, args)) {
+        return;
+}
+if (!thing2 || even_more_complex_condition (thing2)) {
+        /* Note potential for a different message here. */
+        return;
+}
+if (all_sorts_of_stuff_with_thing3) {
+        /* And here too. */
+        return;
+}
+```
+
+### Use 'const' liberally
+
+If a value isn't supposed/expected to change, there's no cost to adding a
+'const' keyword and it will help prevent violation of expectations.
+
+### Avoid global variables (including 'static' auto variables)
+Almost all state in Gluster is contextual and should be contained in the
+appropriate structure reflecting its scope (e.g. `call\_frame\_t`, `call\_stack\_t`,
+`xlator\_t`, `glusterfs\_ctx\_t`).  With dynamic loading and graph switches in play,
+each global requires careful consideration of when it should be initialized or
+reinitialized, when it might _accidentally_ be reinitialized, when its value
+might become stale, and so on.  A few global variables are needed to serve as
+'anchor points' for these structures, and more exceptions to the rule might be
+approved in the future, but new globals should not be added to the codebase
+without explicit approval.
+
+## A skeleton fop function
+
+This is the recommended template for any fop. In the beginning come the
+initializations. After that, the 'success' control flow should be linear.  Any
+error conditions should cause a `goto` to a label at the end.  By convention
+this is 'out' if there is only one such label, but a cascade of such labels is
+allowable to support multi-stage cleanup.  At that point, the code should detect
+the error that has occurred and do appropriate cleanup.
+
+```
+int32_t
+sample_fop (call_frame_t *frame, xlator_t *this, ...)
+{
+        char *            var1     = NULL;
+        int32_t           op_ret   = -1;
+        int32_t           op_errno = 0;
+        DIR *             dir      = NULL;
+        struct posix_fd * pfd      = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+
+        /* other validations */
+
+        dir = opendir (...);
+
+        if (dir == NULL) {
+                op_errno = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "opendir failed on %s (%s)", loc->path,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* another system call */
+        if (...) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "out of memory :(");
+                goto out;
+        }
+
+        /* ... */
+
+ out:
+        if (op_ret == -1) {
+
+          /* check for all the cleanup that needs to be
+             done */
+
+                if (dir) {
+                        closedir (dir);
+                        dir = NULL;
+                }
+
+                if (pfd) {
+                        FREE (pfd->path);
+                        FREE (pfd);
+                        pfd = NULL;
+                }
+        }
+
+        STACK_UNWIND (frame, op_ret, op_errno, fd);
+        return 0;
+}
+```
diff --git a/doc/developer-guide/commit-guidelines.md b/doc/developer-guide/commit-guidelines.md
new file mode 100644
index 00000000000..38bbe525cbd
--- /dev/null
+++ b/doc/developer-guide/commit-guidelines.md
@@ -0,0 +1,136 @@
+## Git Commit Good Practice
+
+The following document is based on experience doing code development, bug troubleshooting and code review across a number of projects using Git. The document is mostly borrowed from [Open Stack](https://wiki.openstack.org/wiki/GitCommitMessages), but made more meaningful in the context of GlusterFS project.
+
+This topic can be split into two areas of concern
+
+* The structured set/split of the code changes
+* The information provided in the commit message
+
+### Executive Summary
+The points and examples that will be raised in this document ought to clearly demonstrate the value in splitting up changes into a sequence of individual commits, and the importance in writing good commit messages to go along with them. If these guidelines were widely applied it would result in a significant improvement in the quality of the GlusterFS Git history. Both a carrot & stick will be required to effect changes. This document intends to be the carrot by alerting people to the benefits, while anyone doing Gerrit code review can act as the stick ;-P
+
+In other words, when reviewing a change in Gerrit:
+* Do not simply look at the correctness of the code.
+* Review the commit message itself and request improvements to its content.
+* Look out for commits which are mixing multiple logical changes and require the submitter to split them into separate commits.
+* Ensure whitespace changes are not mixed in with functional changes.
+* Ensure no-op code refactoring is done separately from functional changes.
+
+And so on.
+
+It might be mentioned that Gerrit's handling of patch series is not entirely perfect. Let that not become a valid reason to avoid creating patch series. The tools being used should be subservient to developers needs, and since they are open source they can be fixed / improved. Software source code is "read mostly, write occassionally" and thus the most important criteria is to improve the long term maintainability by the large pool of developers in the community, and not to sacrifice too much for the sake of the single author who may never touch the code again.
+
+And now the long detailed guidelines & examples of good & bad practice
+
+### Structural split of changes
+The cardinal rule for creating good commits is to ensure there is only one "logical change" per commit. There are many reasons why this is an important rule:
+
+* The smaller the amount of code being changed, the quicker & easier it is to review & identify potential flaws.
+* If a change is found to be flawed later, it may be necessary to revert the broken commit. This is much easier to do if there are not other unrelated code changes entangled with the original commit.
+* When troubleshooting problems using Git's bisect capability, small well defined changes will aid in isolating exactly where the code problem was introduced.
+* When browsing history using Git annotate/blame, small well defined changes also aid in isolating exactly where & why a piece of code came from.
+
+#### Things to avoid when creating commits
+With the above points in mind, there are some commonly encountered examples of bad things to avoid
+
+* Mixing whitespace changes with functional code changes.
+
+The whitespace changes will obscure the important functional changes, making it harder for a reviewer to correctly determine whether the change is correct. Solution: Create 2 commits, one with the whitespace changes, one with the functional changes. Typically the whitespace change would be done first, but that need not be a hard rule.
+
+* Mixing two unrelated functional changes.
+
+Again the reviewer will find it harder to identify flaws if two unrelated changes are mixed together. If it becomes necessary to later revert a broken commit, the two unrelated changes will need to be untangled, with further risk of bug creation.
+
+* Sending large new features in a single giant commit.
+
+It may well be the case that the code for a new feature is only useful when all of it is present. This does not, however, imply that the entire feature should be provided in a single commit. New features often entail refactoring existing code. It is highly desirable that any refactoring is done in commits which are separate from those implementing the new feature. This helps reviewers and test suites validate that the refactoring has no unintentional functional changes.
+
+Even the newly written code can often be split up into multiple pieces that can be independently reviewed. For example, changes which add new internal fops or library functions, can be in self-contained commits. Again this leads to easier code review. It also allows other developers to cherry-pick small parts of the work, if the entire new feature is not immediately ready for merge. This will encourage the author & reviewers to think about the generic library functions' design, and not simply pick a design that is easier for their currently chosen internal implementation.
+
+The basic rule to follow is
+
+If a code change can be split into a sequence of patches/commits, then it should be split. Less is not more. More is more.
+
+##### Examples of bad practice
+
+TODO: Pick glusterfs specific example.
+
+
+##### Examples of good practice
+
+
+### Information in commit messages
+As important as the content of the change, is the content of the commit message describing it. When writing a commit message there are some important things to remember
+
+* Do not assume the reviewer understands what the original problem was.
+
+When reading bug reports, after a number of back & forth comments, it is often as clear as mud, what the root cause problem is. The commit message should have a clear statement as to what the original problem is. The bug is merely interesting historical background on /how/ the problem was identified. It should be possible to review a proposed patch for correctness without needing to read the bug ticket.
+
+* Do not assume the reviewer has access to external web services/site.
+
+In 6 months time when someone is on a train/plane/coach/beach/pub troubleshooting a problem & browsing Git history, there is no guarantee they will have access to the online bug tracker, or online blueprint documents. The great step forward with distributed SCM is that you no longer need to be "online" to have access to all information about the code repository. The commit message should be totally self-contained, to maintain that benefit.
+
+* Do not assume the code is self-evident/self-documenting.
+
+What is self-evident to one person, might be clear as mud to another person. Always document what the original problem was and how it is being fixed, for any change except the most obvious typos, or whitespace only commits.
+
+* Describe why a change is being made.
+
+A common mistake is to just document how the code has been written, without describing /why/ the developer chose to do it that way. By all means describe the overall code structure, particularly for large changes, but more importantly describe the intent/motivation behind the changes.
+
+* Read the commit message to see if it hints at improved code structure.
+
+Often when describing a large commit message, it becomes obvious that a commit should have in fact been split into 2 or more parts. Don't be afraid to go back and rebase the change to split it up into separate commits.
+
+* Ensure sufficient information to decide whether to review.
+
+When Gerrit sends out email alerts for new patch submissions there is minimal information included, principally the commit message and the list of files changes. Given the high volume of patches, it is not reasonable to expect all reviewers to examine the patches in detail. The commit message must thus contain sufficient information to alert the potential reviewers to the fact that this is a patch they need to look at.
+
+* The first commit line is the most important.
+
+In Git commits the first line of the commit message has special significance. It is used as email subject line, git annotate messages, gitk viewer annotations, merge commit messages and many more places where space is at a premium. As well as summarizing the change itself, it should take care to detail what part of the code is affected. eg if it is 'afr', 'dht' or any translator. Or in some cases, it can be touching all these components, but the commit message can be 'coverity:', 'txn-framework:', 'new-fop: ', etc.
+
+* Describe any limitations of the current code.
+
+If the code being changed still has future scope for improvements, or any known limitations then mention these in the commit message. This demonstrates to the reviewer that the broader picture has been considered and what tradeoffs have been done in terms of short term goals vs. long term wishes.
+
+* Do not include patch set-specific comments.
+
+In other words, if you rebase your change please don't add "Patch set 2: rebased" to your commit message. That isn't going to be relevant once your change has merged. Please do make a note of that in Gerrit as a comment on your change, however. It helps reviewers know what changed between patch sets. This also applies to comments such as "Added unit tests", "Fixed localization problems", or any other such patch set to patch set changes that don't affect the overall intent of your commit.
+
+**The main rule to follow is:**
+
+The commit message must contain all the information required to fully understand & review the patch for correctness. Less is not more. More is more.
+
+
+#### Including external references
+
+The commit message is primarily targeted towards human interpretation, but there is always some metadata provided for machine use. In the case of GlusterFS this includes at least the 'Change-id', "bug"/"feature" ID references and "Signed-off-by" tag (generated by 'git commit -s').
+
+The 'Change-id' line is a unique hash describing the change, which is generated by a Git commit hook. This should not be changed when rebasing a commit following review feedback, since it is used by Gerrit, to track versions of a patch.
+
+The 'bug' line can reference a bug in a few ways. Gerrit creates a link to the bug when viewing the patch on review.gluster.org so that reviewers can quickly access the bug/issue on Bugzilla or Github.
+
+**Fixes: bz#1601166** -- use 'Fixes: bz#NNNNN' if the commit is intended to fully fix and close the bug being referenced.
+**Fixes: #411** -- use 'Fixes: #NNN' if the patch fixes the github issue completely.
+
+**Updates: bz#1193929** -- use 'Updates: bz#NNNN' if the commit is only a partial fix and more work is needed.
+**Updates: #175** -- use 'Updates: #NNNN' if the commit is only a partial fix and more work is needed for the feature completion.
+
+We encourage the use of `Co-Authored-By: name <name@example.com>` in commit messages to indicate people who worked on a particular patch. It's a convention for recognizing multiple authors, and our projects would encourage the stats tools to observe it when collecting statistics.
+
+### Summary of Git commit message structure
+
+* Provide a brief description of the change in the first line.
+* The first line should be limited to 50 characters and should not end with a period.
+
+* Insert a single blank line after the first line.
+
+* Provide a detailed description of the change in the following lines, breaking paragraphs where needed.
+
+* Subsequent lines should be wrapped at 72 characters.
+
+Put the 'Change-id', 'Fixes bz#NNNNN' and 'Signed-off-by: <>' lines at the very end.
+
+TODO: Add good examples
diff --git a/doc/developer-guide/daemon-management-framework.md b/doc/developer-guide/daemon-management-framework.md
new file mode 100644
index 00000000000..592192e665d
--- /dev/null
+++ b/doc/developer-guide/daemon-management-framework.md
@@ -0,0 +1,38 @@
+
+How to introduce new daemons using daemon management framework
+==============================================================
+Glusterd manages GlusterFS daemons providing services like NFS, Proactive
+self-heal, Quota, User servicable snapshots etc. Following are some of the
+aspects that come under daemon management.
+
+Data members & functions of different management objects
+
+- **Connection Management**
+    - unix domain sockets based channel for internal communication
+    - rpc connection for the communication
+    - frame timeout value for UDS
+    - Methods - notify
+    - init, connect, termination, disconnect APIs can be invoked using the
+      connection management object
+
+- **Process Management**
+    - Name of the process
+    - pidfile to detect if the daemon is running
+    - loggging directory, log file, volfile, volfileserver & volfileid
+    - init, stop APIs can be invoked using the process management object
+
+- **Service Management**
+    - connection object
+    - process object
+    - online status
+    - Methods - manager, start, stop which can be abstracted as a common methods
+      or specific to service requirements
+    - init API can be invoked using the service management object
+
+ The above structures defines the skeleton of the daemon management framework.
+ Introduction of new daemons in GlusterFS needs to inherit these properties. Any
+ requirement specific to a daemon needs to be implemented in its own service
+ (for eg : snapd defines its own type glusterd_snapdsvc_t using glusterd_svc_t
+ and snapd specific data). New daemons will need to have its own service specific
+ code written in glusterd-<feature>-svc.h{c} and need to reuse the existing
+ framework.
diff --git a/doc/developer-guide/datastructure-inode.md b/doc/developer-guide/datastructure-inode.md
new file mode 100644
index 00000000000..45d7a941e5f
--- /dev/null
+++ b/doc/developer-guide/datastructure-inode.md
@@ -0,0 +1,221 @@
+# Inode and dentry management in GlusterFS:
+
+## Background
+Filesystems internally refer to files and directories via inodes. Inodes
+are unique identifiers of the entities stored in a filesystem. Whenever an
+application has to operate on a file/directory (read/modify), the filesystem
+maps that file/directory to the right inode and start referring to that inode
+whenever an operation has to be performed on the file/directory.
+
+In GlusterFS a new inode gets created whenever a new file/directory is created
+OR when a successful lookup is done on a file/directory for the first time.
+Inodes in GlusterFS are maintained by the inode table which gets initiated when
+the filesystem daemon is started (both for the brick process as well as the
+mount process). Below are some important data structures for inode management.
+
+## Data-structure (inode-table)
+```
+struct _inode_table {
+        pthread_mutex_t    lock;
+        size_t             hashsize;    /* bucket size of inode hash and dentry hash */
+        char              *name;        /* name of the inode table, just for gf_log() */
+        inode_t           *root;        /* root directory inode, with inode
+        number and gfid 1 */
+        xlator_t          *xl;          /* xlator to be called to do purge and
+        the xlator which maintains the inode table*/
+        uint32_t           lru_limit;   /* maximum LRU cache size */
+        struct list_head  *inode_hash;  /* buckets for inode hash table */
+        struct list_head  *name_hash;   /* buckets for dentry hash table */
+        struct list_head   active;      /* list of inodes currently active (in an fop) */
+        uint32_t           active_size; /* count of inodes in active list */
+        struct list_head   lru;         /* list of inodes recently used.
+                                           lru.next most recent */
+        uint32_t           lru_size;    /* count of inodes in lru list  */
+        struct list_head   purge;       /* list of inodes to be purged soon */
+        uint32_t           purge_size;  /* count of inodes in purge list */
+
+        struct mem_pool   *inode_pool;  /* memory pool for inodes */
+        struct mem_pool   *dentry_pool; /* memory pool for dentrys */
+        struct mem_pool   *fd_mem_pool; /* memory pool for fd_t */
+        int                ctxcount;    /* number of slots in inode->ctx */
+};
+```
+
+# Life-cycle
+```
+inode_table_new (size_t lru_limit, xlator_t *xl)
+```
+This is a function which allocates a new inode table. Usually the top xlators in
+the graph such as protocol/server (for bricks), fuse and nfs (for fuse and nfs
+mounts) and libgfapi do inode managements. Hence they are the ones which will
+allocate a new inode table by calling the above function.
+
+Each xlator graph in glusterfs maintains an inode table. So in fuse clients,
+whenever there is a graph change due to add brick/remove brick or
+addition/removal of some other xlators, a new graph is created which creates a
+new inode table.
+
+Thus an allocated inode table is destroyed only when the filesystem daemon is
+killed or unmounted.
+
+
+# what it contains.
+Inode table in glusterfs mainly contains a hash table for maintaining inodes.
+In general a file/directory is considered to be existing if there is a
+corresponding inode present in the inode table. If a inode for a file/directory
+cannot be found in the inode table, glusterfs tries to resolve it by sending a
+lookup on the entry for which the inode is needed. If lookup is successful, then
+a new inode correponding to the entry is added to the hash table present in the
+inode table. Thus an inode present in the hash-table means, its an existing
+file/directory within the filesystem. The inode table also contains the hash
+size of the hash table (as of now it is hard coded to 14057. The hash value of
+a inode is calculated using its gfid).
+
+Apart from the hash table, inode table also maintains 3 important list of inodes
+1. Active list:
+Active list contains all the active inodes (i.e inodes which are currently part
+of some fop).
+2. Lru list:
+Least recently used inodes list. A limit can be set for the size of the lru
+list. For bricks it is 16384 and for clients it is infinity.
+3. Purge list:
+List of all the inodes which have to be purged (i.e inodes which have to be
+deleted from the inode table due to unlink/rmdir/forget).
+
+And at last it also contains the mem-pool for allocating inodes, dentries so
+that frequent malloc/calloc and free of the data structures can be avoided.
+
+
+# Data structure (inode)
+```
+struct _inode {
+        inode_table_t       *table;         /* the table this inode belongs to */
+        uuid_t               gfid;          /* unique identifier of the inode */
+        gf_lock_t            lock;
+        uint64_t             nlookup;
+        uint32_t             fd_count;      /* Open fd count */
+        uint32_t             ref;           /* reference count on this inode */
+        ia_type_t            ia_type;       /* what kind of file */
+        struct list_head     fd_list;       /* list of open files on this inode */
+        struct list_head     dentry_list;   /* list of directory entries for this inode */
+        struct list_head     hash;          /* hash table pointers */
+        struct list_head     list;          /* active/lru/purge */
+
+        struct _inode_ctx   *_ctx;          /* place holder for keeping the
+        information about the inode by different xlators */
+};
+```
+As said above, inodes are internal way of identifying the files/directories. A
+inode uniquely represents a file/directory. A new inode is created whenever a
+create/mkdir/symlink/mknod operations are performed. Apart from that a new inode
+is created upon the successful fresh lookup of a file/directory. Say the
+filesystem contained some file "a" within root and the filesystem was
+unmounted. Now when glusterfs is mounted and some operation is perfomed on "/a",
+glusterfs tries to get the inode for the entry "a" with parent inode as
+root. But, since glusterfs just came up, it will not be able to find the inode
+for "a" and will send a lookup on "/a". If the lookup operation succeeds (i.e.
+the root of glusterfs contains an entry called "a"), then a new inode for "/a"
+is created and added to the inode table.
+
+Depending upon the situation, an inode can be in one of the 3 lists maintained
+by the inode table. If some fop is happening on the inode, then the inode will
+be present in the active inodes list maintained by the inode table. Active
+inodes are those inodes whose refcount is greater than zero. Whenever some
+operation comes on a file/directory, and the resolver tries to find the inode
+for it, it increments the refcount of the inode before returning the inode. The
+refcount of an inode can be incremented by calling the below function
+```
+inode_ref (inode_t *inode)
+```
+Any xlator which wants to operate on a inode as part of some fop (or wants the
+inode in the callback), should hold a ref on the inode.
+Once the fop is completed before sending the reply of the fop to the above
+layers , the inode has to be unrefed. When the refcount of an inode becomes
+zero, it is removed from the active inodes list and put into LRU list maintained
+by the inode table. Thus in short if some fop is happening on a file/directory,
+the corresponding inode will be in the active list or it will be in the LRU
+list.
+
+
+# Life Cycle
+
+A new inode is created whenever a new file/directory/symlink is created OR a
+successful lookup of an existing entry is done. The xlators which does inode
+management (as of now protocol/server, fuse, nfs, gfapi) will perform inode_link
+operation upon successful lookup or successful creation of a new entry.
+```
+inode_link (inode_t *inode, inode_t *parent, const char *name,
+            struct iatt *buf);
+```
+inode_link actually adds the inode to the inode table (to be precise it adds
+the inode to the hash table maintained by the inode table. The hash value is
+calculated based on the gfid). Copies the gfid to the inode (the gfid is
+present in the iatt structure). Creates a dentry with the new name.
+
+A inode is removed from the inode table and eventually destroyed when unlink
+or rmdir operation is performed on a file/directory, or the the lru limit of
+the inode table has been exceeded.
+
+# Data structure (dentry)
+```
+
+struct _dentry {
+        struct list_head   inode_list;   /* list of dentries of inode */
+        struct list_head   hash;         /* hash table pointers */
+        inode_t           *inode;        /* inode of this directory entry */
+        char              *name;         /* name of the directory entry */
+        inode_t           *parent;       /* directory of the entry */
+};
+```
+A dentry is the presence of an entry for a file/directory within its parent
+directory. A dentry usually points to the inode to which it belongs to. In
+glusterfs a dentry contains the following fields.
+1. a hook using which it can add itself to the list of
+the dentries maintained by the inode to which it points to.
+2. A hash table pointer.
+3. Pointer to the inode to which it belongs to.
+4. Name of the dentry
+5. Pointer to the inode of the parent directory in which the dentry is present
+
+A new dentry is created when a new file/directory/symlink is created or a hard
+link to an existing file is created.
+```
+__dentry_create (inode_t *inode, inode_t *parent, const char *name);
+```
+A dentry holds a refcount on the parent
+directory so that the parent inode is never removed from the active inode's list
+and put to the lru list (If the lru limit of the lru list is exceeded, there is
+a chance of parent inode being destroyed. To avoid it, the dentries hold a
+reference to the parent inode). A dentry is removed whenevern a unlink/rmdir
+is perfomed on a file/directory. Or when the lru limit has been exceeded, the
+oldest inodes are purged out of the inode table, during which all the dentries
+of the inode are removed.
+
+Whenever a unlink/rmdir comes on a file/directory, the corresponding inode
+should be removed from the inode table. So upon unlink/rmdir, the inode will
+be moved to the purge list maintained by the inode table and from there it is
+destroyed. To be more specific, if a inode has to be destroyed, its refcount
+and nlookup count both should become 0. For refcount to become 0, the inode
+should not be part of any fop (there should not be any open fds). Or if the
+inode belongs to a directory, then there should not be any fop happening on the
+directory and it should not contain any dentries within it. For nlookup count to
+become zero, a forget has to be sent on the inode with nlookup count set to 0 as
+an argument. For fuse clients, forget is sent by the kernel itself whenever a
+unlink/rmdir is performed. But for brick processes, upon unlink/rmdir, the
+protocol/server itself has to do inode_forget. Whenever the inode has to be
+deleted due to file removal or lru limit being exceeded  the inode is retired
+(i.e. all the dentries of the inode are deleted and the inode is moved to the
+purge list maintained by the inode table), the nlookup count is set to 0 via
+inode_forget api. The inode table, then prunes all the inodes from the purge
+list by destroying the inode contexts maintained by each xlator.
+```
+unlinking of the dentry is done via inode_unlink;
+
+void
+inode_unlink (inode_t *inode, inode_t *parent, const char *name);
+```
+If the inode has multiple hard links, then the unlink operation performed by
+the application results just in the removal of the dentry with the name provided
+by the application. For the inode to be removed, all the dentries of the inode
+should be unlinked.
+
diff --git a/doc/developer-guide/datastructure-iobuf.md b/doc/developer-guide/datastructure-iobuf.md
new file mode 100644
index 00000000000..03604e3672c
--- /dev/null
+++ b/doc/developer-guide/datastructure-iobuf.md
@@ -0,0 +1,259 @@
+# Iobuf-pool
+## Datastructures
+### iobuf
+Short for IO Buffer. It is one allocatable unit for the consumers of the IOBUF
+API, each unit hosts @page_size(defined in arena structure) bytes of memory. As
+initial step of processing a fop, the IO buffer passed onto GlusterFS by the
+other applications (FUSE VFS/ Applications using gfapi) is copied into GlusterFS
+space i.e. iobufs. Hence Iobufs are mostly allocated/deallocated in Fuse, gfapi,
+protocol xlators, and also in performance xlators to cache the IO buffers etc.
+```
+struct iobuf {
+        union {
+                struct list_head      list;
+                struct {
+                        struct iobuf *next;
+                        struct iobuf *prev;
+                };
+        };
+        struct iobuf_arena  *iobuf_arena;
+
+        gf_lock_t            lock; /* for ->ptr and ->ref */
+        int                  ref;  /* 0 == passive, >0 == active */
+
+        void                *ptr;  /* usable memory region by the consumer */
+
+        void                *free_ptr; /* in case of stdalloc, this is the
+                                          one to be freed not the *ptr */
+};
+```
+
+### iobref
+There may be need of multiple iobufs for a single fop, like in vectored read/write.
+Hence multiple iobufs(default 16) are encapsulated under one iobref.
+```
+struct iobref {
+        gf_lock_t          lock;
+        int                ref;
+        struct iobuf     **iobrefs; /* list of iobufs */
+        int                alloced; /* 16 by default, grows as required */
+        int                used;    /* number of iobufs added to this iobref */
+};
+```
+### iobuf_arenas
+One region of memory MMAPed from the operating system. Each region MMAPs
+@arena_size bytes of memory, and hosts @arena_size / @page_size IOBUFs.
+The same sized iobufs are grouped into one arena, for sanity of access.
+
+```
+struct iobuf_arena {
+        union {
+                struct list_head            list;
+                struct {
+                        struct iobuf_arena *next;
+                        struct iobuf_arena *prev;
+                };
+        };
+
+        size_t              page_size;  /* size of all iobufs in this arena */
+        size_t              arena_size; /* this is equal to
+                                           (iobuf_pool->arena_size / page_size)
+                                           * page_size */
+        size_t              page_count;
+
+        struct iobuf_pool  *iobuf_pool;
+
+        void               *mem_base;
+        struct iobuf       *iobufs;     /* allocated iobufs list */
+
+        int                 active_cnt;
+        struct iobuf        active;     /* head node iobuf
+                                           (unused by itself) */
+        int                 passive_cnt;
+        struct iobuf        passive;    /* head node iobuf
+                                           (unused by itself) */
+        uint64_t            alloc_cnt;  /* total allocs in this pool */
+        int                 max_active; /* max active buffers at a given time */
+};
+
+```
+### iobuf_pool
+Pool of Iobufs. As there may be many Io buffers required by the filesystem,
+a pool of iobufs are preallocated and kept, if these preallocated ones are
+exhausted only then the standard malloc/free is called, thus improving the
+performance. Iobuf pool is generally one per process, allocated during
+glusterfs_ctx_t init (glusterfs_ctx_defaults_init), currently the preallocated
+iobuf pool memory is freed on process exit. Iobuf pool is globally accessible
+across GlusterFs, hence iobufs allocated by any xlator can be accessed by any
+other xlators(unless iobuf is not passed).
+```
+struct iobuf_pool {
+        pthread_mutex_t     mutex;
+        size_t              arena_size; /* size of memory region in
+                                           arena */
+        size_t              default_page_size; /* default size of iobuf */
+
+        int                 arena_cnt;
+        struct list_head    arenas[GF_VARIABLE_IOBUF_COUNT];
+        /* array of arenas. Each element of the array is a list of arenas
+           holding iobufs of particular page_size */
+
+        struct list_head    filled[GF_VARIABLE_IOBUF_COUNT];
+        /* array of arenas without free iobufs */
+
+        struct list_head    purge[GF_VARIABLE_IOBUF_COUNT];
+        /* array of of arenas which can be purged */
+
+        uint64_t            request_misses; /* mostly the requests for higher
+                                               value of iobufs */
+};
+```
+~~~
+The default size of the iobuf_pool(as of yet):
+1024 iobufs of 128Bytes = 128KB
+512  iobufs of 512Bytes = 256KB
+512  iobufs of 2KB      = 1MB
+128  iobufs of 8KB      = 1MB
+64   iobufs of 32KB     = 2MB
+32   iobufs of 128KB    = 4MB
+8    iobufs of 256KB    = 2MB
+2    iobufs of 1MB      = 2MB
+Total ~13MB
+~~~
+As seen in the datastructure iobuf_pool has 3 arena lists.
+
+- arenas:  
+The arenas allocated during iobuf_pool create, are part of this list. This list
+also contains arenas that are partially filled i.e. contain few active and few
+passive iobufs (passive_cnt !=0, active_cnt!=0 except for initially allocated
+arenas). There will be by default 8 arenas of the sizes mentioned above.
+- filled:  
+If all the iobufs in the arena are filled(passive_cnt = 0), the arena is moved
+to the filled list. If any of the iobufs from the filled arena is iobuf_put,
+then the arena moves back to the 'arenas' list.
+- purge:  
+If there are no active iobufs in the arena(active_cnt = 0), the arena is moved
+to purge list. iobuf_put() triggers destruction of the arenas in this list. The
+arenas in the purge list are destroyed only if there is  atleast one arena in
+'arenas' list, that way there won't be spurious mmap/unmap of buffers.
+(e.g: If there is an arena (page_size=128KB, count=32) in purge list, this arena
+is destroyed(munmap) only if there is an arena in 'arenas' list with page_size=128KB).
+
+## APIs
+### iobuf_get
+
+```
+struct iobuf *iobuf_get (struct iobuf_pool *iobuf_pool);
+```
+Creates a new iobuf of the default page size(128KB hard coded as of yet).
+Also takes a reference(increments ref count), hence no need of doing it
+explicitly after getting iobuf.
+
+### iobuf_get2
+
+```
+struct iobuf * iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size);
+```
+Creates a new iobuf of a specified page size, if page_size=0 default page size
+is considered.
+```
+if (requested iobuf size > Max iobuf size in the pool(1MB as of yet))
+  {
+     Perform standard allocation(CALLOC) of the requested size and
+     add it to the list iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX].
+  }
+  else
+  {
+     -Round the page size to match the stndard sizes in iobuf pool.
+     (eg: if 3KB is requested, it is rounded to 8KB).
+     -Select the arena list corresponding to the rounded size
+     (eg: select 8KB arena)
+     If the selected arena has passive count > 0, then return the
+     iobuf from this arena, set the counters(passive/active/etc.)
+     appropriately.
+     else the arena is full, allocate new arena with rounded size
+     and standard page numbers and add to the arena list
+     (eg: 128 iobufs of 8KB is allocated).
+  }
+```
+Also takes a reference(increments ref count), hence no need of doing it
+explicitly after getting iobuf.
+
+### iobuf_ref
+
+```
+struct iobuf *iobuf_ref (struct iobuf *iobuf);
+```
+  Take a reference on the iobuf. If using an iobuf allocated by some other
+xlator/function/, its a good practice to take a reference so that iobuf is not
+deleted by the allocator.
+
+### iobuf_unref
+```
+void iobuf_unref (struct iobuf *iobuf);
+```
+Unreference the iobuf, if the ref count is zero iobuf is considered free.
+
+```
+  -Delete the iobuf, if allocated from standard alloc and return.  
+  -set the active/passive count appropriately.  
+  -if passive count > 0 then add the arena to 'arena' list.  
+  -if active count = 0 then add the arena to 'purge' list.  
+```
+Every iobuf_ref should have a corresponding iobuf_unref, and also every
+iobuf_get/2 should have a correspondning iobuf_unref.
+
+### iobref_new
+```
+struct iobref *iobref_new ();
+```
+Creates a new iobref structure and returns its pointer.
+
+### iobref_ref
+```
+struct iobref *iobref_ref (struct iobref *iobref);
+```
+Take a reference on the iobref.
+
+### iobref_unref
+```
+void iobref_unref (struct iobref *iobref);
+```
+Decrements the reference count of the iobref. If the ref count is 0, then unref
+all the iobufs(iobuf_unref) in the iobref, and destroy the iobref.
+
+### iobref_add
+```
+int iobref_add (struct iobref *iobref, struct iobuf *iobuf);
+```
+Adds the given iobuf into the iobref, it takes a ref on the iobuf before adding
+it, hence explicit iobuf_ref is not required if adding to the iobref.
+
+### iobref_merge
+```
+int iobref_merge (struct iobref *to, struct iobref *from);
+```
+Adds all the iobufs in the 'from' iobref to the 'to' iobref. Merge will not
+cause the delete of the 'from' iobref, therefore it will result in another ref
+on all the iobufs added to the 'to' iobref. Hence iobref_unref should be
+performed both on 'from' and 'to' iobrefs (performing iobref_unref only on 'to'
+will not free the iobufs and may result in leak).
+
+### iobref_clear
+```
+void iobref_clear (struct iobref *iobref);
+```
+Unreference all the iobufs in the iobref, and also unref the iobref.
+
+## Iobuf Leaks
+If all iobuf_refs/iobuf_new do not have correspondning iobuf_unref, then the
+iobufs are not freed and recurring execution of such code path may lead to huge
+memory leaks. The easiest way to identify if a memory leak is caused by iobufs
+is to take a statedump. If the statedump shows a lot of filled arenas then it is
+a sure sign of leak. Refer doc/debugging/statedump.md for more details.
+
+If iobufs are leaking, the next step is to find where the iobuf_unref went
+missing. There is no standard/easy way of debugging this, code reading and logs
+are the only ways. If there is a liberty to reproduce the memory leak at will,
+then logs(gf_callinginfo) in iobuf_ref/unref might help.  
+TODO: A easier way to debug iobuf leaks.
diff --git a/doc/developer-guide/datastructure-mem-pool.md b/doc/developer-guide/datastructure-mem-pool.md
new file mode 100644
index 00000000000..225567cbf9f
--- /dev/null
+++ b/doc/developer-guide/datastructure-mem-pool.md
@@ -0,0 +1,124 @@
+# Mem-pool
+## Background
+There was a time when every fop in glusterfs used to incur cost of allocations/de-allocations for every stack wind/unwind between xlators because stack/frame/*_localt_t in every wind/unwind was allocated and de-allocated. Because of all these system calls in the fop path there was lot of latency and the worst part is that most of the times the number of frames/stacks active at any time wouldn't cross a threshold. So it was decided that this threshold number of frames/stacks would be allocated in the beginning of the process only once. Get one of them from the pool of stacks/frames whenever `STACK_WIND` is performed and put it back into the pool in `STACK_UNWIND`/`STACK_DESTROY` without incurring any extra system calls. The data structures are allocated only when threshold number of such items are in active use i.e. pool is in complete use.% increase in the performance once this was added to all the common data structures (inode/fd/dict etc) in xlators throughout the stack was tremendous.
+
+## Data structure
+```
+struct mem_pool {
+        struct list_head  list; /*Each member in the mempool is element padded with a doubly-linked-list + ptr of mempool + is-in
+-use info. This list is used to add the element to the list of free members in the mem-pool*/
+        int               hot_count;/*number of mempool elements that are in active use*/
+        int               cold_count;/*number of mempool elements that are not in use. If a new allocation is required it
+will be served from here until all the elements in the pool are in use i.e. cold-count becomes 0.*/
+        gf_lock_t         lock;/*synchronization mechanism*/
+        unsigned long     padded_sizeof_type;/*Each mempool element is padded with a doubly-linked-list + ptr of mempool + is-in
+-use info to operate the pool of elements, this size is the element-size after padding*/
+        void             *pool;/*Starting address of pool*/
+        void             *pool_end;/*Ending address of pool*/
+/* If an element address is in the range between pool, pool_end addresses  then it is alloced from the pool otherwise it is 'calloced' this is very useful for functions like 'mem_put'*/
+        int               real_sizeof_type;/* size of just the element without any padding*/
+        uint64_t          alloc_count; /*Number of times this type of data is allocated through out the life of this process. This may include calloced elements as well*/
+        uint64_t          pool_misses; /*Number of times the element had to be allocated from heap because all elements from the pool are in active use.*/
+        int               max_alloc; /*Maximum number of elements from the pool in active use at any point in the life of the process. This does *not* include calloced elements*/
+        int               curr_stdalloc;/*Number of elements that are allocated from heap at the moment because the pool is in completed use. It should be '0' when pool is not in complete use*/
+        int               max_stdalloc;/*Maximum number of allocations from heap after the pool is completely used that are in active use at any point in the life of the process.*/
+        char             *name; /*Contains xlator-name:data-type as a string
+        struct list_head  global_list;/*This is used to insert it into the global_list of mempools maintained in 'glusterfs-ctx'
+};
+```
+
+## Life-cycle
+```
+mem_pool_new (data_type, unsigned long count)
+
+This is a macro which expands to mem_pool_new_fn (sizeof (data_type), count, string-rep-of-data_type)
+
+struct mem_pool *
+mem_pool_new_fn (unsigned long sizeof_type, unsigned long count, char *name)
+
+Padded-element:
+ ----------------------------------------
+|list-ptr|mem-pool-address|in-use|Element|
+ ----------------------------------------
+ ```
+
+This function allocates the `mem-pool` structure and sets up the pool for use.
+`name` parameter above is the `string` containing type of the datatype. This `name` is appended to `xlator-name + ':'` so that it can be easily identified in things like statedump. `count` is the number of elements that need to be allocated. `sizeof_type` is the size of each element. Ideally `('sizeof_type'*'count')` should be the size of the total pool. But to manage the pool using `mem_get`/`mem_put` (will be explained after this section) each element needs to be padded in the front with a `('list', 'mem-pool-address', 'in_use')`. So the actual size of the pool it allocates will be `('padded_sizeof_type'*'count')`. Why these extra elements are needed will be evident after understanding how `mem_get` and `mem_put` are implemented. In this function it just initializes all the `list` structures in front of each element and adds them to the `mem_pool->list` which represent the list of `cold` elements which can be allocated whenever `mem_get` is called on this mem_pool. It remembers mem_pool's start and end addresses in `mem_pool->pool`, `mem_pool->pool_end` respectively. Initializes `mem_pool->cold_count` to `count` and `mem_pool->hot_count` to `0`. This mem-pool will be added to the list of `global_list` maintained in `glusterfs-ctx`
+
+
+```
+void* mem_get (struct mem_pool *mem_pool)
+
+Initial-list before mem-get
+----------------
+|     Pool       |
+|   -----------  |       ----------------------------------------       ----------------------------------------
+|  | pool-list | |<---> |list-ptr|mem-pool-address|in-use|Element|<--->|list-ptr|mem-pool-address|in-use|Element|
+|   -----------  |       ----------------------------------------       ----------------------------------------
+----------------
+
+list after mem-get from the pool
+----------------
+|     Pool       |
+|   -----------  |      ----------------------------------------
+|  | pool-list | |<--->|list-ptr|mem-pool-address|in-use|Element|
+|   -----------  |      ----------------------------------------
+----------------
+
+List when the pool is full:
+ ----------------
+|     Pool       |       extra element that is allocated
+|   -----------  |      ----------------------------------------
+|  | pool-list | |     |list-ptr|mem-pool-address|in-use|Element|
+|   -----------  |      ----------------------------------------
+ ----------------
+```
+
+This function is similar to `malloc()` but it gives memory of type `element` of this pool. When this function is called it increments `mem_pool->alloc_count`, checks if there are any free elements in the pool that can be returned by inspecting `mem_pool->cold_count`. If `mem_pool->cold_count` is non-zero then it means there are elements in the pool which are not in active use. It deletes one element from the list of free elements and decrements `mem_pool->cold_count` and increments `mem_pool->hot_count` to indicate there is one more element in active use. Updates `mem_pool->max_alloc` accordingly. Sets `element->in_use` in the padded memory to `1`. Sets `element->mem_pool` address to this mem_pool also in the padded memory(It is useful for mem_put). Returns the address of the memory after the padded boundary to the caller of this function. In the cases where all the elements in the pool are in active use it `callocs` the element with padded size and sets mem_pool address in the padded memory. To indicate the pool-miss and give useful accounting information of the pool-usage it increments `mem_pool->pool_misses`, `mem_pool->curr_stdalloc`. Updates `mem_pool->max_stdalloc` accordingly.
+
+```
+void* mem_get0 (struct mem_pool *mem_pool)
+```
+Just like `calloc` is to `malloc`, `mem_get0` is to `mem_get`. It memsets the memory to all '0' before returning the element.
+
+
+```
+void mem_put (void *ptr)
+
+list before mem-put from the pool
+ ----------------
+|     Pool       |
+|   -----------  |      ----------------------------------------
+|  | pool-list | |<--->|list-ptr|mem-pool-address|in-use|Element|
+|   -----------  |      ----------------------------------------
+ ----------------
+
+list after mem-put to the pool
+ ----------------
+|     Pool       |
+|   -----------  |       ----------------------------------------       ----------------------------------------
+|  | pool-list | |<---> |list-ptr|mem-pool-address|in-use|Element|<--->|list-ptr|mem-pool-address|in-use|Element|
+|   -----------  |       ----------------------------------------       ----------------------------------------
+ ----------------
+
+If mem_put is putting an element not from pool then it is just freed so
+no change to the pool
+ ----------------
+|     Pool       |
+|   -----------  |
+|  | pool-list | |
+|   -----------  |
+ ----------------
+```
+
+This function is similar to `free()`. Remember that ptr passed to this function is the address of the element, so this function gets the ptr to its head of the padding in front of it. If this memory falls in bettween `mem_pool->pool`, `mem_pool->pool_end` then the memory is part of the 'pool' memory that is allocated so it does some sanity checks to see if the memory is indeed head of the element by checking if `in_use` is set to `1`. It resets `in_use`  to `0`. It gets the mem_pool address stored in the padded region and adds this element to the list of free elements. Decreases `mem_pool->hot_count` increases `mem_pool->cold_count`. In the case where padded-element address does not fall in the range of `mem_pool->pool`, `mem_pool->pool_end` it just frees the element and decreases `mem_pool->curr_stdalloc`.
+
+```
+void
+mem_pool_destroy (struct mem_pool *pool)
+```
+Deletes this pool from the `global_list` maintained by `glusterfs-ctx` and frees all the memory allocated in `mem_pool_new`.
+
+
+### How to pick pool-size
+This varies from work-load to work-load. Create the mem-pool with some random size and run the work-load. Take the statedump after the work-load is complete. In the statedump if `max_alloc` is always less than `cold_count` may be reduce the size of the pool closer to `max_alloc`. On the otherhand if there are lots of `pool-misses` then increase the `pool_size` by `max_stdalloc` to achieve better 'hit-rate' of the pool.
diff --git a/doc/developer-guide/dirops-transactions-in-dht.md b/doc/developer-guide/dirops-transactions-in-dht.md
new file mode 100644
index 00000000000..909a97001aa
--- /dev/null
+++ b/doc/developer-guide/dirops-transactions-in-dht.md
@@ -0,0 +1,273 @@
+# dirops transactions in dht
+Need for transactions during operations on directories arise from two
+basic design elements of DHT:
+
+ 1. A directory is created on all subvolumes of dht. Since glusterfs
+ associates each file-system object with an unique gfid, every
+ subvolume should have the same unique mapping of (path of directory,
+ gfid). To elaborate,
+    * Each subvolume should've same gfid associated with a path to
+ directory.
+    * A gfid should not be associated with more than one path in any
+ subvolume.
+
+ So, entire operations like mkdir, renamedir, rmdir and creation of
+ directories during self-heal need to be atomic in dht. In other words,
+ any of these operations shouldn't begin on an inode if one of them is
+ already in progress on the same inode, till it completes on all
+ subvolumes of dht. If not, more than one of these operations
+ happening in parallel can break any or all of the two requirements
+ listed above. This is referred in the rest of the document by the
+ name _Atomicity during namespace operations_.
+   
+ 2. Each directory has an independent layout persisted on
+ subvolumes. Each subvolume contains only part of the layout relevant
+ to it. For performance reasons _and_ since _only_ dht has aggregated
+ view, this layout is cached in memory of client. To make sure dht
+ reads or modifies a single complete layout while parallel modifications of    the layout are in progress, we need atomicity during layout modification and reading. This is referred in the rest of the document as  _Atomicity during layout modification and reading_.
+
+Rest of the document explains how atomicity is achieved for each of
+the case above.
+
+**Atomicity during layout modification and reading**
+File operations a.k.a fops can be classified into two categories based on how they consume layout.
+
+ - Layout writer. Setting of layout during selfheal of a directory is
+ layout writer of _that_ directory. 
+ - Layout reader. 
+     * Any entry fop like create, unlink, rename, link, symlink,
+ unlink, mknod, rename, mkdir, rmdir, renamedir which needs layout of the parent directory. Each of these fops are readers of layout on parent directory.
+     *  setting of layout during mkdir of a directory is considered as
+ a reader of the same directory's layout. The reason for this is that
+ only a parallel lookup on that directory can be a competing fop that modifies the layout (Other fops need gfid of the directory which can be got only after either lookup or mkdir completes). However, healing of layout is considered as a writer and a single writer blocks all readers.
+ 
+*Algorithm*
+Atomicity is achieved by locking on the inode of directory whose
+layout is being modified or read. The fop used is inodelk.
+ -  Writer acquires blocking inodelk (directory-inode, write-lock) on
+ all subvolumes serially. The order of subvols in which they are
+ locked by different clients remains constant for a directory. If locking fails on any subvolume, layout modification is abandoned.
+ - Reader acquires an inodelk (directory-inode, read-lock) on _any_
+ one subvolume. If locking fails on a subvolume (say with
+ ESTALE/ENOTCONN error), locking can be tried on other subvolumes till
+ we get one lock. If we cannot get lock on at least one subvolume,
+ consistency of layout is not guaranteed. Based on the consistency
+ requirements of fops, they can be failed or continued.
+
+Reasons why writer has to lock on _all_ subvols:
+
+ - DHT don't have a metadata server and locking is implemented by brick. So, there is no well-defined subvol/brick that can be used as an arbitrator by different clients while acquiring locks.
+ - readers should acquire as minimum number of locks as possible. In
+ other words, the algorithm aims to have less synchronization cost to
+ readers.
+ - The subvolume to which a directory hashes could be used as a
+ lock server. However, in the case of an entry fop like create
+ (/a/b/c) where we want to block modification of layout of b for the
+ duration of create, we would be required to acquire lock on the
+ subvol to which /a/b hashes. To find out the hashed-subvol of
+ /a/b, we would need layout of /a. Note that how there is a dependency
+ of locking the layouts of ancestors all the way to root. So this
+ locking is not preferred. Also, note that only the immediate parent
+ inode is available in arguments of a fop like create.
+
+**Atomicity during namespace operations**
+
+ -  We use locks on inode of parent directory in the namespace of
+ _"basename"_  during mkdir, rmdir, renamedir and directory
+ creation phase of self-heal. The exact fop we use is _entrylk
+ (parent-inode, "basename")_.
+ - refresh in-memory layout of parent-inode from values stored on backend
+ -  _entrylk (parent-inode, "basename")_ is done on subvolume to which
+ _"basename" hashes_. So, this operation is a _reader_ of the
+ layout on _parent-inode_. Which means an _inodelk (parent-inode,
+ read-lock)_ has to be completed before _entrylk (parent-inode,
+ "basename")_ is issued. Both the locks have to be held till the
+ operation is tried on all subvolumes. If acquiring of any/all of
+ these locks fail, the operation should be failed.
+
+With the above details, algorithms for mkdir, rmdir, renamedir,
+self-heal of directory are explicitly detailed below.
+
+**Self-heal of a directory**
+  
+ -  creation of directories on subvolumes is done only during
+ _named-lookup_ of a directory as we need < parent-inode,
+ "basename" >.
+ -  If a directory is missing on one or more subvolumes,
+    *  acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+    * refresh the in-memory layout of parent-inode from values stored on backend
+    *  acquire _entrylk (parent-inode, "basename")_ on the subvolume
+ to which _"basename"_ hashes.
+    *  If any/all of the locks fail, self-heal is aborted.
+    *  create directories on missing subvolumes.
+    *  release _entrylk (parent-inode, "basename")_.
+    *  release _inodelk (parent-inode, read-lock)_.
+    
+ -  If layout of a directory needs healing
+     * acquire _inodelk (directory-inode, write-lock)_ on _all_ the
+ subvolumes. If locking fails on any of the subvolumes,
+ self-heal is aborted. Blocking Locks are acquired serially across subvolumes in a _well-defined_ order which is _constant_ across all the healers of a directory. One order could be the order in which subvolumes are stored in the array _children_ of dht xlator.
+     *  heal the layout.
+     *  release _inodelk (directory-inode, write-lock)_ on _all_ the
+ subvolumes in parallel.
+     *  Note that healing of layout can be done in both _named_ and
+ _nameless_ lookups of a directory as _only directory-inode_ is needed
+ for healing and it is available during both.
+
+**mkdir (parent-inode, "basename")**
+
+* while creating directory across subvolumes,
+
+ - acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+ - refresh in-memory layout of parent-inode from values stored on backend
+ - acquire _entrylk (parent-inode, "basename")_ on the subvolume to
+ which _"basename"_ hashes.
+ -  If any/all of the above two locks fail, release the locks that
+ were acquired successfully and mkdir should be failed (as perceived by application).
+ - do _mkdir (parent-inode, "basename")_ on the subvolume to which
+ _"basename"_ hashes. If this mkdir fails, mkdir is failed.
+ - do _mkdir (parent-inode, "basename")_ on the remaining subvolumes.
+ - release _entrylk (parent-inode, "basename")_.
+ - release _inodelk (parent-inode, "read-lock")_.
+*  while setting the layout of a directory,
+ - acquire _inodelk (directory-inode, read-lock)_ on _any one_ of the
+ subvolumes.
+ -  If locking fails, cleanup the locks that were acquired
+ successfully and abort layout setting. Note that we'll have a
+ directory without a layout till a lookup happens on the
+ directory. This means entry operations within this directory fail
+ in this time window. We can also consider failing mkdir. The
+ problem of dealing with a directory without layout is out of the
+ scope of this document. 
+ -  set the layout on _directory-inode_.
+ -  release _inodelk (directory-inode, read-lock)_.
+* Note that during layout setting we consider mkdir as a  _reader_ not
+ _writer_, though it is setting the layout. Reasons are:
+ - Before any of other readers like create, link etc that operate on
+ this directory to happen, _gfid_ of this directory has to be
+ resolved. But _gfid_ is only available only if either of following
+ conditions are true: 
+      * after mkdir is complete.
+      * a lookup on the same path happens parallel to in-progress
+ mkdir.
+
+     But, on completion of any of the above two operations, layout
+     will be healed. So, none of the _readers_ will happen on a
+     directory with partial layout.
+
+* Note that since we've an _entrylk (parent-inode, "basename")_ for
+   the entire duration of (attempting) creating directories, parallel
+   mkdirs will no longer contend on _mkdir_ on subvolume _to which  "basename"  hashes_. But instead, contend on _entrylk (parent-inode, "basename")_ on the subvolume _to which "basename" hashes_. So, we can attempt the _mkdir_ in _parallel_ on all subvolumes instead of two stage mkdir on hashed first and the rest of them in parallel later. However, we need to make sure that mkdir is successful on the subvolume _to which "basename" hashes_ for mkdir to be successful (as perceived by application). In the case of failed mkdir (as perceived by application), a cleanup should be performed on all the subvolumes before _entrylk (parent-inode, "basename")_ is released.
+
+**rmdir (parent-inode, "basename", directory-inode)**
+
+ -  acquire _inodelk (parent-inode, read-lock)_ on _any one_
+ subvolume.
+ -  refresh in-memory layout of parent-inode from values stored on backend
+ -  acquire _entrylk (parent-inode, "basename")_ on the subvolume to
+ which _"basename" hashes_.
+ -  If any/all of the above locks fail, rmdir is failed after cleanup
+ of the locks that were acquired successfully.
+ -  do _rmdir (parent-inode, "basename")_ on the subvolumes to which
+ _"basename" doesn't hash to_.
+    * If successful, continue. 
+    * Else,
+     *  recreate directories on those subvolumes where rmdir
+  succeeded.
+     *  heal the layout of _directory-inode_. Note that this will have
+  same synchronization requirements as discussed during layout
+  healing part of the section "Directoy self-heal" above.
+     *  release _entrylk (parent-inode, "basename")_.
+     *  release _inodelk (parent-inode, read-lock)_.
+     *  fail _rmdir (parent-inode, "basename")_ to application.
+ - do _rmdir (parent-inode, "basename")_ on the subvolume to which
+ _"basename" hashes_. 
+ - If successful, continue.
+ -  Else, Go to the failure part of _rmdir (parent-inode, "basename")_
+ on subvolumes to which "basename" _doesn't hash to_. 
+ - release _entrylk (parent-inode, "basename")_.
+ - release _inodelk (parent-inode, read-lock)_.
+ - return success to application.
+ 
+**renamedir (src-parent-inode, "src-basename", src-directory-inode, dst-parent-inode, "dst-basename", dst-directory-inode)**
+
+ -  requirement is to prevent any operation in both _src-namespace_
+ and _dst-namespace_. So, we need to acquire locks on both
+ namespaces.We also need to have constant ordering while acquiring
+ locks during parallel renames of the form _rename (src, dst)_ and
+ _rename (dst, src)_ to prevent deadlocks. We can sort gfids of
+ _src-parent-inode_ and _dst-parent-inode_ and use that order to
+ acquire locks. For the sake of explanation lets say we ended up
+ with order of _src_ followed by _dst_.
+ -  acquire _inodelk (src-parent-inode, read-lock)_.
+ -  refresh in-memory layout of src-parent-inode from values stored on backend
+ -  acquire _entrylk (src-parent-inode, "src-basename")_.
+ -  acquire _inodelk (dst-parent-inode, read-lock)_.
+ -  refresh in-memory layout of dst-parent-inode from values stored on backend
+ -  acquire _entrylk (dst-parent-inode, "dst-basename")_.
+ - If acquiring any/all of the locks above fail,
+      * release the locks that were successfully acquired.
+      * fail the renamedir operation to application
+      *  done
+ - do _renamedir ("src", "dst")_ on the subvolume _to which "dst-basename" hashes_.
+    *  If failure, Goto point _If acquiring any/all of the locks above fail_.
+    *  else, continue.
+ - do _renamedir ("src", "dst")_ on rest of the subvolumes.
+    *  If there is any failure, 
+      * revert the successful renames.
+      *  Goto to point  _If acquiring any/all of the locks above fail_.
+    *  else,
+        -  release all the locks acquired.
+        -  return renamedir as success to application.
+
+**Some examples of races**
+This section gives concrete examples of races that can result in inconsistencies explained in the beginning of the document. 
+
+Some assumptions are:
+
+*  We consider an example distribute of three subvols s1, s2 and s3.
+*  For examples of renamedir ("src", "dst"), _src_ hashes to s1 and _dst_ hashes to s2. _src_ and _dst_ are associated with _gfid-src_ and _gfid-dst_ respectively
+*  For non renamedir examples, _dir_ is the name of directory and it hashes to s1.
+
+And the examples are:  
+
+ -  mkdir vs rmdir - inconsistency in namespace.
+   *  mkdir ("dir", gfid1) is complete on s1
+   *  rmdir is issued on same directory. Note that, since rmdir needs a gfid, a lookup should be complete before rmdir. lookup creates the directory on rest of the subvols as part of self-heal.
+   *  rmdir (gfid1) deletes directory from all subvols.
+   *  A new mkdir ("dir", gfid2) is issued. It is successful on s1 associating "dir" with gfid2.
+   *  mkdir ("dir", gfid1) resumes and creates directory on s2 and s3 associating "dir" with gfid1.
+   *  mkdir ("dir", gfid2) fails with EEXIST on s2 and s3. Since, EEXIST errors are ignored, mkdir is considered successful to application.
+   *  In this example we have multiple inconsitencies
+      *  "dir" is associated with gfid1 on s2, s3 and with gfid2 on s1
+      *  Even if mkdir ("dir", gfid2) was not issued, we would've a case of a directory magically reappearing after a successful rmdir.
+ -  lookup heal vs rmdir
+   * rmdir ("dir", gfid1) is issued. It is successful on s2 and s3 (non-hashed subvols for name "dir")
+   *  lookup ("dir") is issued. Since directory is present on s1 yet, it is created on s2 and s3 associating with gfid1 as part of self-heal
+   *  rmdir ("dir", gfid1) is complete on s1 and it is successful
+   *  Another lookup ("dir") creates the directory on s1 too
+   *  "dir" magically reappears after a successful rmdir
+ - lookup heal (src) vs renamedir ("src", "dst")
+   *  renamedir ("src", "dst") complete on s2
+   *  lookup ("src") recreates _src_ with _gfid-src_ on s2
+   *  renamedir ("src", "dst") completes on s1, s3. After rename is complete path _dst_ will be associated with gfid _gfid-src_
+   *  Another lookup ("src") recreates _src_ on subvols s1 and s3, associating it with gfid _gfid-src_
+   *  Inconsistencies are
+      * after a successful renamedir ("src", "dst"), both src and dst exist
+      *  Two directories - src and dst - are associated with same gfid. One common symptom is that some entries (of the earlier _src_ and current _dst_ directory) being missed out in readdir listing as the gfid handle might be pointing to the empty healed directory than the actual directory containing entries
+ - lookup heal (dst) vs renamedir ("src", "dst")
+   *  dst exists and empty when renamdir started
+   *  dst doesn't exist when renamedir started
+      -  renamedir ("src", "dst") complete on s2 and s3
+      -  lookup ("dst") creates _dst_ associating it with _gfid-src_ on s1
+      -  An entry is created in _dst_ on either s1
+      -  renamedir ("src", "dst") on s1 will result in a directory _dst/dst_ as _dst_ is no longer empty and _man 2 rename_ states that if _dst_ is not empty, _src_ is renamed _as a subdirectory of dst_
+      -  A lookup ( _dst/dst_) creates _dst/dst_ on s2 and s3 associating with _gfid-src_ as part of self-heal
+      -  Inconsistencies are:
+         * Two directories - _dst_ and _dst/dst_ - exist even though both of them didn't exist at the beginning of renamedir
+         *  Both _dst_ and _dst/dst_ have same gfid - _gfid-src_. As observed earlier, symptom might be directory listing being incomplete
+ - mkdir (dst) vs renamedir ("src", "dst")
+ - rmdir (src) vs renamedir ("src", "dst")
+ - rmdir (dst) vs renamedir ("src", "dst")
diff --git a/doc/developer-guide/ec-implementation.md b/doc/developer-guide/ec-implementation.md
new file mode 100644
index 00000000000..77e62583caa
--- /dev/null
+++ b/doc/developer-guide/ec-implementation.md
@@ -0,0 +1,588 @@
+Erasure coding implementation
+=============================
+
+This document provides information about how [erasure code][1] has
+been implemented into ec translator. It describes the algorithm used
+and the optimizations made, but it doesn't contain a full description
+of the mathematical background needed to understand erasure coding in
+general. It either describes the other parts of ec not directly
+related to the encoding/decoding procedure, like synchronization or
+fop management.
+
+
+Introduction
+------------
+
+EC is based on [Reed-Solomon][2] erasure code. It's a very old code.
+It's not considered the best one nowadays, but is good enough and it's
+one of the few codes that is not covered by any patent and can be
+freely used.
+
+To define the Reed-Solomon code we use 3 parameters:
+
+  * __Key fragments (K)__  
+    It represents the minimum number of healthy fragments that will be
+    needed to be able to recover the original data. Any subset of K
+    out of the total number of fragments will serve.
+
+  * __Redundancy fragments (R)__  
+    It represents the number of extra fragments to compute for each
+    original data block. This value determines how many fragments can
+    be lost before being unable to recover the original data.
+
+  * __Fragment size (S)__  
+    This determines the size of each fragment. The original data
+    block size is computed as S * K. Currently this values is fixed
+    to 512 bytes.
+
+  * __Total number of fragments (N = K + R)__  
+    This isn't a real parameter but it will be useful to simplify
+    the following descriptions.
+
+From the point of view of the implementation, it only consists on
+matrix multiplications. There are two kinds of matrices to use for
+Reed-Solomon:
+
+  * __[Systematic][3]__  
+    This kind of matrix has the particularity that K of the encoded
+    fragments are simply a copy of the original data, divided into K
+    pieces. Thus no real encoding needs to be done for them and only
+    the R redundancy fragments need to be computed.
+
+    This kind of matrices contain one KxK submatrix that is the
+    [identity matrix][4].
+
+  * __Non-systematic__  
+    This kind of matrix doesn't contain an identity submatrix. This
+    means that all of the N fragments need to be encoded, requiring
+    more computation. On the other hand, these matrices have some nice
+    properties that allow faster implementations of some algorithms,
+    like the matrix inversion used to decode the data.
+
+    Another advantage of non-systematic matrices is that the decoding
+    time is constant, independently of how many fragments are lost,
+    while systematic approach can suffer from performance degradation
+    when one fragment is lost.
+
+All non-systematic matrices can be converted to systematic ones, but
+then we lose the good properties of the non-systematic. We have to
+choose betwee best peek performance (systematic) and performance
+stability (non-systematic).
+
+
+Encoding procedure
+------------------
+
+To encode a block of data we need a KxN matrix where each subset of K
+rows is [linearly independent][5]. In other words, the determinant of
+each KxK submatrix is not 0.
+
+There are some known ways to obtain this kind of matrices. EC uses a
+small variation of a matrix known as [Vandermonde Matrix][6] where
+each element of the matrix is defined as:
+
+    a(i, j) = i ^ (K - j)
+
+    where i is the row from 1 to N, and j is the column from 1 to K.
+
+This is exactly the Vandermonde Matrix but with the elements of each
+row in reverse order. This change is made to be able to implement a
+small optimization in the matrix multiplication.
+
+Once we have the matrix, we only need to compute the multiplication
+of this matrix by a vector composed of K elements of data coming from
+the original data block.
+
+     /                 \             /                               \
+    |    1   1  1  1  1 |    / \    |     a +    b +   c +  d + e = t |
+    |   16   8  4  2  1 |   | a |   |   16a +   8b +  4c + 2d + e = u |
+    |   81  27  9  3  1 |   | b | = |   81a +  27b +  9c + 3d + e = v |
+    |  256  64 16  4  1 | * | c |   |  256a +  64b + 16c + 4d + e = w |
+    |  625 125 25  5  1 |   | d |   |  625a + 125b + 25c + 5d + e = x |
+    | 1296 216 36  6  1 |   | e |   | 1296a + 216b + 36c + 6d + e = y |
+    | 2401 343 49  7  1 |    \ /    | 2401a + 343b + 49c + 7d + e = z |
+     \                 /             \                               /
+
+The optimization that can be done here is this:
+
+    16a + 8b + 4c + 2d + e = 2(2(2(2a + b) + c) + d) + e
+
+So all the multiplications are always by the number of the row (2 in
+this case) and we don't need temporal storage for intermediate
+results:
+
+    a *= 2
+    a += b
+    a *= 2
+    a += c
+    a *= 2
+    a += d
+    a *= 2
+    a += e
+
+Once we have the result vector, each element is a fragment that needs
+to be stored in a separate place.
+
+
+Decoding procedure
+------------------
+
+To recover the data we need exactly K of the fragments. We need to
+know which K fragments we have (i.e. the original row number from
+which each fragment was calculated). Once we have this data we build
+a square KxK matrix composed by the rows corresponding to the given
+fragments and invert it.
+
+With the inverted matrix, we can recover the original data by
+multiplying it with the vector composed by the K fragments.
+
+In our previous example, if we consider that we have recovered
+fragments t, u, v, x and z, corresponding to rows 1, 2, 3, 5 and 7,
+we can build the following matrix:
+
+     /                 \
+    |    1   1  1  1  1 |
+    |   16   8  4  2  1 |
+    |   81  27  9  3  1 |
+    |  625 125 25  5  1 |
+    | 2401 343 49  7  1 |
+     \                 /
+
+And invert it:
+
+     /                                     \
+    |    1/48  -1/15    1/16  -1/48   1/240 |
+    |  -17/48  16/15  -15/16  13/48 -11/240 |
+    |  101/48 -86/15   73/16 -53/48  41/240 |
+    | -247/48 176/15 -129/16  83/48 -61/240 |
+    |   35/8   -7      35/8   -7/8    1/8   |
+     \                                     /
+
+Multiplying it by the vector (t, u, v, x, z) we recover the original
+data (a, b, c, d, e):
+
+     /                                     \     / \     / \
+    |    1/48  -1/15    1/16  -1/48   1/240 |   | t |   | a |
+    |  -17/48  16/15  -15/16  13/48 -11/240 |   | u |   | b |
+    |  101/48 -86/15   73/16 -53/48  41/240 | * | v | = | c |
+    | -247/48 176/15 -129/16  83/48 -61/240 |   | x |   | d |
+    |   35/8   -7      35/8   -7/8    1/8   |   | z |   | e |
+     \                                     /     \ /     \ /
+
+
+Galois Field
+------------
+
+This encoding/decoding procedure is quite complex to compute using
+regular mathematical operations and it's not well suited for what
+we want to do (note that matrix elements can grow unboundly).
+
+To solve this problem, exactly the same procedure is done inside a
+[Galois Field][7] of characteristic 2, which is a finite field with
+some interesting properties that make it specially useful for fast
+operations using computers.
+
+There are two main differences when we use this specific Galois Field:
+
+  * __All regular additions are replaced by bitwise xor's__  
+    For todays computers it's not really faster to execute an xor
+    compared to an addition, however replacing additions by xor's
+    inside a multiplication has many advantages (we will make use of
+    this to optimize the multiplication).
+
+    Another consequence of this change is that additions and
+    substractions are really the same xor operation.
+
+  * __The elements of the matrix are bounded__  
+    The field uses a modulus that keep all possible elements inside
+    a delimited region, avoiding really big numbers and fixing the
+    number of bits needed to represent each value.
+
+    In the current implementation EC uses 8 bits per field element.
+
+It's very important to understand how multiplications are computed
+inside a Galois Field to be able to understand how has it been
+optimized.
+
+We'll start with a simple 'old school' multiplication but in base 2.
+For example, if we want to multiply 7 * 5 (111b * 101b in binary), we
+do the following:
+
+          1 1 1 (= 7)
+    *     1 0 1 (= 5)
+    -----------
+          1 1 1 (= 7)
+    +   0 0 0   (= 0)
+    + 1 1 1     (= 7)
+    -----------
+    1 0 0 0 1 1 (= 35)
+
+This is quite simple. Note that the addition of the third column
+generates a carry that is propagated to all the other left columns.
+
+The next step is to define the modulus of the field. Suppose we use
+11 as the modulus. Then we convert the result into an element of the
+field by dividing by the modulus and taking the residue. We also use
+the 'old school' method in binary:
+
+
+      1 0 0 0 1 1 (= 35) | 1 0 1 1 (= 11)
+    - 0 0 0 0            ----------------
+    ---------              0 1 1 (= 3)
+      1 0 0 0 1
+    -   1 0 1 1
+    -----------
+      0 0 1 1 0 1
+    -     1 0 1 1
+    -------------
+          0 0 1 0 (= 2)
+
+So, 7 * 5 in a field with modulus 11 is 2. Note that the main
+objective in each iteration of the division is to make higher bits
+equal to 0 when possible (if it's not possible in one iteration, it
+will be zeroed on the next).
+
+If we do the same but changing additions with xors we get this:
+
+          1 1 1 (= 7)
+    *     1 0 1 (= 5)
+    -----------
+          1 1 1 (= 7)
+    x   0 0 0   (= 0)
+    x 1 1 1     (= 7)
+    -----------
+      1 1 0 1 1 (= 27)
+
+In this case, the xor of the third column doesn't generate any carry.
+
+Now we need to divide by the modulus. We can also use 11 as the
+modulus since it still satisfies the needed conditions to work on a
+Galois Field of characteristic 2 with 3 bits:
+
+      1 1 0 1 1 (= 27) | 1 0 1 1 (= 11)
+    x 1 0 1 1          ----------------
+    ---------            1 1 1 (= 7)
+      0 1 1 0 1
+    x   1 0 1 1
+    -----------
+        0 1 1 0 1
+    x     1 0 1 1
+    -------------
+          0 1 1 0 (= 6)
+
+Note that, in this case, to make zero the higher bit we need to
+consider the result of the xor operation, not the addition operation.
+
+So, 7 * 5 in a Galois Field of 3 bits with modulus 11 is 6.
+
+
+Optimization
+------------
+
+To compute all these operations in a fast way some methods have been
+traditionally used. Maybe the most common is the [lookup table][8].
+
+The problem with this method is that it requires 3 lookups for each
+byte multiplication, greatly amplifying the needed memory bandwidth
+and making it difficult to take advantage of any SIMD support on the
+processor.
+
+What EC does to improve the performance is based on the following
+property (using the 3 bits Galois Field of the last example):
+
+    A * B mod N = (A * b{2} * 4 mod N) x
+                  (A * b{1} * 2 mod N) x
+                  (A * b{0} mod N)
+
+This is basically a rewrite of the steps made in the previous example
+to multiply two numbers but moving the modulus calculation inside each
+intermediate result. What we can see here is that each term of the
+xor can be zeroed if the corresponding bit of B is 0, so we can ignore
+that factor. If the bit is 1, we need to compute A multiplied by a
+power of two and take the residue of the division by the modulus. We
+can precompute these values:
+
+    A0 = A (we don't need to compute the modulus here)
+    A1 = A0 * 2 mod N
+    A2 = A1 * 2 mod N
+
+Having these values we only need to add those corresponding to bits
+set to 1 in B. Using our previous example:
+
+    A = 1 1 1 (= 7)
+    B = 1 0 1 (= 5)
+
+    A0 = 1 1 1 (= 7)
+    A1 = 1 1 1 * 1 0 mod 1 0 1 1 = 1 0 1 (= 5)
+    A2 = 1 0 1 * 1 0 mod 1 0 1 1 = 0 0 1 (= 1)
+
+    Since only bits 0 and 2 are 1 in B, we add A0 and A2:
+
+    A0 + A2 = 1 1 1 x 0 0 1 = 1 1 0 (= 6)
+
+If we carefully look at what we are doing when computing each Ax, we
+see that we do two basic things:
+
+  - Shift the original value one bit to the left
+  - If the highest bit is 1, xor with the modulus
+
+Let's write this in a detailed way (representing each bit):
+
+    Original value:      a{2} a{1} a{0}
+    Shift 1 bit:    a{2} a{1} a{0}  0
+
+    If a{2} is 0 we already have the result:
+        a{1} a{0} 0
+
+    If a{2} is 1 we need to xor with the modulus:
+        1 a{1} a{0} 0 x 1 0 1 1 = a{1} (a{0} x 1) 1
+
+An important thing to see here is that if a{2} is 0, we can get the
+same result by xoring with all 0 instead of the modulus. For this
+reason we can rewrite the modulus as this:
+
+    Modulus: a{2} 0 a{2} a{2}
+
+This means that the modulus will be 0 0 0 0 is a{2} is 0, so the value
+won't change, and it will be 1 0 1 1 if a{2} is 1, giving the correct
+result. So, the computation is simply:
+
+    Original value:      a{2} a{1} a{0}
+    Shift 1 bit:    a{2} a{1} a{0}  0
+    Apply modulus:  a{1} (a{0} x a{2}) a{2}
+
+We can compute all Ax using this method. We'll get this:
+
+    A0 = a{2} a{1} a{0}
+    A1 = a{1} (a{0} x a{2}) a{2}
+    A2 = (a{0} x a{2}) (a{1} x a{2}) a{1}
+
+Once we have all terms, we xor the ones corresponding to the bits set
+to 1 in B. In out example this will be A0 and A2:
+
+   Result: (a{2} x a{0} x a{2}) (a{1} x a{1} x a{2}) (a{0} x a{1})
+
+We can easily see that we can remove some redundant factors:
+
+   Result: a{0} a{2} (a{0} x a{1})
+
+This way we have come up with a simply set of equations to compute the
+multiplication of any number by 5. If A is 1 1 1 (= 7), the result
+must be 1 1 0 (= 6) using the equations, as we expected. If we try
+another numbe for A, like 0 1 0 (= 2), the result must be 0 0 1 (= 1).
+
+This seems a really fast way to compute the multiplication without
+using any table lookup. The problem is that this is only valid for
+B = 5. For other values of B another set of equations will be found.
+To solve this problem we can pregenerate the equations for all
+possible values of B. Since the Galois Field we use is small, this is
+feasible.
+
+One thing to be aware of is that, in general, two equations for
+different bits of the same B can share common subexpressions. This
+gives space for further optimizations to reduce the total number of
+xors used in the final equations for a given B. However this is not
+easy to find, since finding the smallest number of xors that give the
+correct result is an NP-Problem. For EC an exhaustive search has been
+made to find the best combinations for each possible value.
+
+
+Implementation
+--------------
+
+All this seems great from the hardware point of view, but implementing
+this using normal processor instructions is not so easy because we
+would need a lot of shifts, ands, xors and ors to move the bits of
+each number to the correct position to compute the equation and then
+another shift to put each bit back to its final place.
+
+For example, to implement the functions to multiply by 5, we would
+need something like this:
+
+    Bit 2:  T2 = (A & 1) << 2
+    Bit 1:  T1 = (A & 4) >> 1
+    Bit 0:  T0 = ((A >> 1) x A) & 1
+    Result: T2 + T1 + T0
+
+This doesn't look good. So here we make a change in the way we get
+and process the data: instead of reading full numbers into variables
+and operate with them afterwards, we use a single independent variable
+for each bit of the number.
+
+Assume that we can read and write independent bits from memory (later
+we'll see how to solve this problem when this is not possible). In
+this case, the code would look something like this:
+
+    Bit 2:        T2 = Mem[2]
+    Bit 1:        T1 = Mem[1]
+    Bit 0:        T0 = Mem[0]
+    Computation:  T1 ^= T0
+    Store result: Mem[2] = T0
+                  Mem[1] = T2
+                  Mem[0] = T1
+
+Note that in this case we handle the final reordering of bits simply
+by storing the right variable to the right place, without any shifts,
+ands nor ors. In fact we only have memory loads, memory stores and
+xors. Note also that we can do all the computations directly using the
+variables themselves, without additional storage. This true for most
+of the values, but in some cases an additional one or two temporal
+variables will be needed to store intermediate results.
+
+The drawback of this approach is that additions, that are simply a
+xor of two numbers will need as many xors as bits are in each number.
+
+
+SIMD optimization
+-----------------
+
+So we have a good way to compute the multiplications, but even using
+this we'll need several operations for each byte of the original data.
+We can improve this by doing multiple multiplications using the same
+set of instructions.
+
+With the approach taken in the implementation section, we can see that
+in fact it's really easy to add SIMD support to this method. We only
+need to store in each variable one bit from multiple numbers. For
+example, when we load T2 from memory, instead of reading the bit 2 of
+the first number, we can read the bit 2 of the first, second, third,
+fourth, ... numbers. The same can be done when loading T1 and T0.
+
+Obviously this needs to have a special encoding of the numbers into
+memory to be able to do that in a single operation, but since we can
+choose whatever encoding we want for EC, we have chosen to have
+exactly that. We interpret the original data as a stream of bits, and
+we split it into subsequences of length L, each containing one bit of
+a number. Every S subsequences form a set of numbers of S bits that
+are encoded and decoded as a single group. This repeats for any
+remaining data.
+
+For example, in a simple case with L = 8 and S = 3, the original data
+would contain something like this (interpreted as a sequence of bits,
+offsets are also bit-based):
+
+    Offset  0: a{0} b{0} c{0} d{0} e{0} f{0} g{0} h{0}
+    Offset  8: a{1} b{1} c{1} d{1} e{1} f{1} g{1} h{1}
+    Offset 16: a{2} b{2} c{2} d{2} e{2} f{2} g{2} h{2}
+    Offset 24: i{0} j{0} k{0} l{0} m{0} n{0} o{0} p{0}
+    Offset 32: i{1} j{1} k{1} l{1} m{1} n{1} o{1} p{1}
+    Offset 40: i{2} j{2} k{2} l{2} m{2} n{2} o{2} p{2}
+
+Note: If the input file is not a multiple of S * L, 0-padding is done.
+
+Here we have 16 numbers encoded, from A to P. This way we can easily
+see that reading the first byte of the file will read all bits 0 of
+number A, B, C, D, E, F, G and H. The same happens with bits 1 and 2
+when we read the second and third bytes respectively. Using this
+encoding and the implementation described above, we can see that the
+same set of instructions will be computing the multiplication of 8
+numbers at the same time.
+
+This can be further improved if we use L = 64 with 64 bits variables
+on 64-bits processor. It's even faster if we use L = 128 using SSE
+registers or L = 256 using AVX registers on Intel processors.
+
+Currently EC uses L = 512 and S = 8. This means that numbers are
+packed in blocks of 512 bytes and gives space for even bigger
+processor registers up to 512 bits.
+
+
+Conclusions
+-----------
+
+This method requires a single variable/processor register for each
+bit. This can be challenging if we want to avoid additional memory
+accesses, even if we use modern processors that have many registers.
+However, the implementation we chose for the Vandermonde Matrix
+doesn't require temporary storage, so we don't need a full set of 8
+new registers (one for each bit) to store partial computations.
+Additionally, the computation of the multiplications requires, at
+most, 2 extra registers, but this is afordable.
+
+Xors are a really fast operation in modern processors. Intel CPU's
+can dispatch up to 3 xors per CPU cycle if there are no dependencies
+with ongoing previous instructions. Worst case is 1 xor per cycle. So,
+in some configurations, this method could be very near to the memory
+speed.
+
+Another interesting thing of this method is that all data it needs to
+operate is packed in small sequential blocks of memory, meaning that
+it can take advantage of the faster internal CPU caches.
+
+
+Results
+-------
+
+For the particular case of 8 bits, EC can compute each multiplication
+using 12.8 xors on average (without counting 0 and 1 that do not
+require any xor). Some numbers require less, like 2 that only requires
+3 xors.
+
+Having all this, we can check some numbers to see the performance of
+this method.
+
+Maybe the most interesting thing is the average number of xors needed
+to encode a single byte of data. To compute this we'll need to define
+some variables:
+
+  * K: Number of data fragments  
+  * R: Number of redundancy fragments  
+  * N: K + R  
+  * B: Number of bits per number  
+  * A: Average number of xors per number  
+  * Z: Bits per CPU register (can be up to 256 for AVX registers)  
+  * X: Average number of xors per CPU cycle  
+  * L: Average cycles per load  
+  * S: Average cycles per store  
+  * G: Core speed in Hz  
+
+_Total number of bytes processed for a single matrix multiplication_:
+
+  * __Read__:    K * B * Z / 8  
+  * __Written__: N * B * Z / 8  
+
+_Total number of memory accesses_:
+
+  * __Loads__:  K * B * N  
+  * __Stores__: B * N  
+
+>  We need to read the same K * B * Z bits, in registers of Z bits, N
+>  times, one for each row of the matrix. However the last N - 1 reads
+>  could be made from the internal CPU caches if conditions are good.
+
+_Total number of operations_:
+
+  * __Additions__:       (K - 1) * N  
+  * __Multiplications__: K * N  
+
+__Total number of xors__: B * (K - 1) * N + A * K * N =
+                          N * ((A + B) * K - B)
+
+__Xors per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z)
+
+__CPU cycles per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z * X) +
+                         8 * L * N / Z +      (loads)
+                         8 * S * N / (K * Z)  (stores)
+
+__Bytes per second__: G / {CPU cycles per byte}
+
+Some xors per byte numbers for specific configurations (B=8):
+
+              Z=64  Z=128  Z=256
+    K=2/R=1   0.79  0.39   0.20
+    K=4/R=2   1.76  0.88   0.44
+    K=4/R=3   2.06  1.03   0.51
+    K=8/R=3   3.40  1.70   0.85
+    K=8/R=4   3.71  1.86   0.93
+    K=16/R=4  6.34  3.17   1.59
+
+
+
+[1]: https://en.wikipedia.org/wiki/Erasure_code
+[2]: https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction
+[3]: https://en.wikipedia.org/wiki/Systematic_code
+[4]: https://en.wikipedia.org/wiki/Identity_matrix
+[5]: https://en.wikipedia.org/wiki/Linear_independence
+[6]: https://en.wikipedia.org/wiki/Vandermonde_matrix
+[7]: https://en.wikipedia.org/wiki/Finite_field
+[8]: https://en.wikipedia.org/wiki/Finite_field_arithmetic#Implementation_tricks
diff --git a/doc/developer-guide/fuse-interrupt.md b/doc/developer-guide/fuse-interrupt.md
new file mode 100644
index 00000000000..ec991b81ec5
--- /dev/null
+++ b/doc/developer-guide/fuse-interrupt.md
@@ -0,0 +1,211 @@
+# Fuse interrupt handling
+
+## Conventions followed
+
+- *FUSE* refers to the "wire protocol" between kernel and userspace and
+  related specifications.
+- *fuse* refers to the kernel subsystem and also to the GlusterFs translator.
+
+## FUSE interrupt handling spec
+
+The [Linux kernel FUSE documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fuse.txt?h=v4.18#n148)
+desrcibes how interrupt handling happens in fuse.
+
+## Interrupt handling in the fuse translator
+
+### Declarations
+
+This document describes the internal API in the fuse translator with which
+interrupt can be handled.
+
+The API being internal (to be used only in fuse-bridge.c; the functions are
+not exported to a header file).
+
+```
+enum fuse_interrupt_state {
+    /* ... */
+    INTERRUPT_SQUELCHED,
+    INTERRUPT_HANDLED,
+    /* ... */
+};
+typedef enum fuse_interrupt_state fuse_interrupt_state_t;
+struct fuse_interrupt_record;
+typedef struct fuse_interrupt_record fuse_interrupt_record_t;
+typedef void (*fuse_interrupt_handler_t)(xlator_t *this,
+                                         fuse_interrupt_record_t *);
+struct fuse_interrupt_record {
+    fuse_in_header_t fuse_in_header;
+    void *data;
+    /*
+       ...
+     */
+};
+
+fuse_interrupt_record_t *
+fuse_interrupt_record_new(fuse_in_header_t *finh,
+                          fuse_interrupt_handler_t handler);
+
+void
+fuse_interrupt_record_insert(xlator_t *this, fuse_interrupt_record_t *fir);
+
+gf_boolean_t
+fuse_interrupt_finish_fop(call_frame_t *frame, xlator_t *this,
+                          gf_boolean_t sync, void **datap);
+
+void
+fuse_interrupt_finish_interrupt(xlator_t *this, fuse_interrupt_record_t *fir,
+                                fuse_interrupt_state_t intstat,
+                                gf_boolean_t sync, void **datap);
+```
+
+The code demonstrates the usage of the API through `fuse_flush()`. (It's a
+dummy implementation only for demonstration purposes.) Flush is chosen
+because a `FLUSH` interrupt is easy to trigger (see
+*tests/features/interrupt.t*). Interrupt handling for flush is switched on
+by `--fuse-flush-handle-interrupt` (a hidden glusterfs command line flag).
+The implementation of flush interrupt is contained in the
+`fuse_flush_interrupt_handler()` function and blocks guarded by the
+
+```
+if (priv->flush_handle_interrupt) { ...
+```
+
+conditional (where `priv` is a `*fuse_private_t`).
+
+### Overview
+
+"Regular" fuse fops and interrupt handlers interact via a list containing
+interrupt records.
+
+If a fop wishes to have its interrupts handled, it needs to set up an
+interrupt record and insert it into the list; also when it's to finish
+(ie. in its "cbk" stage) it needs to delete the record from the list.
+
+If no interrupt happens, basically that's all to it - a list insertion
+and deletion.
+
+However, if an interrupt comes for the fop, the interrupt FUSE request
+will carry the data identifying an ongoing fop (that is, its `unique`),
+and based on that, the interrupt record will be looked up in the list, and
+the specific interrupt handler (a member of the interrupt record) will be
+called.
+
+Usually the fop needs to share some data with the interrupt handler to
+enable it to perform its task (also shared via the interrupt record).
+The interrupt API offers two approaches to manage shared data:
+- _Async or reference-counting strategy_: from the point on when the interrupt
+  record is inserted to the list, it's owned jointly by the regular fop and
+  the prospective interrupt handler. Both of them need to check before they
+  return if the other is still holding a reference; if not, then they are
+  responsible for reclaiming the shared data.
+- _Sync or borrow strategy_: the interrupt handler is considered a borrower
+  of the shared data. The interrupt handler should not reclaim the shared
+  data. The fop will wait for the interrupt handler to finish (ie., the borrow
+  to be returned), then it has to reclaim the shared data.
+
+The user of the interrupt API need to call the following functions to
+instrument this control flow:
+- `fuse_interrupt_record_insert()` in the fop to insert the interrupt record to
+   the list;
+- `fuse_interrupt_finish_fop()`in the fop (cbk) and
+- `fuse_interrupt_finish_interrupt()`in the interrupt handler
+
+to perform needed synchronization at the end their tenure. The data management
+strategies are implemented by the `fuse_interrupt_finish_*()` functions (which
+have an argument to specify which strategy to use); these routines take care
+of freeing the interrupt record itself, while the reclamation of the shared data
+is left to the API user.
+
+### Usage
+
+A given FUSE fop can be enabled to handle interrupts via the following
+steps:
+
+- Define a handler function (of type `fuse_interrupt_handler_t`).
+  It should implement the interrupt handling logic and in the end
+  call (directly or as async callback) `fuse_interrupt_finish_interrupt()`.
+  The `intstat` argument to `fuse_interrupt_finish_interrupt` should be
+  either `INTERRUPT_SQUELCHED` or `INTERRUPT_HANDLED`.
+    - `INTERRUPT_SQUELCHED` means that the interrupt could not be delivered
+      and the fop is going on uninterrupted.
+    - `INTERRUPT_HANDLED` means that the interrupt was actually handled. In
+      this case the fop will be answered from interrupt context with errno
+      `EINTR` (that is, the fop should not send a response to the kernel).
+
+  (the enum `fuse_interrupt_state` includes further members, which are reserved
+  for internal use).
+
+  We return to the `sync` and `datap` arguments later.
+- In the `fuse_<FOP>` function create an interrupt record using
+  `fuse_interrupt_record_new()`, passing the incoming `fuse_in_header` and
+  the above handler function to it.
+    - Arbitrary further data can be referred to via the `data` member of the
+      interrupt record that is to be passed on from fop context to
+      interrupt context.
+- When it's set up, pass the interrupt record to
+  `fuse_interrupt_record_insert()`.
+- In `fuse_<FOP>_cbk` call `fuse_interrupt_finish_fop()`.
+    - `fuse_interrupt_finish_fop()` returns a Boolean according to whether the
+      interrupt was handled. If it was, then the FUSE request is already
+      answered and the stack gets destroyed in `fuse_interrupt_finish_fop` so
+      `fuse_<FOP>_cbk()` can just return (zero). Otherwise follow the standard
+      cbk logic (answer the FUSE request and destroy the stack -- these are
+      typically accomplished by `fuse_err_cbk()`).
+- The last two argument of `fuse_interrupt_finish_fop()` and
+  `fuse_interrupt_finish_interrupt()` are `gf_boolean_t sync` and
+  `void **datap`.
+    - `sync` represents the strategy for freeing the interrupt record. The
+      interrupt handler and the fop handler are in race to get at the interrupt
+      record first (interrupt handler for purposes of doing the interrupt
+      handling, fop handler for purposes of deactivating the interrupt record
+      upon completion of the fop handling).
+        - If `sync` is true, then the fop handler will wait for the interrupt
+          handler to finish and it takes care of freeing.
+        - If `sync` is false, the loser of the above race will perform freeing.
+
+      Freeing is done within the respective interrupt finish routines, except
+      for the `data` field of the interrupt record; with respect to that, see
+      the discussion of the `datap` parameter below. The strategy has to be
+      consensual, that is, `fuse_interrupt_finish_fop()` and
+      `fuse_interrupt_finish_interrupt()` must pass the same value for `sync`.
+      If dismantling the resources associated with the interrupt record is
+      simple, `sync = _gf_false` is the suggested choice; `sync = _gf_true` can
+      be useful in the opposite case, when dismantling those resources would
+      be inconvenient to implement in two places or to enact in non-fop context.
+    - If `datap` is `NULL`, the `data` member of the interrupt record will be
+      freed within the interrupt finish routine.  If it points to a valid
+      `void *` pointer, and if caller is doing the cleanup (see `sync` above),
+      then that pointer will be directed to the `data` member of the interrupt
+      record and it's up to the caller what it's doing with it.
+        - If `sync` is true, interrupt handler can use `datap = NULL`, and
+          fop handler will have `datap` point to a valid pointer.
+        - If `sync` is false, and handlers pass a pointer to a pointer for
+          `datap`, they should check if the pointed pointer is NULL before
+          attempting to deal with the data.
+
+### FUSE answer for the interrupted fop
+
+The kernel acknowledges a successful interruption for a given FUSE request
+if the filesystem daemon answers it with errno EINTR; upon that, the syscall
+which induced the request will be abruptly terminated with an interrupt, rather
+than returning a value.
+
+In glusterfs, this can be arranged in two ways.
+
+- If the interrupt handler wins the race for the interrupt record, ie.
+  `fuse_interrupt_finish_fop()` returns true to `fuse_<FOP>_cbk()`, then, as
+  said above, `fuse_<FOP>_cbk()` does not need to answer the FUSE request.
+  That's because then the interrupt handler will take care about answering
+  it (with errno EINTR).
+- If `fuse_interrupt_finish_fop()` returns false to `fuse_<FOP>_cbk()`, then
+  this return value does not inform the fop handler whether there was an interrupt
+  or not. This return value occurs both when fop handler won the race for the
+  interrupt record against the interrupt handler, and when there was no interrupt
+  at all.
+
+  However, the internal logic of the fop handler might detect from other
+  circumstances that an interrupt was delivered. For example, the fop handler
+  might be sleeping, waiting for some data to arrive, so that a premature
+  wakeup (with no data present) occurs if the interrupt handler intervenes. In
+  such cases it's the responsibility of the fop handler to reply the FUSE
+  request with errro EINTR.
diff --git a/doc/developer-guide/gfapi-symbol-versions.md b/doc/developer-guide/gfapi-symbol-versions.md
new file mode 100644
index 00000000000..e4f4fe9f052
--- /dev/null
+++ b/doc/developer-guide/gfapi-symbol-versions.md
@@ -0,0 +1,270 @@
+
+## Symbol Versions and SO_NAMEs
+
+  In general, adding new APIs to a shared library does not require that
+symbol versions be used or the the SO_NAME be "bumped." These actions
+are usually reserved for when a major change is introduced, e.g. many
+APIs change or a signficant change in the functionality occurs.
+
+  Over the normal lifetime of a When a new API is added, the library is
+recompiled, consumers of the new API are able to do so, and existing,
+legacy consumers of the original API continue as before. If by some
+chance an old copy of the library is installed on a system, it's unlikely
+that most applications will be affected. New applications that use the
+new API will incur a run-time error terminate.
+
+  Bumping the SO_NAME, i.e. changing the shared lib's file name, e.g.
+from libfoo.so.0 to libfoo.so.1, which also changes the ELF SO_NAME
+attribute inside the file, works a little differently. libfoo.so.0
+contains only the old APIs. libfoo.so.1 contains both the old and new
+APIs. Legacy software that was linked with libfoo.so.0 continues to work
+as libfoo.so.0 is usually left installed on the system. New software that
+uses the new APIs is linked with libfoo.so.1, and works as long as
+long as libfoo.so.1 is installed on the system. Accidentally (re)installing
+libfoo.so.0 doesn't break new software as long as reinstalling doesn't
+erase libfoo.so.1.
+
+  Using symbol versions is somewhere in the middle. The shared library
+file remains libfoo.so.0 forever. Legacy APIs may or may not have an
+associated symbol version. New APIs may or may not have an associated
+symbol version either. In general symbol versions are reserved for APIs
+that have changed. Either the function's signature has changed, i.e. the
+return time or the number of paramaters, and/or the parameter types have
+changed. Another reason for using symbol versions on an API is when the
+behaviour or functionality of the API changes dramatically. As with a
+library that doesn't use versioned symbols, old and new applications
+either find or don't find the versioned symbols they need. If the versioned
+symbol doesn't exist in the installed library, the application incurs a
+run-time error and terminates.
+
+  GlusterFS wanted to keep tight control over the APIs in libgfapi.
+Originally bumping the SO_NAME was considered, and GlusterFS-3.6.0 was
+released with libgfapi.so.7. Not only was "7" a mistake (it should have
+been "6"), but it was quickly pointed out that many dependent packages
+that use libgfapi would be forced to be recompiled/relinked. Thus no
+packages of 3.6.0 were ever released and 3.6.1 was quickly released with
+libgfapi.so.0, but with symbol versions. There's no strong technical
+reason for either; the APIs have not changed, only new APIs have been
+added. It's merely being done in anticipation that some APIs might change
+sometime in the future.
+
+  Enough about that now, let's get into the nitty gritty——
+
+## Adding new APIs
+
+### Adding a public API.
+
+  This is the default, and the easiest thing to do. Public APIs have
+declarations in either glfs.h, glfs-handles.h, or, at your discretion,
+in a new header file intended for consumption by other developers.
+
+Here's what you need to do to add a new public API:
+
++ Write the declaration, e.g. in glfs.h:
+
+```C
+    int glfs_dtrt (const char *volname, void *stuff) __THROW
+```
+
++ Write the definition, e.g. in glfs-dtrt.c:
+
+```C
+   int
+   pub_glfs_dtrt (const char *volname, void *stuff)
+   {
+       ...
+       return 0;
+   }
+```
+
++ Add the symbol version magic for ELF, gnu toolchain to the definition.
+
+  following the definition of your new function in glfs-dtrtops.c, add a
+  line like this:
+
+```C
+  GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dtrt, 3.7.0)
+```
+
+  The whole thing should look like:
+
+```C
+      int
+      pub_glfs_dtrt (const char *volname, void *stuff)
+      {
+           ...
+      }
+      GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dtrt, 3.7.0);
+```
+
+  In this example, 3.7.0 refers to the Version the symbol will first
+  appear in. There's nothing magic about it, it's just a string token.
+  The current versions we have are 3.4.0, 3.4.2, 3.5.0, 3.5.1, and 3.6.0.
+  They are to be considered locked or closed. You can not, must not add
+  any new APIs and use these versions. Most new APIs will use 3.7.0. If
+  you add a new API appearing in 3.6.2 (and mainline) then you would use
+  3.6.2.
+
++ Add the symbol version magic for OS X to the declaration.
+
+  following the declaration in glfs.h, add a line like this:
+
+```C
+  GFAPI_PUBLIC(glfs_dtrt, 3.7.0)
+```
+
+  The whole thing should look like:
+
+```C
+     int glfs_dtrt (const char *volname, void *stuff) __THROW
+              GFAPI_PUBLIC(glfs_dtrt, 3.7.0);
+```
+
+  The version here must match the version associated with the definition.
+
++ Add the new API to the ELF, gnu toolchain link map file, gfapi.map
+
+  Most new public APIs will probably be added to a new section that
+  looks like this:
+
+```
+  GFAPI_3.7.0 {
+          global:
+                  glfs_dtrt;
+  } GFAPI_PRIVATE_3.7.0;
+```
+
+  if you're adding your new API to, e.g. 3.6.2, it'll look like this:
+
+```
+  GFAPI_3.6.2 {
+          global:
+                  glfs_dtrt;
+  } GFAPI_3.6.0;
+```
+
+  and you must change the
+```
+    GFAPI_PRIVATE_3.7.0 { ...} GFAPI_3.6.0;
+```
+  section to:
+```
+    GFAPI_PRIVATE_3.7.0 { ...} GFAPI_3.6.2;
+```
+
++ Add the new API to the OS X alias list file, gfapi.aliases.
+
+  Most new APIs will use a line that looks like this:
+
+```C
+ _pub_glfs_dtrt _glfs_dtrt$GFAPI_3.7.0
+```
+
+  if you're adding your new API to, e.g. 3.6.2, it'll look like this:
+
+```C
+ _pub_glfs_dtrt _glfs_dtrt$GFAPI_3.6.2
+```
+
+And that's it.
+
+
+### Adding a private API.
+
+  If you're thinking about adding a private API that isn't declared in
+one of the header files, then you should seriously rethink what you're
+doing and figure out how to put it in libglusterfs instead.
+
+If that hasn't convinced you, follow the instructions above, but use the
+_PRIVATE versions of macros, symbol versions, and aliases. If you're 1337
+enough to ignore this advice, then you're 1337 enough to figure out how
+to do it.
+
+
+## Changing an API.
+
+### Changing a public API.
+
+  There are two ways an API might change, 1) its signature has changed, or
+2) its new functionality or behavior is substantially different than the
+old. An APIs signature consists of the function return type, and the number
+and/or type of its parameters. E.g. the original API:
+
+```C
+   int glfs_dtrt (const char *volname, void *stuff);
+```
+
+and the changed API:
+
+```C
+   void *glfs_dtrt (const char *volname, glfs_t *ctx, void *stuff);
+```
+
+  One way to avoid a change like this, and which is preferable in many
+ways, is to leave the legacy glfs_dtrt() function alone, document it as
+deprecated, and simply add a new API, e.g. glfs_dtrt2(). Practically
+speaking, that's effectively what we'll be doing anyway, the difference
+is only that we'll use a versioned symbol to do it.
+
+  On the assumption that adding a new API is undesirable for some reason,
+perhaps the use of glfs_gnu() is just so pervasive that we really don't
+want to add glfs_gnu2().
+
++ change the declaration in glfs.h:
+
+```C
+  glfs_t *glfs_gnu (const char *volname, void *stuff) __THROW
+          GFAPI_PUBLIC(glfs_gnu, 3.7.0);
+````
+
+Note that there is only the single, new declaration.
+
++ change the old definition of glfs_gnu() in glfs.c:
+
+```C
+    struct glfs *
+    pub_glfs_gnu340 (const char * volname)
+    {
+        ...
+    }
+    GFAPI_SYMVER_PUBLIC(glfs_gnu340, glfs_gnu, 3.4.0);
+```
+
++ create the new definition of glfs_gnu in glfs.c:
+
+```C
+    struct glfs *
+    pub_glfs_gnu (const char * volname, void *stuff)
+    {
+        ...
+    }
+    GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_gnu, 3.7.0);
+```
+
++ Add the new API to the ELF, gnu toolchain link map file, gfapi.map
+
+```
+  GFAPI_3.7.0 {
+          global:
+                  glfs_gnu;
+  } GFAPI_PRIVATE_3.7.0;
+```
+
++ Update the OS X alias list file, gfapi.aliases, for both versions:
+
+Change the old line:
+```C
+  _pub_glfs_gnu _glfs_gnu$GFAPI_3.4.0
+```
+to:
+```C
+  _pub_glfs_gnu340 _glfs_gnu$GFAPI_3.4.0
+```
+
+Add a new line:
+```C
+  _pub_glfs_gnu _glfs_gnu$GFAPI_3.7.0
+```
+
++ Lastly, change all gfapi internal calls glfs_gnu to the new API.
+
diff --git a/doc/developer-guide/identifying-resource-leaks.md b/doc/developer-guide/identifying-resource-leaks.md
new file mode 100644
index 00000000000..950cae79b0a
--- /dev/null
+++ b/doc/developer-guide/identifying-resource-leaks.md
@@ -0,0 +1,200 @@
+# Identifying Resource Leaks
+
+Like most other pieces of software, GlusterFS is not perfect in how it manages
+its resources like memory, threads and the like. Gluster developers try hard to
+prevent leaking resources but releasing and unallocating the used structures.
+Unfortunately every now and then some resource leaks are unintentionally added.
+
+This document tries to explain a few helpful tricks to identify resource leaks
+so that they can be addressed.
+
+
+## Debug Builds
+
+There are certain techniques used in GlusterFS that make it difficult to use
+tools like Valgrind for memory leak detection. There are some build options
+that make it more practical to use Valgrind and other tools. When running
+Valgrind, it is important to have GlusterFS builds that contain the
+debuginfo/symbols. Some distributions (try to) strip the debuginfo to get
+smaller executables. Fedora and RHEL based distributions have sub-packages
+called ...-debuginfo that need to be installed for symbol resolving.
+
+
+### Memory Pools
+
+By using memory pools, there are no allocation/freeing of single structures
+needed. This improves performance, but also makes it impossible to track the
+allocation and freeing of srtuctures.
+
+It is possible to disable the use of memory pools, and use standard `malloc()`
+and `free()` functions provided by the C library. Valgrind is then able to
+track the allocated areas and verify if they have been free'd. In order to
+disable memory pools, the Gluster sources needs to be configured with the
+`--enable-debug` option:
+
+```shell
+./configure --enable-debug
+```
+
+When building RPMs, the `.spec` handles the `--with=debug` option too:
+
+```shell
+make dist
+rpmbuild -ta --with=debug glusterfs-....tar.gz
+```
+
+### Dynamically Loaded xlators
+
+Valgrind tracks the call chain of functions that do memory allocations. The
+addresses of the functions are stored and before Valgrind exits the addresses
+are resolved into human readable function names and offsets (line numbers in
+source files). Because Gluster loads xlators dynamically, and unloads then
+before exiting, Valgrind is not able to resolve the function addresses into
+symbols anymore. Whenever this happend, Valgrind shows `???` in the output,
+like
+
+```
+  ==25170== 344 bytes in 1 blocks are definitely lost in loss record 233 of 324
+  ==25170==    at 0x4C29975: calloc (vg_replace_malloc.c:711)
+  ==25170==    by 0x52C7C0B: __gf_calloc (mem-pool.c:117)
+  ==25170==    by 0x12B0638A: ???
+  ==25170==    by 0x528FCE6: __xlator_init (xlator.c:472)
+  ==25170==    by 0x528FE16: xlator_init (xlator.c:498)
+  ...
+```
+
+These `???` can be prevented by not calling `dlclose()` for unloading the
+xlator. This will cause a small leak of the handle that was returned with
+`dlopen()`, but for improved debugging this can be acceptible. For this and
+other Valgrind features, a `--enable-valgrind` option is available to
+`./configure`. When GlusterFS is built with this option, Valgrind will be able
+to resolve the symbol names of the functions that do memory allocations inside
+xlators.
+
+```shell
+./configure --enable-valgrind
+```
+
+When building RPMs, the `.spec` handles the `--with=valgrind` option too:
+
+```shell
+make dist
+rpmbuild -ta --with=valgrind glusterfs-....tar.gz
+```
+
+## Running Valgrind against a single xlator
+
+Debugging a single xlator is not trivial. But there are some tools to make it
+easier. The `sink` xlator does not do any memory allocations itself, but
+contains just enough functionality to mount a volume with only the `sink`
+xlator. There is a little gfapi application under `tests/basic/gfapi/` in the
+GlusterFS sources that can be used to run only gfapi and the core GlusterFS
+infrastructure with the `sink` xlator. By extending the `.vol` file to load
+more xlators, each xlator can be debugged pretty much separately (as long as
+the xlators have no dependencies on each other). A basic Valgrind run with the
+suitable configure options looks like this:
+
+```shell
+./autogen.sh
+./configure --enable-debug --enable-valgrind
+make && make install
+cd tests/basic/gfapi/
+make gfapi-load-volfile
+valgrind ./gfapi-load-volfile sink.vol
+```
+
+Combined with other very useful options to Valgrind, the following execution
+shows many more useful details:
+
+```shell
+valgrind \
+        --fullpath-after= --leak-check=full --show-leak-kinds=all \
+        ./gfapi-load-volfile sink.vol
+```
+
+Note that the `--fullpath-after=` option is left empty, this makes Valgrind
+print the full path and filename that contains the functions:
+
+```
+==2450== 80 bytes in 1 blocks are definitely lost in loss record 8 of 60
+==2450==    at 0x4C29975: calloc (/builddir/build/BUILD/valgrind-3.11.0/coregrind/m_replacemalloc/vg_replace_malloc.c:711)
+==2450==    by 0x52C6F73: __gf_calloc (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/mem-pool.c:117)
+==2450==    by 0x12F10CDA: init (/usr/src/debug/glusterfs-3.11dev/xlators/meta/src/meta.c:231)
+==2450==    by 0x528EFD5: __xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:472)
+==2450==    by 0x528F105: xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:498)
+==2450==    by 0x52D9D8B: glusterfs_graph_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/graph.c:321)
+...
+```
+
+In the above example, the `init` function in `xlators/meta/src/meta.c` does a
+memory allocation on line 231. This memory is never free'd again, and hence
+Valgrind logs this call stack. When looking in the code, it seems that the
+allocation of `priv` is assigned to the `this->private` member of the
+`xlator_t` structure. Because the allocation is done in `init()`, free'ing is
+expected to happen in `fini()`. Both functions are shown below, with the
+inclusion of the empty `fini()`:
+
+
+```
+226 int
+227 init (xlator_t *this)
+228 {
+229         meta_priv_t *priv = NULL;
+230 
+231         priv = GF_CALLOC (sizeof(*priv), 1, gf_meta_mt_priv_t);
+232         if (!priv)
+233                 return -1;
+234 
+235         GF_OPTION_INIT ("meta-dir-name", priv->meta_dir_name, str, out);
+236 
+237         this->private = priv;
+238 out:
+239         return 0;
+240 }
+241 
+242 
+243 int
+244 fini (xlator_t *this)
+245 {
+246         return 0;
+247 }
+```
+
+In this case, the resource leak can be addressed by adding a single line to the
+`fini()` function:
+
+```
+243 int
+244 fini (xlator_t *this)
+245 {
+246         GF_FREE (this->private);
+247         return 0;
+248 }
+```
+
+Running the same Valgrind command and comparing the output will show that the
+memory leak in `xlators/meta/src/meta.c:init` is not reported anymore.
+
+### Running DRD, the Valgrind thread error detector
+
+When configuring GlusterFS with:
+
+```shell
+./configure --enable-valgrind
+```
+
+the default Valgrind tool (Memcheck) is enabled. But it's also possble to select
+one of Memcheck or DRD by using:
+
+```shell
+./configure --enable-valgrind=memcheck
+```
+
+or:
+
+```shell
+./configure --enable-valgrind=drd
+```
+
+respectively. When using DRD, it's recommended to consult
+https://valgrind.org/docs/manual/drd-manual.html before running.
diff --git a/doc/developer-guide/logging-guidelines.md b/doc/developer-guide/logging-guidelines.md
new file mode 100644
index 00000000000..0e6b2588535
--- /dev/null
+++ b/doc/developer-guide/logging-guidelines.md
@@ -0,0 +1,132 @@
+Guidelines on using the logging framework within a component
+============================================================
+Gluster library libglusterfs.so provides message logging abstractions that
+are intended to be used across all code/components within gluster.
+
+There could be potentially 2 major cases how the logging infrastructure is
+used,
+  - A new gluster service daemon or end point is created
+    - The service daemon infrastructure itself initlializes the logging
+    infrastructure (i.e calling gf_log_init and related set functions)
+      - See, glusterfsd.c:logging_init
+    - Alternatively there could be a case where an end point service (say
+    gfapi) may need to do the required initialization
+    - This document does not (yet?) cover guidelines for these cases. Best
+    bet would be to look at code in glusterfsd.c:logging_init (or equivalent)
+    in case a need arises and you reach this document.
+  - A new xlator or subcomponent is written as a part of the stack
+    - Primarily in this case, the consumer of the logging APIs would only
+    invoke an API to *log* a particular message at a certain severity
+    - This document elaborates on this use of the message logging framework
+    in this context
+
+There are 2 interfaces provided to log messages,
+1. The older gf_log* interface
+2. The newer gf_msg* interface
+
+1. Older _to_be_deprecated_ gf_log* interface
+  - Not much documentation is provided for this interface here, as these are
+  meant to be deprecated and all code should be moved to the newer gf_msg*
+  interface
+
+2. New gf_msg* interface
+  - The set of interfaces provided by gf_msg are,
+    - NOTE: It is best to consult logging.h for the latest interfaces, as
+    this document may get out of sync with the code
+
+    *gf_msg(dom, levl, errnum, msgid, fmt...)*
+      - Used predominantly to log a message above TRACE and DEBUG levels
+      - See further guidelines below
+
+    *gf_msg_debug(dom, errnum, fmt...)*
+    *gf_msg_trace(dom, errnum, fmt...)*
+      - Above are handy shortcuts for TRACE and DEBUG level messages
+      - See further guidelines below
+
+    *gf_msg_callingfn(dom, levl, errnum, msgid, fmt...)*
+      - Useful function when a backtrace to detect calling routines that
+      reached this execution point needs to be logged in addition to the
+      message
+      - This is very handy for framework libraries or code, as there could
+      be many callers to the same, and the real failure would need the call
+      stack to determine who the caller actually was
+      - A good example is the dict interfaces using this function to log who
+      called, when a particular error/warning needs to be displayed
+      - See further guidelines below
+
+    *gf_msg_plain(levl, fmt...)*
+    *gf_msg_plain_nomem(levl, msg)*
+    *gf_msg_vplain(levl, fmt, va)*
+    *gf_msg_backtrace_nomem*
+      - The above interfaces are provided to log messages without any typical
+      headers (like the time stamp, dom, errnum etc.). The primary users of
+      the above interfaces are, when printing the final graph, or printing
+      the configuration when a process is about dump core or abort, or
+      printing the backtrace when a process receives a critical signal
+      - These interfaces should not be used outside the scope of the users
+      above, unless you know what you are doing
+
+    *gf_msg_nomem(dom, levl, size)*
+      - Very crisp messages, throwing out file, function, line numbers, in
+      addition to the passed in size
+      - These are used in the memory allocation abstractions interfaces in
+      gluster, when the allocation fails (hence a no-mem log message, so that
+      further allocation is not attempted by the logging infrastructure)
+      - If you are contemplating using these, then you know why and how
+
+  - Guidelines for the various arguments to the interfaces
+    **dom**
+      - The domain from which this message appears, IOW for xlators it should
+      be the name of the xlator (as in this->name)
+
+      - The intention of this string is to distinguish from which
+      component/xlator the message is being printed
+
+      - This information can also be gleaned from the FILE:LINE:FUNCTION
+      information printed in the message anyway, but is retained to provide
+      backward compatability to the messages as seen by admins earlier
+
+    **levl**
+      - Consult logging.h:gf_loglevel_t for logging levels (they pretty much
+      map to syslog levels)
+
+    **errnum**
+      - errno should be passed in here, if available, 0 otherwise. This auto
+      enables the logging infrastructure to print (man 3) strerror form of the
+      same at the end of the message in a consistent format
+
+      - If any message is already adding the strerror as a parameter to the
+      message, it is suggested/encouraged to remove the same and pass it as
+      errnum
+
+      - The intention is that, if all messages did this using errnum and not
+      as a part of their own argument list, the output would look consistent
+      for the admin and cross component developers when reading logs
+
+    **msgid**
+      - This is a unique message ID per message (which now is more a message
+      ID per group of messages in the implementation)
+
+      - Rules for generating this ID can be found in dht-messages.h (it used
+      to be template-common-messages.h, but that template is not updated,
+      this comment should be changed once that is done) and glfs-message-id.h
+
+      - Every message that is *above* TRACE and DEBUG should get a message
+      ID, as this helps generating message catalogs that can help admins to
+      understand the context of messages better. Another intention is that
+      automated message parsers could detect a class of message IDs and send
+      out notifications on the same, rather than parse the message string
+      itself (which if it changes, the input to the parser has to change, etc.
+      and hence is better to retain message IDs)
+
+      - Ok so if intention is not yet clear, look at journald MESSAGE ID
+      motivation, as that coupled with ident/dom above is expected to provide
+      us with similar advantages
+
+      - Bottomline: Every message gets its own ID, in case a message is
+      *not* DEBUG or TRACE and still is developer centric, a generic message
+      ID per component *maybe* assigned to the same to provide ease of use
+      of the API
+
+    **fmt**
+      - As in format argument of (man 3) printf
diff --git a/doc/developer-guide/network_compression.md b/doc/developer-guide/network_compression.md
new file mode 100644
index 00000000000..1222a765276
--- /dev/null
+++ b/doc/developer-guide/network_compression.md
@@ -0,0 +1,71 @@
+# On-Wire Compression + Decompression
+
+The 'compression translator' compresses and decompresses data in-flight
+between client and bricks.
+
+### Working
+When a writev call occurs, the client compresses the data before sending it to
+brick. On the brick, compressed data is decompressed. Similarly, when a readv
+call occurs, the brick compresses the data before sending it to client. On the
+client, the compressed data is decompressed. Thus, the amount of data sent over
+the wire is minimized. Compression/Decompression is done using Zlib library.
+
+During normal operation, this is the format of data sent over wire:
+
+~~~
+<compressed-data> + trailer(8 bytes)
+~~~
+
+The trailer contains the CRC32 checksum and length of original uncompressed
+data. This is used for validation.
+
+### Usage
+
+Turning on compression xlator:
+
+~~~
+gluster volume set <vol_name> network.compression on
+~~~
+
+### Configurable parameters (optional)
+
+**Compression level**
+~~~
+gluster volume set <vol_name> network.compression.compression-level 8
+~~~
+
+~~~
+ 0  : no compression
+ 1  : best speed
+ 9  : best compression
+-1  : default compression
+~~~
+
+**Minimum file size**
+
+~~~
+gluster volume set <vol_name> network.compression.min-size 50
+~~~
+
+Data is compressed only when its size exceeds the above value in bytes.
+
+**Other paramaters**
+
+Other less frequently used parameters include `network.compression.mem-level`
+and `network.compression.window-size`. More details can about these options
+can be found by running `gluster volume set help` command.
+
+### Known Issues and Limitations
+
+* Compression translator cannot work with striped volumes.
+* Mount point hangs when writing a file with write-behind xlator turned on. To
+overcome this, turn off `performance.write-behind` entirely OR
+set`performance.strict-write-ordering` to on.
+* For glusterfs versions <= 3.5, compression traslator can ONLY work with pure
+distribute volumes. This limitation is caused by AFR not being able to
+propagate xdata. This issue has been fixed in glusterfs versions > 3.5
+
+### TODO
+Although zlib offers high compression ratio, it is very slow. We can make the
+translator pluggable to add support for other compression methods such as
+[lz4 compression](https://code.google.com/p/lz4/)
diff --git a/doc/developer-guide/options-to-contribute.md b/doc/developer-guide/options-to-contribute.md
new file mode 100644
index 00000000000..3f0d84e7645
--- /dev/null
+++ b/doc/developer-guide/options-to-contribute.md
@@ -0,0 +1,212 @@
+# A guide for contributors
+
+While you have gone through 'how to contribute' guides, if you are
+not sure what to work on, but really want to help the project, you
+have now landed on the right document :-)
+
+### Basic
+
+Instead of planning to fix **all** the below issues in one patch,
+we recommend you to have a a constant, continuous flow of improvements
+for the project. We recommend you to pick 1 file (or just few files) at
+a time to address below issues.
+Pick any `.c` (or `.h`) file, and you can send a patch which fixes **any**
+of the below themes. Ideally, fix all such occurrences in the file, even
+though, the reviewers would review even a single line change patch
+from you.
+
+1. Check for variable definitions, and if there is an array definition,
+which is very large at the top of the function, see if you can re-scope
+the variable to relevant sections (if it helps).
+
+Most of the time, some of these arrays may be used for 'error' handling,
+and it is possible to use them only in that scope.
+
+Reference: https://review.gluster.org/20846/
+
+
+2. Check for complete string initialization at the beginning of a function.
+Ideally, there is no reason to initialize a string. Fix it across the file.
+
+Example:
+
+`char new_path_name[PATH_MAX] = {0};` to `char new_path_name[PATH_MAX];`
+
+
+3. Change `calloc()` to `malloc()` wherever it makes sense.
+
+In a case of allocating a structures, where you expect certain (or most of)
+variables to be 0 (or NULL), it makes sense to use calloc(). But otherwise,
+there is an extra cost to `memset()` the whole object after allocating it.
+While it is not a significant improvement in performance, code which gets
+hit 1000s of times in a second, it would add some value.
+
+Reference: https://review.gluster.org/20878/
+
+
+4. You can consider using `snprintf()`, instead of `strncpy()` while dealing
+with strings.
+
+strncpy() won't null terminate if the dest buffer isn't big enough; snprintf()
+does. While most of the string operations in the code is on array, and larger
+size than required, strncpy() does an extra copy of 0s at the end of
+string till the size of the array. It makes sense to use `snprintf()`,
+which doesn't suffer from that behavior.
+
+Also check the return value from snprintf() for buffer overflow and handle
+accordingly
+
+Reference: https://review.gluster.org/20925/
+
+
+5. Now, pick a `.h` file, and see if a structure is very large, and see
+if re-aligning them as per [coding-standard](./coding-standard.md) gives any size benefit,
+if yes, go ahead and change it. Make sure you check all the structures
+in the file for similar pattern.
+
+Reference: [Check this section](https://github.com/gluster/glusterfs/blob/master/doc/developer-guide/coding-standard.md#structure-members-should-be-aligned-based-on-the-padding-requirements
+
+
+### If you are up for more :-)
+
+Good progress! Glad you are interested to know more. We are surely interested
+in next level of contributions from you!
+
+#### Coverity
+
+Visit [Coverity Dashboard](https://scan.coverity.com/projects/gluster-glusterfs?tab=overview).
+
+Now, if the number of defect is not 0, you have an opportunity to contribute.
+
+You get all the detail on why the particular defect is mentioned there, and
+most probable hint on how to fix it. Do it!
+
+Reference: https://review.gluster.org/21394/
+
+Use the same reference Id (789278) as the patch, so we can capture it is in
+single bugzilla.
+
+#### Clang-Scan
+
+Clang-Scan is a tool which scans the .c files and reports the possible issues,
+similar to coverity, but a different tool. Over the years we have seen, they
+both report very different set of issues, and hence there is a value in fixing it.
+
+GlusterFS project gets tested with clang-scan job every night, and the report is
+posted in the [job details page](https://build.gluster.org/job/clang-scan/lastCompletedBuild/clangScanBuildBugs/).
+As long as the number is not 0 in the report here, you have an opportunity to
+contribute! Similar to coverity dashboard, click on 'Details' to find out the
+reason behind that report, and send a patch.
+
+Reference: https://review.gluster.org/21025
+
+Again, you can use reference Id (1622665) for these patches!
+
+
+### I am good with programming, I would like to do more than above!
+
+#### Locked regions / Critical sections
+
+In the file you open, see if the lock is taken only to increment or decrement
+a flag, counter etc. If yes, then recommend you to convert it to ATOMIC locks.
+It is simple activity, but, if you know programing, you would know the benefit
+here.
+
+NOTE: There may not always a possibility to do this! You may have to check
+with developers first before going ahead.
+
+Reference: https://review.gluster.org/21221/
+
+
+#### ASan (address sanitizer)
+
+[The job](https://build.gluster.org/job/asan/) runs regression with asan builds,
+and you can also run glusterfs with asan on your workload to identify the leaks.
+If there are any leaks reported, feel free to check it, and send us patch.
+
+You can also run `valgrind` and let us know what it reports.
+
+Reference: https://review.gluster.org/21397
+
+
+#### Porting to different architecture
+
+This is something which we are not focusing right now, happy to collaborate!
+
+Reference: https://review.gluster.org/21276
+
+
+#### Fix 'TODO/FIXME' in codebase
+
+There are few cases of pending features, or pending validations, which are
+pending from sometime. You can pick them in the given file, and choose to
+fix it.
+
+
+### I don't know C, but I am interested to contribute in some way!
+
+You are most welcome! Our community is open for your contribution! First thing
+which comes to our mind is **documentation**. Next is, **testing** or validation.
+
+If you have some hardware, and want to run some performance comparisons with
+different version, or options, and help us to tune better is also a great help.
+
+
+#### Documentation
+
+1. We have some documentation in [glusterfs repo](../), go through these, and
+see if you can help us to keep up-to-date.
+
+2. The https://docs.gluster.org is powered by https://github.com/gluster/glusterdocs
+repo. You can check out the repo, and help in keeping that up-to-date.
+
+3. [Our website](https://gluster.org) is maintained by https://github.com/gluster/glusterweb
+repo. Help us to keep this up-to-date, and add content there.
+
+4. Write blogs about Gluster, and your experience, and make world know little
+more about Gluster, and your use-case, and how it helped to solve the problem.
+
+
+#### Testing
+
+1. There is a regression test suite in glusterfs, which runs with every patch, and is
+triggered by just running `./run-tests.sh` from the root of the project repo.
+
+You can add more test case to match your use-case, and send it as a patch, so you
+can make sure all future patches in glusterfs would keep your usecase intact.
+
+2. [Glusto-Tests](https://github.com/gluster/glusto-tests): This is another testing
+framework written for gluster, and makes use of clustered setup to test different
+use-cases, and helps to validate many bugs.
+
+
+#### Ansible
+
+Gluster Organization has rich set of ansible roles, which are actively maintained.
+Feel free to check them out here - https://github.com/gluster/gluster-ansible
+
+
+#### Monitoring
+
+We have prometheus repo, and are actively working on adding more metrics. Add what
+you need @ https://github.com/gluster/gluster-prometheus
+
+
+#### Health-Report
+
+This is a project, where at any given point in time, you want to run some set of
+commands locally, and get an output to analyze the status, it can be added.
+Contribute @ https://github.com/gluster/gluster-health-report
+
+
+### All these C/bash/python is old-school, I want something in containers.
+
+We have something for you too :-)
+
+Please visit our https://github.com/gluster/gcs repo for checking how you can help,
+and how gluster can help you in container world.
+
+
+### Note
+
+For any queries, best way is to contact us through mailing-list, <mailto:gluster-devel@gluster.org>
diff --git a/doc/developer-guide/posix.md b/doc/developer-guide/posix.md
new file mode 100644
index 00000000000..84c813e55a2
--- /dev/null
+++ b/doc/developer-guide/posix.md
@@ -0,0 +1,59 @@
+storage/posix translator
+========================
+
+Notes
+-----
+
+### `SET_FS_ID`
+
+This is so that all filesystem checks are done with the user's
+uid/gid and not GlusterFS's uid/gid.
+
+### `MAKE_REAL_PATH`
+
+This macro concatenates the base directory of the posix volume
+('option directory') with the given path.
+
+### `need_xattr` in lookup
+
+If this flag is passed, lookup returns a xattr dictionary that contains
+the file's create time, the file's contents, and the version number
+of the file.
+
+This is a hack to increase small file performance. If an application
+wants to read a small file, it can finish its job with just a lookup
+call instead of a lookup followed by read.
+
+### `getdents`/`setdents`
+
+These are used by unify to set and get directory entries.
+
+### `ALIGN_BUF`
+
+Macro to align an address to a page boundary (4K).
+
+### `priv->export_statfs`
+
+In some cases, two exported volumes may reside on the same
+partition on the server. Sending statvfs info for both
+the volumes will lead to erroneous df output at the client,
+since free space on the partition will be counted twice.
+
+In such cases, user can disable exporting statvfs info
+on one of the volumes by setting this option.
+
+### `xattrop`
+
+This fop is used by replicate to set version numbers on files.
+
+### `getxattr`/`setxattr` hack to read/write files
+
+A key, `GLUSTERFS_FILE_CONTENT_STRING`, is handled in a special way by
+`getxattr`/`setxattr`. A getxattr with the key will return the entire
+content of the file as the value. A `setxattr` with the key will write
+the value as the entire content of the file.
+
+### `posix_checksum`
+
+This calculates a simple XOR checksum on all entry names in a
+directory that is used by unify to compare directory contents.
diff --git a/doc/rpc-for-glusterfs.changes-done.txt b/doc/developer-guide/rpc-for-glusterfs.changes-done.txt
index 6bbbca78826..6bbbca78826 100644
--- a/doc/rpc-for-glusterfs.changes-done.txt
+++ b/doc/developer-guide/rpc-for-glusterfs.changes-done.txt
diff --git a/doc/developer-guide/rpc-for-glusterfs.new-versions.md b/doc/developer-guide/rpc-for-glusterfs.new-versions.md
new file mode 100644
index 00000000000..e3da5efa4a2
--- /dev/null
+++ b/doc/developer-guide/rpc-for-glusterfs.new-versions.md
@@ -0,0 +1,32 @@
+# GlusterFS RPC program versions
+
+## Compatibility
+
+RPC layer of glusterfs is implemented with possible changes over the protocol layers in mind. If there are any changes in the FOPs from what is assumed to be client side, and whats in serverside, they are to be added as a separate program table.
+
+### Program tables and Versions
+
+A given RPC program has a specific Task, and Version along with actors belonging to the program. In any of the programs, if a new actor is added, it is very important to define one more program with different version, and then keep both, if both are supported. Or else, it is important to handle the 'handshake' properly.
+
+#### Server details
+
+More info on RPC program is at `rpc/rpc-lib/src/rpcsvc.h` and check for structure `rpcsvc_actor_t` and `struct rpcsvc_program`. For usage, check `xlators/protocol/server/src/server-rpc-fops.c`
+
+#### Client details
+
+For details on client structures check `rpc/rpc-lib/src/rpc-clnt.h` for `rpc_clnt_procedure_t` and `rpc_clnt_program_t`. For usage, check `xlators/protocol/client/src/client-rpc-fops.c`
+
+## Protocol
+
+A protocol is what is agreed between two parties. In glusterfs, a RPC protocol is defined as .x file, which then gets converted to .c/.h file using `rpcgen`. There are different protocols defined for communication between `xlators/mgmt/glusterd <==> glusterfsd`, `gluster CLI <==> glusterd`, and `client-protocol <==> server-protocol`
+
+Once a protocol is defined and a release is made with that protocol, make sure no one changes it. Any edits to a given structure there should be a new version of the structure, and also it should get used in new actor, and thus new program version.
+
+## Server and Client Handshake
+
+When a client succeeds to establish a connect (it can be any transport, socket, ib-verbs or unix), client sends a dump (GF_DUMP_DUMP) request to server, which will respond back with all the supported versions of the server RPC (the supported programs which are registered with `rpcsvc_program_register()`).
+
+A client which expects certain programs to be present in server, it should be taking care of looking for it in the handshake methods, and take appropriate action depending on what to do next. In general a compatibility issue should be handled at handshake level itself, thus we can clearly let user/admin know of any 'in-compatibilities'. 
+As a developer of GlusterFS protocol layer, one just has to make sure *never to make changes to existing program structures*, but they have to add new programs if required. New programs can have the same actors as present in existing, and also little more. Or it can even have same actor behave differently, take different parameter.
+
+If this is followed properly, there would be smooth upgrade / downgrade of versions. If not, technically, it is 100% guarantee of getting compatibility related issues.
diff --git a/doc/developer-guide/syncop.md b/doc/developer-guide/syncop.md
new file mode 100644
index 00000000000..bcc8bd08e01
--- /dev/null
+++ b/doc/developer-guide/syncop.md
@@ -0,0 +1,72 @@
+# syncop framework
+A coroutines-based, cooperative multi-tasking framework.
+
+## Topics
+
+- Glossary
+- Lifecycle of a synctask
+- Existing usage
+
+
+## Glossary
+
+### syncenv
+
+syncenv is an object that provides access to a pool of worker threads.
+synctasks execute in a syncenv.
+
+### synctask
+
+synctask can be informally defined as a pair of function pointers, namely _the
+call_ and _the callback_ (see syncop.h for more details).
+
+        synctask_fn_t - 'the call'
+        synctask_cbk_t - 'the callback'
+
+synctask has two modes of operation,
+
+1. The calling thread waits for the synctask to complete.
+2. The calling thread schedules the synctask and continues.
+
+synctask guarantees that the callback is called _after_ the call completes.
+
+### Lifecycle of a synctask
+
+A synctask could go into the following stages while in execution.
+
+- CREATED  - On calling synctask_create/synctask_new.
+
+- RUNNABLE - synctask is queued in env->runq.
+
+- RUNNING  - When one of syncenv's worker threads calls synctask_switch_to.
+
+- WAITING  - When a synctask calls synctask_yield.
+
+- DONE     - When a synctask has run to completion.
+
+
+                                +-------------------------------+
+                                |            CREATED            |
+                                +-------------------------------+
+                                  |
+                                  | synctask_new/synctask_create
+                                  v
+                                +-------------------------------+
+                                |    RUNNABLE (in env->runq)    | <+
+                                +-------------------------------+  |
+                                  |                                |
+                                  | synctask_switch_to             |
+                                  v                                |
+ +------+  on task completion   +-------------------------------+  |
+ | DONE | <-------------------- |            RUNNING            |  | synctask_wake/wake
+ +------+                       +-------------------------------+  |
+                                  |                                |
+                                  | synctask_yield/yield           |
+                                  v                                |
+                                +-------------------------------+  |
+                                |    WAITING (in env->waitq)    | -+
+                                +-------------------------------+
+
+Note: A synctask is not guaranteed to run on the same thread throughout its
+lifetime. Every time a synctask yields, it is possible for it to run on a
+different thread.
diff --git a/doc/developer-guide/thread-naming.md b/doc/developer-guide/thread-naming.md
new file mode 100644
index 00000000000..513140d4437
--- /dev/null
+++ b/doc/developer-guide/thread-naming.md
@@ -0,0 +1,104 @@
+Thread Naming
+================
+Gluster processes spawn many threads; some threads are created by libglusterfs
+library, while others are created by xlators. When gfapi library is used in an
+application, some threads belong to the application and some are spawned by
+gluster libraries. We also have features where n number of threads are spawned
+to act as worker threads for same operation.
+
+In all the above cases, it is useful to be able to determine the list of threads
+that exist in runtime. Naming threads when you create them is the easiest way to
+provide that information to kernel so that it can then be queried by any means.
+
+How to name threads
+-------------------
+We have two wrapper functions in libglusterfs for creating threads. They take
+name as an argument and set thread name after its creation.
+
+```C
+gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
+                  void *(*start_routine)(void *), void *arg, const char *name)
+```
+
+```C
+gf_thread_create_detached (pthread_t *thread,
+                           void *(*start_routine)(void *), void *arg,
+                           const char *name)
+```
+
+As max name length for a thread in POSIX is only 16 characters including the
+'\0' character, you have to be a little creative with naming. Also, it is
+important that all Gluster threads have common prefix. Considering these
+conditions, we have "glfs_" as prefix for all the threads created by these
+wrapper functions. It is responsibility of the owner of thread to provide the
+suffix part of the name. It does not have to be a descriptive name, as it has
+only 10 letters to work with. However, it should be unique enough such that it
+can be matched with a table which describes it.
+
+If n number of threads are spwaned to perform same function, it is must that the
+threads are numbered.
+
+Table of thread names
+---------------------
+Thread names don't have to be a descriptive; however, it should be unique enough
+such that it can be matched with a table below without ambiguity.
+
+- bdaio    - block device aio
+- brfsscan - bit rot fs scanner
+- brhevent - bit rot event handler
+- brmon    - bit rot monitor
+- brosign  - bit rot one shot signer
+- brpobj   - bit rot object processor
+- brsproc  - bit rot scrubber
+- brssign  - bit rot stub signer
+- brswrker - bit rot worker
+- clogc    - changelog consumer
+- clogcbki - changelog callback invoker
+- clogd    - changelog dispatcher
+- clogecon - changelog reverse connection
+- clogfsyn - changelog fsync
+- cloghcon - changelog history consumer
+- clogjan  - changelog janitor
+- clogpoll - changelog poller
+- clogproc - changelog process
+- clogro   - changelog rollover
+- ctrcomp  - change time recorder compaction
+- dhtdf    - dht defrag task
+- dhtdg    - dht defrag start
+- dhtfcnt  - dht rebalance file counter
+- ecshd    - ec heal daemon
+- epollN   -  epoll thread
+- fdlwrker - fdl worker
+- fusenoti - fuse notify
+- fuseproc - fuse main thread
+- gdhooks  - glusterd hooks
+- glfspoll -  gfapi poller thread
+- idxwrker - index worker
+- iosdump  - io stats dump
+- iotwr    - io thread worker
+- jbrflush - jbr flush
+- leasercl - lease recall
+- memsweep - sweeper thread for mem pools
+- nfsauth  - nfs auth
+- nfsnsm   - nfs nsm
+- nfsudp   - nfs udp mount
+- nlmmon   - nfs nlm/nsm mon
+- posixaio - posix aio
+- posixfsy - posix fsync
+- posixhc  - posix heal
+- posixjan - posix janitor
+- posixrsv - posix reserve
+- quiesce  - quiesce dequeue
+- rdmaAsyn - rdma async event handler
+- rdmaehan - rdma completion handler
+- rdmarcom - rdma receive completion handler
+- rdmascom - rdma send completion handler
+- rpcsvcrh - rpcsvc request handler
+- scleanup - socket cleanup
+- shdheal  - self heal daemon
+- sigwait  -  glusterfsd sigwaiter
+- spoller  - socket poller
+- sprocN   - syncop worker thread
+- tbfclock - token bucket filter token generator thread
+- timer    - timer thread
+- upreaper - upcall reaper
diff --git a/doc/developer-guide/translator-development.md b/doc/developer-guide/translator-development.md
new file mode 100644
index 00000000000..f75935519f6
--- /dev/null
+++ b/doc/developer-guide/translator-development.md
@@ -0,0 +1,683 @@
+Translator development
+======================
+
+Setting the Stage
+-----------------
+
+This is the first post in a series that will explain some of the details of
+writing a GlusterFS translator, using some actual code to illustrate.
+
+Before we begin, a word about environments. GlusterFS is over 300K lines of
+code spread across a few hundred files. That's no Linux kernel or anything, but
+ you're still going to be navigating through a lot of code in every
+code-editing session, so some kind of cross-referencing is *essential*. I use
+cscope with the vim bindings, and if I couldn't do Crtl+G and such to jump
+between definitions all the time my productivity would be cut in half. You may
+prefer different tools, but as I go through these examples you'll need
+something functionally similar to follow on. OK, on with the show.
+
+The first thing you need to know is that translators are not just bags of
+functions and variables. They need to have a very definite internal structure
+so that the translator-loading code can figure out where all the pieces are.
+The way it does this is to use dlsym to look for specific names within your
+shared-object file, as follow (from `xlator.c`):
+
+```
+if (!(xl->fops = dlsym (handle, "fops"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(fops) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->cbks = dlsym (handle, "cbks"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(cbks) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->init = dlsym (handle, "init"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(init) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->fini = dlsym (handle, "fini"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(fini) on %s",
+        dlerror ());
+    goto out;
+}
+```
+
+In this example, `xl` is a pointer to the in-memory object for the translator
+we're loading. As you can see, it's looking up various symbols *by name* in the
+ shared object it just loaded, and storing pointers to those symbols. Some of
+them (e.g. init) are functions, while others (e.g. fops) are dispatch tables
+containing pointers to many functions. Together, these make up the translator's
+ public interface.
+
+Most of this glue or boilerplate can easily be found at the bottom of one of
+the source files that make up each translator. We're going to use the `rot-13`
+translator just for fun, so in this case you'd look in `rot-13.c` to see this:
+
+```
+struct xlator_fops fops = {
+    .readv          = rot13_readv,
+    .writev         = rot13_writev
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+{ .key    = {"encrypt-write"},
+  .type = GF_OPTION_TYPE_BOOL
+},
+{ .key    = {"decrypt-read"},
+  .type = GF_OPTION_TYPE_BOOL
+},
+{ .key    = {NULL} },
+};
+```
+
+The `fops` table, defined in `xlator.h`, is one of the most important pieces.
+This table contains a pointer to each of the filesystem functions that your
+translator might implement -- `open`, `read`, `stat`, `chmod`, and so on. There
+are 82 such functions in all, but don't worry; any that you don't specify here
+will be see as null and filled with defaults from `defaults.c` when your
+translator is loaded. In this particular example, since `rot-13` is an
+exceptionally simple translator, we only fill in two entries for `readv` and
+`writev`.
+
+There are actually two other tables, also required to have predefined names,
+that are also used to find translator functions: `cbks` (which is empty in this
+ snippet) and `dumpops` (which is missing entirely). The first of these specify
+ entry points for when inodes are forgotten or file descriptors are released.
+In other words, they're destructors for objects in which your translator might
+ have an interest. Mostly you can ignore them, because the default behavior
+handles even the simpler cases of translator-specific inode/fd context
+automatically. However, if the context you attach is a complex structure
+requiring complex cleanup, you'll need to supply these functions. As for
+dumpops, that's just used if you want to provide functions to pretty-print
+various structures in logs. I've never used it myself, though I probably
+should. What's noteworthy here is that we don't even define dumpops. That's
+because all of the functions that might use these dispatch functions will check
+ for `xl->dumpops` being `NULL` before calling through it. This is in sharp
+contrast to the behavior for `fops` and `cbks`, which *must* be present. If
+they're not, translator loading will fail because these pointers are not
+checked every time and if they're `NULL` then we'll segfault. That's why we
+provide an empty definition for cbks; it's OK for the individual function
+pointers to be NULL, but not for the whole table to be absent.
+
+The last piece I'll cover today is options. As you can see, this is a table of
+translator-specific option names and some information about their types.
+GlusterFS actually provides a pretty rich set of types (`volume_option_type_t`
+in `options.`h) which includes paths, translator names, percentages, and times
+in addition to the obvious integers and strings. Also, the `volume_option_t`
+structure can include information about alternate names, min/max/default
+values, enumerated string values, and descriptions. We don't see any of these
+here, so let's take a quick look at some more complex examples from afr.c and
+then come back to `rot-13`.
+
+```
+{ .key  = {"data-self-heal-algorithm"},
+  .type = GF_OPTION_TYPE_STR,
+  .default_value = "",
+  .description   = "Select between \"full\", \"diff\". The "
+           "\"full\" algorithm copies the entire file from "
+           "source to sink. The \"diff\" algorithm copies to "
+           "sink only those blocks whose checksums don't match "
+           "with those of source.",
+  .value = { "diff", "full", "" }
+},
+{ .key  = {"data-self-heal-window-size"},
+  .type = GF_OPTION_TYPE_INT,
+  .min  = 1,
+  .max  = 1024,
+  .default_value = "1",
+  .description = "Maximum number blocks per file for which "
+         "self-heal process would be applied simultaneously."
+},
+```
+
+When your translator is loaded, all of this information is used to parse the
+options actually provided in the volfile, and then the result is turned into a
+dictionary and stored as `xl->options`. This dictionary is then processed by
+your init function, which you can see being looked up in the first code
+fragment above. We're only going to look at a small part of the `rot-13`'s
+init for now.
+
+```
+priv->decrypt_read = 1;
+priv->encrypt_write = 1;
+
+data = dict_get (this->options, "encrypt-write");
+if (data) {
+    if (gf_string2boolean (data->data, &priv->encrypt_write
+        == -1) {
+        gf_log (this->name, GF_LOG_ERROR,
+            "encrypt-write takes only boolean options");
+        return -1;
+    }
+}
+```
+
+What we can see here is that we're setting some defaults in our priv structure,
+then looking to see if an `encrypt-write` option was actually provided. If so,
+we convert and store it. This is a pretty classic use of dict_get to fetch a
+field from a dictionary, and of using one of many conversion functions in
+`common-utils.c` to convert `data->data` into something we can use.
+
+So far we've covered the basic of how a translator gets loaded, how we find its
+various parts, and how we process its options. In my next Translator 101 post,
+we'll go a little deeper into other things that init and its companion fini
+might do, and how some other fields in our `xlator_t` structure (commonly
+referred to as this) are commonly used.
+
+`init`, `fini`, and private context
+-----------------------------------
+
+In the previous Translator 101 post, we looked at some of the dispatch tables
+and options processing in a translator. This time we're going to cover the rest
+ of the "shell" of a translator -- i.e. the other global parts not specific to
+handling a particular request.
+
+Let's start by looking at the relationship between a translator and its shared
+library. At a first approximation, this is the relationship between an object
+and a class in just about any object-oriented programming language. The class
+defines behaviors, but has to be instantiated as an object to have any kind of
+existence. In our case the object is an `xlator_t`. Several of these might be
+created within the same daemon, sharing all of the same code through init/fini
+and dispatch tables, but sharing *no data*. You could implement shared data (as
+ static variables in your shared libraries) but that's strongly discouraged.
+Every function in your shared library will get an `xlator_t` as an argument,
+and should use it. This lack of class-level data is one of the points where
+the analogy to common OOP systems starts to break down. Another place is the
+complete lack of inheritance. Translators inherit behavior (code) from exactly
+one shared library -- looked up and loaded using the `type` field in a volfile
+`volume ... end-volume` block -- and that's it -- not even single inheritance,
+no subclasses or superclasses, no mixins or prototypes, just the relationship
+between an object and its class. With that in mind, let's turn to the init
+function that we just barely touched on last time.
+
+```
+int32_t
+init (xlator_t *this)
+{
+    data_t *data = NULL;
+    rot_13_private_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log ("rot13", GF_LOG_ERROR,
+            "FATAL: rot13 should have exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log (this->name, GF_LOG_WARNING,
+            "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
+    if (!priv)
+        return -1;
+```
+
+At the very top, we see the function signature -- we get a pointer to the
+`xlator_t` object that we're initializing, and we return an `int32_t` status.
+As with most functions in the translator API, this should be zero to indicate
+success. In this case it's safe to return -1 for failure, but watch out: in
+dispatch-table functions, the return value means the status of the *function
+call* rather than the *request*. A request error should be reflected as a
+callback with a non-zero `op_re`t value, but the dispatch function itself
+should still return zero. In fact, the handling of a non-zero return from a
+dispatch function is not all that robust (we recently had a bug report in
+HekaFS related to this) so it's something you should probably avoid
+altogether. This only underscores the difference between dispatch functions
+and `init`/`fini` functions, where non-zero returns *are* expected and handled
+logically by aborting the translator setup. We can see that down at the
+bottom, where we return -1 to indicate that we couldn't allocate our
+private-data area (more about that later).
+
+The first thing this init function does is check that the translator is being
+set up in the right kind of environment. Translators are called by parents and
+in turn call children. Some translators are "initial" translators that inject
+requests into the system from elsewhere -- e.g. mount/fuse injecting requests
+from the kernel, protocol/server injecting requests from the network. Those
+translators don't need parents, but `rot-13` does and so we check for that.
+Similarly, some translators are "final" translators that (from the perspective
+of the current process) terminate requests instead of passing them on -- e.g.
+`protocol/client` passing them to another node, `storage/posix` passing them to
+a local filesystem. Other translators "multiplex" between multiple children --
+ passing each parent request on to one (`cluster/dht`), some
+(`cluster/stripe`), or all (`cluster/afr`) of those children. `rot-13` fits
+into none of those categories either, so it checks that it has *exactly one*
+child. It might be more convenient or robust if translator shared libraries
+had standard variables describing these requirements, to be checked in a
+consistent way by the translator-loading infrastructure itself instead of by
+each separate init function, but this is the way translators work today.
+
+The last thing we see in this fragment is allocating our private data area.
+This can literally be anything we want; the infrastructure just provides the
+priv pointer as a convenience but takes no responsibility for how it's used. In
+ this case we're using `GF_CALLOC` to allocate our own `rot_13_private_t`
+structure. This gets us all the benefits of GlusterFS's memory-leak detection
+infrastructure, but the way we're calling it is not quite ideal. For one thing,
+ the first two arguments -- from `calloc(3)` -- are kind of reversed. For
+another, notice how the last argument is zero. That can actually be an
+enumerated value, to tell the GlusterFS allocator *what* type we're
+allocating. This can be very useful information for memory profiling and leak
+detection, so it's recommended that you follow the example of any
+x`xx-mem-types.h` file elsewhere in the source tree instead of just passing
+zero here (even though that works).
+
+To finish our tour of standard initialization/termination, let's look at the
+end of `init` and the beginning of `fini`:
+
+```
+    this->private = priv;
+    gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
+    return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+    rot_13_private_t *priv = this->private;
+
+    if (!priv)
+        return;
+    this->private = NULL;
+    GF_FREE (priv);
+```
+
+At the end of init we're just storing our private-data pointer in the `priv`
+field of our `xlator_t`, then returning zero to indicate that initialization
+succeeded. As is usually the case, our fini is even simpler. All it really has
+to do is `GF_FREE` our private-data pointer, which we do in a slightly
+roundabout way here. Notice how we don't even have a return value here, since
+there's nothing obvious and useful that the infrastructure could do if `fini`
+failed.
+
+That's practically everything we need to know to get our translator through
+loading, initialization, options processing, and termination. If we had defined
+ no dispatch functions, we could actually configure a daemon to use our
+translator and it would work as a basic pass-through from its parent to a
+single child. In the next post I'll cover how to build the translator and
+configure a daemon to use it, so that we can actually step through it in a
+debugger and see how it all fits together before we actually start adding
+functionality.
+
+This Time For Real
+------------------
+
+In the first two parts of this series, we learned how to write a basic
+translator skeleton that can get through loading, initialization, and option
+processing. This time we'll cover how to build that translator, configure a
+volume to use it, and run the glusterfs daemon in debug mode.
+
+Unfortunately, there's not much direct support for writing new translators. You
+can check out a GlusterFS tree and splice in your own translator directory, but
+ that's a bit painful because you'll have to update multiple makefiles plus a
+bunch of autoconf garbage. As part of the HekaFS project, I basically reverse
+engineered the truly necessary parts of the translator-building process and
+then pestered one of the Fedora glusterfs package maintainers (thanks
+daMaestro!) to add a `glusterfs-devel` package with the required headers. Since
+ then the complexity level in the HekaFS tree has crept back up a bit, but I
+still remember the simple method and still consider it the easiest way to get
+started on a new translator. For the sake of those not using Fedora, I'm going
+to describe a method that doesn't depend on that header package. What it does
+depend on is a GlusterFS source tree, much as you might have cloned from GitHub
+ or the Gluster review site. This tree doesn't have to be fully built, but you
+do need to run `autogen.sh` and configure in it. Then you can take the
+following simple makefile and put it in a directory with your actual source.
+
+```
+# Change these to match your source code.
+TARGET  = rot-13.so
+OBJECTS = rot-13.o
+
+# Change these to match your environment.
+GLFS_SRC = /srv/glusterfs
+GLFS_LIB = /usr/lib64
+HOST_OS  = GF_LINUX_HOST_OS
+
+# You shouldn't need to change anything below here.
+
+CFLAGS  = -fPIC -Wall -O0 -g \
+      -DHAVE_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE \
+      -D$(HOST_OS) -I$(GLFS_SRC) -I$(GLFS_SRC)/contrib/uuid \
+      -I$(GLFS_SRC)/libglusterfs/src
+LDFLAGS = -shared -nostartfiles -L$(GLFS_LIB)
+LIBS = -lglusterfs -lpthread
+
+$(TARGET): $(OBJECTS)
+    $(CC) $(OBJECTS) $(LDFLAGS) -o $(TARGET) $(OBJECTS) $(LIBS)
+```
+
+Yes, it's still Linux-specific. Mea culpa. As you can see, we're sticking with
+the `rot-13` example, so you can just copy the files from
+`xlators/encryption/rot-13/src` in your GlusterFS tree to follow on. Type
+`make` and you should be rewarded with a nice little `.so` file.
+
+```
+xlator_example$ ls -l rot-13.so
+-rwxr-xr-x. 1 jeff jeff 40784 Nov 16 16:41 rot-13.so
+```
+
+Notice that we've built with optimization level zero and debugging symbols
+included, which would not typically be the case for a packaged version of
+GlusterFS. Let's put our version of `rot-13.so` into a slightly different file
+on our system, so that it doesn't stomp on the installed version (not that
+you'd ever want to use that anyway).
+
+```
+xlator_example# ls /usr/lib64/glusterfs/3git/xlator/encryption/
+crypt.so crypt.so.0 crypt.so.0.0.0 rot-13.so rot-13.so.0
+rot-13.so.0.0.0
+xlator_example# cp rot-13.so \
+    /usr/lib64/glusterfs/3git/xlator/encryption/my-rot-13.so
+```
+
+These paths represent the current Gluster filesystem layout, which is likely to
+be deprecated in favor of the Fedora layout; your paths may vary. At this point
+ we're ready to configure a volume using our new translator. To do that, I'm
+going to suggest something that's strongly discouraged except during
+development (the Gluster guys are going to hate me for this): write our own
+volfile. Here's just about the simplest volfile you'll ever see.
+
+```
+volume my-posix
+    type storage/posix
+    option directory /srv/export
+end-volume
+
+volume my-rot13
+    type encryption/my-rot-13
+    subvolumes my-posix
+end-volume
+```
+
+All we have here is a basic brick using `/srv/export` for its data, and then
+an instance of our translator layered on top -- no client or server is
+necessary for what we're doing, and the system will automatically push a
+mount/fuse translator on top if there's no server translator. To try this out,
+all we need is the following command (assuming the directories involved already
+ exist).
+
+```
+xlator_example$ glusterfs --debug -f my.vol /srv/import
+```
+
+You should be rewarded with a whole lot of log output, including the text of
+the volfile (this is very useful for debugging problems in the field). If you
+go to another window on the same machine, you can see that you have a new
+filesystem mounted.
+
+```
+~$ df /srv/import
+Filesystem   1K-blocks     Used    Available Use% Mounted on
+/srv/xlator_example/my.vol
+             114506240     2706176 105983488   3% /srv/import
+```
+
+Just for fun, write something into a file in `/srv/import`, then look at the
+corresponding file in `/srv/export` to see it all `rot-13`'ed for you.
+
+```
+~$ echo hello > /srv/import/a_file
+~$ cat /srv/export/a_file
+uryyb
+```
+
+There you have it -- functionality you control, implemented easily, layered on
+top of local storage. Now you could start adding functionality -- real
+encryption, perhaps -- and inevitably having to debug it. You could do that the
+ old-school way, with `gf_log` (preferred) or even plain old `printf`, or you
+could  run daemons under `gdb` instead. Alternatively, you could wait for the
+next  Translator 101 post, where we'll be doing exactly that.
+
+Debugging a Translator
+----------------------
+
+Now that we've learned what a translator looks like and how to build one, it's
+time to run one and actually watch it work. The best way to do this is good
+old-fashioned `gdb`, as follows (using some of the examples from last time).
+
+```
+xlator_example# gdb glusterfs
+GNU gdb (GDB) Red Hat Enterprise Linux (7.2-50.el6)
+...
+(gdb) r --debug -f my.vol /srv/import
+Starting program: /usr/sbin/glusterfs --debug -f my.vol /srv/import
+...
+[2011-11-23 11:23:16.495516] I [fuse-bridge.c:2971:fuse_init]
+    0-glusterfs-fuse: FUSE inited with protocol versions:
+    glusterfs 7.13 kernel 7.13
+```
+
+If you get to this point, your glusterfs client process is already running. You
+can go to another window to see the mountpoint, do file operations, etc.
+
+```
+~# df /srv/import
+Filesystem   1K-blocks     Used    Available Use% Mounted on
+/root/xlator_example/my.vol
+             114506240     2643968 106045568   3% /srv/import
+~# ls /srv/import
+a_file
+~# cat /srv/import/a_file
+hello
+```
+
+Now let's interrupt the process and see where we are.
+
+```
+
+Program received signal SIGINT, Interrupt.
+0x0000003a0060b3dc in pthread_cond_wait@@GLIBC_2.3.2 ()
+                   from /lib64/libpthread.so.0
+(gdb) info threads
+  5 Thread 0x7fffeffff700 (LWP 27206)  0x0000003a002dd8c7
+    in readv ()
+    from /lib64/libc.so.6
+  4 Thread 0x7ffff50e3700 (LWP 27205)  0x0000003a0060b75b
+    in pthread_cond_timedwait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+  3 Thread 0x7ffff5f02700 (LWP 27204)  0x0000003a0060b3dc
+    in pthread_cond_wait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+  2 Thread 0x7ffff6903700 (LWP 27203)  0x0000003a0060f245
+    in sigwait ()
+    from /lib64/libpthread.so.0
+* 1 Thread 0x7ffff7957700 (LWP 27196)  0x0000003a0060b3dc
+    in pthread_cond_wait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+```
+
+Like any non-toy server, this one has multiple threads. What are they all
+doing? Honestly, even I don't know. Thread 1 turns out to be in
+`event_dispatch_epoll`, which means it's the one handling all of our network
+I/O. Note that with socket multi-threading patch this will change, with one
+thread in `socket_poller` per connection. Thread 2 is in `glusterfs_sigwaiter`
+which means signals will be isolated to that thread. Thread 3 is in
+`syncenv_task`, so it's a worker process for synchronous requests such as
+those used by the rebalance and repair code. Thread 4 is in
+`janitor_get_next_fd`, so it's waiting for a chance to close no-longer-needed
+file descriptors on the local filesystem. (I admit I had to look that one up,
+BTW.) Lastly, thread 5 is in `fuse_thread_proc`, so it's the one fetching
+requests from our FUSE interface. You'll often see many more threads than
+this, but it's a pretty good basic set. Now, let's set a breakpoint so we can
+actually watch a request.
+
+```
+(gdb) b rot13_writev
+Breakpoint 1 at 0x7ffff50e4f0b: file rot-13.c, line 119.
+(gdb) c
+Continuing.
+```
+
+At this point we go into our other window and do something that will involve a write.
+
+```
+~# echo goodbye > /srv/import/another_file
+(back to the first window)
+[Switching to Thread 0x7fffeffff700 (LWP 27206)]
+
+Breakpoint 1, rot13_writev (frame=0x7ffff6e4402c, this=0x638440,
+    fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1, offset=0,
+    iobref=0x7fffe8001070) at rot-13.c:119
+119   rot_13_private_t *priv = (rot_13_private_t *)this->private;
+```
+
+Remember how we built with debugging symbols enabled and no optimization? That
+will be pretty important for the next few steps. As you can see, we're in
+`rot13_writev`, with several parameters.
+
+* `frame` is our always-present frame pointer for this request. Also,
+  `frame->local` will point to any local data we created and attached to the
+  request ourselves.
+* `this` is a pointer to our instance of the `rot-13` translator. You can examine
+  it if you like to see the name, type, options, parent/children, inode table,
+  and other stuff associated with it.
+* `fd` is a pointer to a file-descriptor *object* (`fd_t`, not just a
+  file-descriptor index which is what most people use "fd" for). This in turn
+  points to an inode object (`inode_t`) and we can associate our own
+  `rot-13`-specific data with either of these.
+* `vector` and `count` together describe the data buffers for this write, which
+  we'll get to in a moment.
+* `offset` is the offset into the file at which we're writing.
+* `iobref` is a buffer-reference object, which is used to track the life cycle
+  of buffers containing read/write data. If you look closely, you'll notice that
+  `vector[0].iov_base` points to the same address as `iobref->iobrefs[0].ptr`, which
+  should give you some idea of the inter-relationships between vector and iobref.
+
+OK, now what about that `vector`? We can use it to examine the data being
+written, like this.
+
+```
+(gdb) p vector[0]
+$2 = {iov_base = 0x7ffff7936000, iov_len = 8}
+(gdb) x/s 0x7ffff7936000
+0x7ffff7936000:     "goodbye\n"
+```
+
+It's not always safe to view this data as a string, because it might just as
+well be binary data, but since we're generating the write this time it's safe
+and convenient. With that knowledge, let's step through things a bit.
+
+```
+(gdb) s
+120        if (priv->encrypt_write)
+(gdb)
+121            rot13_iovec (vector, count);
+(gdb)
+rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57
+57        for (i = 0; i < count; i++) {
+(gdb)
+58            rot13 (vector[i].iov_base, vector[i].iov_len);
+(gdb)
+rot13 (buf=0x7ffff7936000 "goodbye\n", len=8) at rot-13.c:45
+45        for (i = 0; i < len; i++) {
+(gdb)
+46            if (buf[i] >= 'a' && buf[i] <= 'z')
+(gdb)
+47                buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
+```
+
+Here we've stepped into `rot13_iovec`, which iterates through our vector
+calling `rot13`, which in turn iterates through the characters in that chunk
+doing the  `rot-13` operation if/as appropriate. This is pretty straightforward
+stuff, so  let's skip to the next interesting bit.
+
+```
+(gdb) fin
+Run till exit from #0  rot13 (buf=0x7ffff7936000 "goodbye\n",
+    len=8) at rot-13.c:47
+rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57
+57        for (i = 0; i < count; i++) {
+(gdb) fin
+Run till exit from #0  rot13_iovec (vector=0x7fffe8000cd8,
+    count=1) at rot-13.c:57
+rot13_writev (frame=0x7ffff6e4402c, this=0x638440,
+    fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1,
+    offset=0, iobref=0x7fffe8001070) at rot-13.c:123
+123        STACK_WIND (frame,
+(gdb) b 129
+Breakpoint 2 at 0x7ffff50e4f35: file rot-13.c, line 129.
+(gdb) b rot13_writev_cbk
+Breakpoint 3 at 0x7ffff50e4db3: file rot-13.c, line 106.
+(gdb) c
+```
+
+So we've set breakpoints on both the callback and the statement following the
+`STACK_WIND`. Which one will we hit first?
+
+```
+Breakpoint 3, rot13_writev_cbk (frame=0x7ffff6e4402c,
+    cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0,
+    prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30)
+    at rot-13.c:106
+106  STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
+                          prebuf, postbuf);
+(gdb) bt
+#0  rot13_writev_cbk (frame=0x7ffff6e4402c,
+    cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0,
+    prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30)
+    at rot-13.c:106
+#1  0x00007ffff52f1b37 in posix_writev (frame=0x7ffff6e440d8,
+    this=<value optimized out>, fd=<value optimized out>,
+    vector=<value optimized out>, count=1,
+    offset=<value optimized out>, iobref=0x7fffe8001070)
+    at posix.c:2217
+#2  0x00007ffff50e513e in rot13_writev (frame=0x7ffff6e4402c,
+    this=0x638440, fd=0x7ffff409802c, vector=0x7fffe8000cd8,
+    count=1, offset=0, iobref=0x7fffe8001070) at rot-13.c:123
+```
+
+Surprise! We're in `rot13_writev_cbk` now, called (indirectly) while we're
+still in `rot13_writev` before `STACK_WIND` returns (still at rot-13.c:123). If
+ you did any request cleanup here, then you need to be careful about what you
+do in the remainder of `rot13_writev` because data may have been freed etc.
+It's tempting to say you should just do the cleanup in `rot13_writev` after
+the `STACK_WIND,` but that's not valid because it's also possible that some
+other translator returned without calling `STACK_UNWIND` -- i.e. before
+`rot13_writev` is called, so then it would be the one getting null-pointer
+errors instead. To put it another way, the callback and the return from
+`STACK_WIND` can occur in either order or even simultaneously on different
+threads. Even if you were to use reference counts, you'd have to make sure to
+use locking or atomic operations to avoid races, and it's not worth it. Unless
+you *really* understand the possible flows of control and know what you're
+doing, it's better to do cleanup in the callback and nothing after
+`STACK_WIND.`
+
+At this point all that's left is a `STACK_UNWIND` and a return. The
+`STACK_UNWIND` invokes our parent's completion callback, and in this case our
+parent is FUSE so at that point the VFS layer is notified of the write being
+complete. Finally, we return through several levels of normal function calls
+until we come back to fuse_thread_proc, which waits for the next request.
+
+So that's it. For extra fun, you might want to repeat this exercise by stepping
+through some other call -- stat or setxattr might be good choices -- but you'll
+ have to use a translator that actually implements those calls to see much
+that's interesting. Then you'll pretty much know everything I knew when I
+started writing my first for-real translators, and probably even a bit more. I
+hope you've enjoyed this series, or at least found it useful, and if you have
+any suggestions for other topics I should cover please let me know (via
+comments or email, IRC or Twitter).
+
+Other versions
+--------------
+
+Original author's site:
+
+ * [Translator 101 - Setting the Stage](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-class-1-setting-the-stage/)
+
+ * [Translator 101 - Init, Fini and Private Context](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-2-init-fini-and-private-context/)
+
+ * [Translator 101 - This Time for Real](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-3-this-time-for-real/)
+
+ * [Translator 101 - Debugging a Translator](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-4-debugging-a-translator/)
+
+Gluster community site:
+
+ * [Translators](https://docs.gluster.org/en/latest/Quick-Start-Guide/Architecture/#translators)
diff --git a/doc/developer-guide/unittest.md b/doc/developer-guide/unittest.md
new file mode 100644
index 00000000000..5c6c0a8a039
--- /dev/null
+++ b/doc/developer-guide/unittest.md
@@ -0,0 +1,228 @@
+# Unit Tests in GlusterFS
+
+## Overview
+[Art-of-unittesting][definitionofunittest] provides a good definition for unit tests.  A good unit test is:
+
+* Able to be fully automated
+* Has full control over all the pieces running (Use mocks or stubs to achieve this isolation when needed)
+* Can be run in any order  if part of many other tests
+* Runs in memory (no DB or File access, for example)
+* Consistently returns the same result (You always run the same test, so no random numbers, for example. save those for integration or range tests)
+* Runs fast
+* Tests a single logical concept in the system
+* Readable
+* Maintainable
+* Trustworthy (when you see its result, you don’t need to debug the code just to be sure)
+
+## cmocka
+GlusterFS unit test framework is based on [cmocka][].  cmocka provides
+developers with methods to isolate and test modules written in C language.  It
+also provides integration with Jenkins by providing JUnit XML compliant unit
+test results.
+
+cmocka
+
+## Running Unit Tests
+To execute the unit tests, all you need is to type `make check`.  Here is a step-by-step example assuming you just cloned a GlusterFS tree:
+
+```
+$ ./autogen.sh
+$ ./configure --enable-debug
+$ make check
+```
+
+Sample output:
+
+```
+PASS: mem_pool_unittest
+============================================================================
+Testsuite summary for glusterfs 3git
+============================================================================
+# TOTAL: 1
+# PASS:  1
+# SKIP:  0
+# XFAIL: 0
+# FAIL:  0
+# XPASS: 0
+# ERROR: 0
+============================================================================
+```
+
+In this example, `mem_pool_unittest` has multiple tests inside, but `make check` assumes that the program itself is the test, and that is why it only shows one test.  Here is the output when we run `mem_pool_unittest` directly:
+
+```
+$ ./libglusterfs/src/mem_pool_unittest
+[==========] Running 10 test(s).
+[ RUN      ] test_gf_mem_acct_enable_set
+Expected assertion data != ((void *)0) occurred
+[       OK ] test_gf_mem_acct_enable_set
+[ RUN      ] test_gf_mem_set_acct_info_asserts
+Expected assertion xl != ((void *)0) occurred
+Expected assertion size > ((4 + sizeof (size_t) + sizeof (xlator_t *) + 4 + 8) + 8) occurred
+Expected assertion type <= xl->mem_acct.num_types occurred
+[       OK ] test_gf_mem_set_acct_info_asserts
+[ RUN      ] test_gf_mem_set_acct_info_memory
+[       OK ] test_gf_mem_set_acct_info_memory
+[ RUN      ] test_gf_calloc_default_calloc
+[       OK ] test_gf_calloc_default_calloc
+[ RUN      ] test_gf_calloc_mem_acct_enabled
+[       OK ] test_gf_calloc_mem_acct_enabled
+[ RUN      ] test_gf_malloc_default_malloc
+[       OK ] test_gf_malloc_default_malloc
+[ RUN      ] test_gf_malloc_mem_acct_enabled
+[       OK ] test_gf_malloc_mem_acct_enabled
+[ RUN      ] test_gf_realloc_default_realloc
+[       OK ] test_gf_realloc_default_realloc
+[ RUN      ] test_gf_realloc_mem_acct_enabled
+[       OK ] test_gf_realloc_mem_acct_enabled
+[ RUN      ] test_gf_realloc_ptr
+Expected assertion ((void *)0) != ptr occurred
+[       OK ] test_gf_realloc_ptr
+[==========] 10 test(s) run.
+[  PASSED  ] 10 test(s).
+[  FAILED  ] 0 test(s).
+[  REPORT  ] Created libglusterfs_mem_pool_xunit.xml report
+```
+
+
+## Writing Unit Tests
+
+### Enhancing your C functions
+
+#### Programming by Contract
+Add the following to your C file:
+
+```c
+#include <cmocka_pbc.h>
+```
+
+```c
+/*
+ * Programming by Contract is a programming methodology
+ * which binds the caller and the function called to a
+ * contract. The contract is represented using Hoare Triple:
+ *      {P} C {Q}
+ * where {P} is the precondition before executing command C,
+ * and {Q} is the postcondition.
+ *
+ * See also:
+ * http://en.wikipedia.org/wiki/Design_by_contract
+ * http://en.wikipedia.org/wiki/Hoare_logic
+ * http://dlang.org/dbc.html
+ */
+ #ifndef CMOCKERY_PBC_H_
+#define CMOCKERY_PBC_H_
+
+#if defined(UNIT_TESTING) || defined (DEBUG)
+
+#include <assert.h>
+
+/*
+ * Checks caller responsibility against contract
+ */
+#define REQUIRE(cond) assert(cond)
+
+/*
+ * Checks function reponsability against contract.
+ */
+#define ENSURE(cond) assert(cond)
+
+/*
+ * While REQUIRE and ENSURE apply to functions, INVARIANT
+ * applies to classes/structs.  It ensures that intances
+ * of the class/struct are consistent. In other words,
+ * that the instance has not been corrupted.
+ */
+#define INVARIANT(invariant_fnc) do{ (invariant_fnc) } while (0);
+
+#else
+#define REQUIRE(cond) do { } while (0);
+#define ENSURE(cond) do { } while (0);
+#define INVARIANT(invariant_fnc) do{ } while (0);
+
+#endif /* defined(UNIT_TESTING) || defined (DEBUG) */
+#endif /* CMOCKERY_PBC_H_ */
+```
+
+##### Example
+This is an _extremely_ simple example:
+
+```c
+int divide (int n, int d)
+{
+    int ans;
+
+    REQUIRE(d != 0);
+
+    ans = n / d;
+
+    // As code is added to this function throughout its lifetime,
+    // ENSURE will assert that data will be returned
+    // according to the contract.  Again this is an
+    // extremely simple example. :-D
+    ENSURE( ans == (n / d) );
+
+    return ans;
+}
+
+```
+
+##### Important Note
+`REQUIRE`, `ENSURE`, and `INVARIANT` are only available when `DEBUG` or `UNIT_TESTING` are set in the CFLAGS.  You must pass `--enable-debug` to `./configure` to enable PBC on your non-unittest builds.
+
+#### Overriding functions
+Cmockery2 provides its own memory allocation functions which check for buffer overrun and memory leaks.  The following header file must be included **last** to be able to override any of the memory allocation functions:
+
+```c
+#include <cmocka.h>
+```
+
+This file will only take effect with the `UNIT_TESTING` CFLAG is set.
+
+### Creating a unit test
+Once you identify the C file you would like to test, first create a `unittest` directory under the directory where the C file is located.  This will isolate the unittests to a different directory.
+
+Next, you need to edit the `Makefile.am` file in the directory where your C file is located.  Initialize the
+`Makefile.am` if it does not already have the following sections:
+
+```
+#### UNIT TESTS #####
+CLEANFILES += *.gcda *.gcno *_xunit.xml
+noinst_PROGRAMS =
+TESTS =
+```
+
+Now you can add the following for each of the unit tests that you would like to build:
+
+```
+### UNIT TEST xxx_unittest ###
+xxx_unittest_CPPFLAGS = $(xxx_CPPFLAGS)
+xxx_unittest_SOURCES = xxx.c \
+                       unittest/xxx_unittest.c
+xxx_unittest_CFLAGS = $(UNITTEST_CFLAGS)
+xxx_unittest_LDFLAGS = $(UNITTEST_LDFLAGS)
+noinst_PROGRAMS += xxx_unittest
+TESTS += xxx_unittest
+```
+
+Where `xxx` is the name of your C file. For example, look at `libglusterfs/src/Makefile.am`.
+
+Copy the simple unit test from the [cmocka API][cmockaapi] to `unittest/xxx_unittest.c`.  If you would like to see an example of a unit test, please refer to `libglusterfs/src/unittest/mem_pool_unittest.c`.
+
+#### Mocking
+You may see that the linker will complain about missing functions needed by the C file you would like to test.  Identify the required functions, then place their stubs in a file called `unittest/xxx_mock.c`, then include this file in `Makefile.am` in `xxx_unittest_SOURCES`.  This will allow you to you Cmockery2's mocking functions.
+
+#### Running the unit test
+You can type `make` in the directory where the C file is located.  Once you built it and there are no errors, you can execute the test either by directly executing the program (in our example above it is called `xxx_unittest` ), or by running `make check`.
+
+#### Debugging
+Sometimes you may need to debug your unit test.  To do that, you will have to point `gdb` to the binary which is located in the same directory as the source.  For example, you can do the following from the root of the source tree to debug `mem_pool_unittest`:
+
+```
+$ gdb libglusterfs/src/mem_pool_unittest
+```
+
+
+[cmocka]: https://cmocka.org
+[definitionofunittest]: http://artofunittesting.com/definition-of-a-unit-test/
+[cmockapi]: https://api.cmocka.org
diff --git a/doc/developer-guide/versioning.md b/doc/developer-guide/versioning.md
new file mode 100644
index 00000000000..10c1511b79b
--- /dev/null
+++ b/doc/developer-guide/versioning.md
@@ -0,0 +1,44 @@
+Versioning
+==========
+
+### current
+
+The number of the current interface exported by the library. A current value
+of '1', means that you are calling the interface exported by this library
+interface 1.
+
+### revision
+
+The implementation number of the most recent interface exported by this library.
+In this case, a revision value of `0` means that this is the first
+implementation of the interface.
+
+If the next release of this library exports the same interface, but has a
+different implementation (perhaps some bugs have been fixed), the revision
+number will be higher, but current number will be the same. In that case, when
+given a choice, the library with the highest revision will always be used by
+the runtime loader.
+
+### age
+
+The number of previous additional interfaces supported by this library. If age
+were '2', then this library can be linked into executables which were built with
+a release of this library that exported the current interface number, current,
+or any of the previous two interfaces. By definition age must be less than or
+equal to current. At the outset, only the first ever interface is implemented,
+so age can only be `0'.
+
+For every release of the library `-version-info` argument needs to be set
+correctly depending on any interface changes you have made.
+
+This is quite straightforward when you understand what the three numbers mean:
+
+If you have changed any of the sources for this library, the revision number
+must be incremented. This is a new revision of the current interface. If the
+interface has changed, then current must be incremented, and revision reset
+to '0'.
+
+This is the first revision of a new interface. If the new interface is a
+superset of the previous interface (that is, if the previous interface has not
+been broken by the changes in this new release), then age must be incremented.
+This release is backwards compatible with the previous release.
diff --git a/doc/developer-guide/write-behind.md b/doc/developer-guide/write-behind.md
new file mode 100644
index 00000000000..0d78964fa20
--- /dev/null
+++ b/doc/developer-guide/write-behind.md
@@ -0,0 +1,56 @@
+performance/write-behind translator
+===================================
+
+Basic working
+--------------
+
+Write behind is basically a translator to lie to the application that the 
+write-requests are finished, even before it is actually finished.
+
+On a regular translator tree without write-behind, control flow is like this:
+
+1. application makes a `write()` system call.
+2. VFS ==> FUSE ==> `/dev/fuse`.
+3. fuse-bridge initiates a glusterfs `writev()` call.
+4. `writev()` is `STACK_WIND()`ed up to client-protocol or storage translator.
+5. client-protocol, on receiving reply from server, starts `STACK_UNWIND()` towards the fuse-bridge.
+
+On a translator tree with write-behind, control flow is like this:
+
+1. application makes a `write()` system call.
+2. VFS ==> FUSE ==> `/dev/fuse`.
+3. fuse-bridge initiates a glusterfs `writev()` call.
+4. `writev()` is `STACK_WIND()`ed up to write-behind translator.
+5. write-behind adds the write buffer to its internal queue and does a `STACK_UNWIND()` towards the fuse-bridge.
+
+write call is completed in application's percepective. after 
+`STACK_UNWIND()`ing towards the fuse-bridge, write-behind initiates a fresh 
+writev() call to its child translator, whose replies will be consumed by 
+write-behind itself. Write-behind _doesn't_ cache the write buffer, unless 
+`option flush-behind on` is specified in volume specification file.
+
+Windowing
+---------
+
+With respect to write-behind, each write-buffer has three flags: `stack_wound`, `write_behind` and `got_reply`.
+
+* `stack_wound`: if set, indicates that write-behind has initiated `STACK_WIND()` towards child translator.
+* `write_behind`: if set, indicates that write-behind has done `STACK_UNWIND()` towards fuse-bridge.
+* `got_reply`: if set, indicates that write-behind has received reply from child translator for a `writev()` `STACK_WIND()`. a request will be destroyed by write-behind only if this flag is set.
+
+Currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0.
+
+window size limits the aggregate size of currently pending write requests. once 
+the pending requests' size has reached the window size, write-behind blocks  
+writev() calls from fuse-bridge. Blocking is only from application's 
+perspective. Write-behind does `STACK_WIND()` to child translator 
+straight-away, but hold behind the `STACK_UNWIND()` towards fuse-bridge. 
+`STACK_UNWIND()` is done only once write-behind gets enough replies to 
+accommodate for currently blocked request.
+
+Flush behind
+------------
+
+If `option flush-behind on` is specified in volume specification file, then 
+write-behind sends aggregate write requests to child translator, instead of 
+regular per request `STACK_WIND()`s.
diff --git a/doc/developer-guide/writing-a-cloudsync-plugin.md b/doc/developer-guide/writing-a-cloudsync-plugin.md
new file mode 100644
index 00000000000..907860aaed8
--- /dev/null
+++ b/doc/developer-guide/writing-a-cloudsync-plugin.md
@@ -0,0 +1,164 @@
+## How to write your Cloudsync Plugin
+
+### Background
+
+Cloudsync translator is part of the archival feature in Gluster. This translator
+does the retrieval/download part. Each cold  file will be archived to a remote
+storage (public or private cloud). On future access to the file, it will be
+retrieved from the remote storage by Cloudsync translator. Each remote storage
+would need a unique plugin. Cloudsync translator will load this plugin and
+call the necessary plugin functions.
+
+Upload can be done by a script or program. There are some basic mandatory steps
+for uploading the data. There is a sample script for crawl and upload given at
+the end of this guide.
+
+### Necessary changes to create a plugin
+
+1. Define store_methods:
+
+* This structure is the container of basic functions that will be called by
+  cloudsync xlator.
+
+        typedef struct store_methodds {
+                int (*fop_download) (call_frame_t *frame, void *config);
+                /* return type should be the store config */
+                void *(*fop_init) (xlator_t *this);
+                int (*fop_reconfigure) (xlator_t *this, dict_t *options);
+                void (*fop_fini) (void *config);
+        } store_methods_t;
+
+
+        Member details:
+        fop_download:
+                This is the download function pointer.
+
+                frame: This will have the fd to write data downloaded from
+                       cloud to GlusterFS.(frame->local->fd)
+
+                config: This is the plugin configuration variable.
+
+                Note: Structure cs_local_t has member dlfd and dloffset which
+                      can be used to manage the writes to Glusterfs.
+                      Include cloudsync-common.h to access these structures.
+
+        fop_init:
+                This is similar to xlator init. But here the return value is
+                the plugin configuration pointer. This pointer will be stored
+                in the cloudsync private object (priv->stores->config). And
+                the cloudsync private object can be accessed by "this->private"
+                where "this" is of type xlator_t.
+
+        fop_reconfigure:
+                This is similar to xlator_reconfigure.
+
+        fop_fini:
+                Free plugin resources.
+
+        Note: Store_methods_t is part of cs_private_t which in turn part of
+              xlator_t. Create a store_methods_t object named "store_ops" in
+              your plugin. For example
+
+              store_methods_t store_ops = {
+                .fop_download = aws_download_s3,
+                .fop_init     = aws_init,
+                .fop_reconfigure = aws_reconfigure,
+                .fop_fini     = aws_fini,
+              };
+
+
+2 - Making Cloudsync xlator aware of the plugin:
+
+        Add an entry in to the cs_plugin structure. For example
+        struct cs_plugin plugins[] = {
+                {
+                  .name = "amazons3",
+                  .library = "libamazons3.so",
+                  .description = "amazon s3 store."
+                },
+
+                {.name = NULL},
+        };
+
+        Description about individual members:
+                name: name of the plugin
+                library: This is the shared object created. Cloudsync will load
+                         this library during init.
+                description: Describe about the plugin.
+
+3- Makefile Changes in Cloudsync:
+
+		Add <plugin.la> to cloudsync_la_LIBADD variable.
+
+4 - Configure.ac changes:
+
+                In cloudsync section add the necessary dependency checks for
+                the plugin.
+
+5 - Export symbols:
+
+        Cloudsync needs "store_ops" to resolve all plugin functions.
+        Create a file <plugin>.sym and add write "store_ops" to it.
+
+
+### Sample script for upload
+This script assumes amazon s3 is the target cloud and bucket name is
+gluster-bucket. User can do necessary aws configuration using command
+"aws configure". Currently for amazons3 there are four gluster settings
+available.
+1- features.s3plugin-seckey -> s3 secret key
+2- features.s3plugin-keyid -> s3 key id
+3- features.s3plugin-bucketid -> bucketid
+4- features.s3plugin-hostname -> hostname e.g. s3.amazonaws.com
+
+Additionally set cloudsync storetype to amazons3.
+
+gluster v set <VOLNAME> cloudsync-storetype amazons3
+
+Now create a mount dedicated for this upload task.
+
+That covers necessary configurations needed.
+
+Below is the sample script for upload. The script will crawl directly on the
+brick and will upload those files which are not modified for last one month.
+It needs two arguments.
+1st arguement - Gluster Brick path
+2nd arguement - coldness that is how many days since the file was modified.
+3rd argument - dedicated gluster mount point created for uploading.
+
+Once the cloud setup is done, run the following script on individual bricks.
+Note: For an AFR volume, pick only the fully synchronized brick among the
+replica bricks.
+
+```
+target_folder=$1
+coldness=$2
+mnt=$3
+
+cd $target_folder
+for i in `find . -type f  | grep -v "glusterfs" | sed 's/..//'`
+do
+        echo "processing $mnt/$i"
+
+        #check whether the file is already archived
+        getfattr -n trusted.glusterfs.cs.remote $i &> /dev/null
+        if [ $? -eq 0 ]
+        then
+                echo "file $mnt/$i is already archived"
+        else
+                #upload to cloud
+                aws s3 cp $mnt/$i s3://gluster-bucket/
+                mtime=`stat -c "%Y" $mnt/$i`
+
+                #post processing of upload
+                setfattr -n trusted.glusterfs.csou.complete -v $mtime $mnt/$i
+                if [ $? -ne 0 ]
+                then
+                        echo "archiving of file $mnt/$i failed"
+                else
+                        echo "archiving of file $mnt/$i succeeded"
+                fi
+
+        fi
+done
+```
diff --git a/doc/developer-guide/xlator-classification.md b/doc/developer-guide/xlator-classification.md
new file mode 100644
index 00000000000..6073df9375f
--- /dev/null
+++ b/doc/developer-guide/xlator-classification.md
@@ -0,0 +1,221 @@
+# xlator categories and expectations
+
+The purpose of the document is to define a category for various xlators
+and expectations around what each category means from a perspective of
+health and maintenance of a xlator.
+
+The need to do this is to ensure certain categories are kept in good
+health, and helps the community and contributors focus their efforts around the
+same.
+
+This document also provides implementation details for xlator developers to
+declare a category for any xlator.
+
+## Table of contents
+1. Audience
+2. Categories (and expectations of each category)
+3. Implementation and usage details
+
+## Audience
+
+This document is intended for the following community participants,
+- New xlator contributors
+- Existing xlator maintainers
+- Packaging and gluster management stack maintainers
+
+For a more user facing understanding it is recommended to read section (TBD)
+in the gluster documentation.
+
+## Categories
+1. Experimental (E)
+2. TechPreview  (TP)
+3. Maintained (M)
+4. Deprecated (D)
+5. Obsolete (O)
+
+### Experimental (E)
+
+Developed in the experimental branch, for exploring new features. These xlators
+are NEVER packaged as a part of releases, interested users and contributors can
+build and work with these from sources. In the future, these maybe available as
+an package based on a weekly build of the same.
+
+#### Quality expectations
+- Compiles or passes smoke tests
+- Does not break nightly experimental regressions
+  - NOTE: If a nightly is broken, then all patches that were merged are reverted
+  till the errant patch is found and subsequently fixed
+
+### TechPreview (TP)
+
+Xlators in master or release branches that are not deemed fit to be in
+production deployments, but are feature complete to invite feedback and host
+user data.
+
+These xlators will be worked upon with priority by maintainers/authors who are
+involved in making them more stable than xlators in the Experimental/Deprecated/
+Obsolete categories.
+
+There is no guarantee that these xlators will move to the Maintained state, and
+may just get Obsoleted based on feedback, or other project goals or technical
+alternatives.
+
+#### Quality expectations
+- Same as Maintained, minus
+  - Performance, Scale, other(?)
+  - *TBD* *NOTE* Need inputs, Intention is all quality goals as in Maintained,
+  other than the list above (which for now has scale and performance)
+
+### Maintained (M)
+
+These xltors are part of the core Gluster functionality and are maintained
+actively. These are part of master and release branches and are higher in
+priority of maintainers and other interested contributors.
+
+#### Quality expectations
+
+NOTE: A short note on what each of these mean are added here, details to follow.
+
+NOTE: Out of the gate all of the following are not mandated, consider the
+following a desirable state to reach as we progress on each
+
+- Bug backlog: Actively address bug backlog
+- Enhancement backlog: Actively maintain outstanding enhancement backlog (need
+        not be acted on, but should be visible to all)
+- Review backlog: Actively keep this below desired counts and states
+- Static code health:  Actively meet near-zero issues in this regard
+  - Coverity, spellcheck and other checks
+- Runtime code health: Actively meet defined coverage levels in this regard
+  - Coverage, others?
+  - Per-patch regressions
+  - Glusto runs
+  - Performance
+  - Scalability
+- Technical specifications: Implementation details should be documented and
+        updated at regular cadence (even per patch that change assumptions in
+        here)
+- User documentation: User facing details should be maintained to current
+        status in the documentation
+- Debuggability: Steps, tools, procedures should be documented and maintained
+        each release/patch as applicable
+- Troubleshooting: Steps, tools, procedures should be documented and maintained
+        each release/patch as applicable
+  - Steps/guides for self service
+  - Knowledge base for problems
+- Other common criteria that will apply: Required metrics/desired states to be
+        defined per criteria
+  - Monitoring, usability, statedump, and other such xlator expectations
+
+### Deprecated (D)
+
+Xlators on master or release branches that would be obsoleted and/or replaced
+with similar or other functionality in the next major release.
+
+#### Quality expectations
+- Retain status-quo when moved to this state, till it is moved to obsoleted
+- Provide migration steps if feature provided by the xlator is replaced with
+other xlators
+
+### Obsolete (O)
+
+Xlator/code still in tree, but not packaged or shipped or maintained in any
+form. This is noted as a category till the code is removed from the tree.
+
+These xlators and their corresponding code and test health will not be executed.
+
+#### Quality expectations
+- None
+
+## Implementation and usage details
+
+### How to specify an xlators category
+
+While defining 'xlator_api_t' structure for the corresponding xlator, add a
+flag like below:
+
+```
+diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c
+index 0f0e53bac2..8267d6897c 100644
+--- a/xlators/performance/nl-cache/src/nl-cache.c
++++ b/xlators/performance/nl-cache/src/nl-cache.c
+@@ -869,4 +869,5 @@ xlator_api_t xlator_api = {
+         .cbks          = &nlc_cbks,
+         .options       = nlc_options,
+         .identifier    = "nl-cache",
++        .category      = GF_TECH_PREVIEW,
+ };
+diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
+index 8d39720e7f..235de27c19 100644
+--- a/xlators/performance/quick-read/src/quick-read.c
++++ b/xlators/performance/quick-read/src/quick-read.c
+@@ -1702,4 +1702,5 @@ xlator_api_t xlator_api = {
+         .cbks          = &qr_cbks,
+         .options       = qr_options,
+         .identifier    = "quick-read",
++        .category      = GF_MAINTAINED,
+ };
+```
+
+Similarly, if a particular option is in different state other than
+the xlator state, one can add the same flag in options structure too.
+
+```
+diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
+index 0e86e33d03..81996743d1 100644
+--- a/xlators/cluster/afr/src/afr.c
++++ b/xlators/cluster/afr/src/afr.c
+@@ -772,6 +772,7 @@ struct volume_options options[] = {
+           .description = "Maximum latency for shd halo replication in msec."
+         },
+         { .key   = {"halo-enabled"},
++          .category = GF_TECH_PREVIEW,
+           .type  = GF_OPTION_TYPE_BOOL,
+           .default_value = "False",
+
+```
+
+
+### User experience using the categories
+
+#### Ability to use a category
+
+This section details which category of xlators can be used when and specifics
+around when each category is enabled.
+
+1. Maintained category xlators can be used by default, this implies, volumes
+created with these xlators enabled will throw no warnings, or need no user
+intervention to use the xlator.
+
+2. Tech Preview category xlators needs cluster configuration changes to allow
+these xlatorss to be used in volumes, further, logs will contain a message
+stating TP xlators are in use. Without the cluster configured to allow TP
+xlators, volumes created or edited to use such xlators would result in errors.
+  - (TBD) Cluster configuration option
+  - (TBD) Warning message
+  - (TBD) Code mechanics on how this is achieved
+
+3. Deprecated category xlators can be used by default, but will throw a warning
+in the logs that such are in use and will be deprecated in the future.
+  - (TBD) Warning message
+
+4. Obsolete category xlators will not be packaged and hence cannot be used from
+release builds.
+
+5. Experimental category xlators will not be packaged and hence cannot be used
+from release builds, if running experimental (weekly or other such) builds,
+these will throw a warning in the logs stating experimental xlators are in use.
+  - (TBD) Warning message
+
+#### Ability to query xlator category
+
+(TBD) Need to provide the ability to query xlator categories, or list xlators
+and their respective categories.
+
+#### User facing changes
+
+User facing changes that are expected due to this change include the following,
+- Cluster wide option to enable TP xlators, or more generically a category
+level of xlators
+- Errors in commands that fail due to invalid categories
+- Warning messages in logs to denote certain categories of xlators are in use
+- (TBD) Ability to query xlators and their respective categories
diff --git a/doc/errno.list.bsd.txt b/doc/errno.list.bsd.txt
deleted file mode 100644
index 350af25e4ab..00000000000
--- a/doc/errno.list.bsd.txt
+++ /dev/null
@@ -1,376 +0,0 @@
-/*-
- * Copyright (c) 1982, 1986, 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- * (c) UNIX System Laboratories, Inc.
- * All or some portions of this file are derived from material licensed
- * to the University of California by American Telephone and Telegraph
- * Co. or Unix System Laboratories, Inc. and are reproduced herein with
- * the permission of UNIX System Laboratories, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)errno.h	8.5 (Berkeley) 1/21/94
- * $FreeBSD: src/sys/sys/errno.h,v 1.28 2005/04/02 12:33:28 das Exp $
- */
-
-#ifndef _SYS_ERRNO_H_
-#define _SYS_ERRNO_H_
-
-#ifndef _KERNEL
-#include <sys/cdefs.h>
-__BEGIN_DECLS
-int *	__error(void);
-__END_DECLS
-#define	errno		(* __error())
-#endif
-
-#define	EPERM		1		/* Operation not permitted */
-#define	ENOENT		2		/* No such file or directory */
-#define	ESRCH		3		/* No such process */
-#define	EINTR		4		/* Interrupted system call */
-#define	EIO		5		/* Input/output error */
-#define	ENXIO		6		/* Device not configured */
-#define	E2BIG		7		/* Argument list too long */
-#define	ENOEXEC		8		/* Exec format error */
-#define	EBADF		9		/* Bad file descriptor */
-#define	ECHILD		10		/* No child processes */
-#define	EDEADLK		11		/* Resource deadlock avoided */
-					/* 11 was EAGAIN */
-#define	ENOMEM		12		/* Cannot allocate memory */
-#define	EACCES		13		/* Permission denied */
-#define	EFAULT		14		/* Bad address */
-#ifndef _POSIX_SOURCE
-#define	ENOTBLK		15		/* Block device required */
-#endif
-#define	EBUSY		16		/* Device busy */
-#define	EEXIST		17		/* File exists */
-#define	EXDEV		18		/* Cross-device link */
-#define	ENODEV		19		/* Operation not supported by device */
-#define	ENOTDIR		20		/* Not a directory */
-#define	EISDIR		21		/* Is a directory */
-#define	EINVAL		22		/* Invalid argument */
-#define	ENFILE		23		/* Too many open files in system */
-#define	EMFILE		24		/* Too many open files */
-#define	ENOTTY		25		/* Inappropriate ioctl for device */
-#ifndef _POSIX_SOURCE
-#define	ETXTBSY		26		/* Text file busy */
-#endif
-#define	EFBIG		27		/* File too large */
-#define	ENOSPC		28		/* No space left on device */
-#define	ESPIPE		29		/* Illegal seek */
-#define	EROFS		30		/* Read-only filesystem */
-#define	EMLINK		31		/* Too many links */
-#define	EPIPE		32		/* Broken pipe */
-
-/* math software */
-#define	EDOM		33		/* Numerical argument out of domain */
-#define	ERANGE		34		/* Result too large */
-
-/* non-blocking and interrupt i/o */
-#define	EAGAIN		35		/* Resource temporarily unavailable */
-#ifndef _POSIX_SOURCE
-#define	EWOULDBLOCK	EAGAIN		/* Operation would block */
-#define	EINPROGRESS	36		/* Operation now in progress */
-#define	EALREADY	37		/* Operation already in progress */
-
-/* ipc/network software -- argument errors */
-#define	ENOTSOCK	38		/* Socket operation on non-socket */
-#define	EDESTADDRREQ	39		/* Destination address required */
-#define	EMSGSIZE	40		/* Message too long */
-#define	EPROTOTYPE	41		/* Protocol wrong type for socket */
-#define	ENOPROTOOPT	42		/* Protocol not available */
-#define	EPROTONOSUPPORT	43		/* Protocol not supported */
-#define	ESOCKTNOSUPPORT	44		/* Socket type not supported */
-#define	EOPNOTSUPP	45		/* Operation not supported */
-#define	ENOTSUP		EOPNOTSUPP	/* Operation not supported */
-#define	EPFNOSUPPORT	46		/* Protocol family not supported */
-#define	EAFNOSUPPORT	47		/* Address family not supported by protocol family */
-#define	EADDRINUSE	48		/* Address already in use */
-#define	EADDRNOTAVAIL	49		/* Can't assign requested address */
-
-/* ipc/network software -- operational errors */
-#define	ENETDOWN	50		/* Network is down */
-#define	ENETUNREACH	51		/* Network is unreachable */
-#define	ENETRESET	52		/* Network dropped connection on reset */
-#define	ECONNABORTED	53		/* Software caused connection abort */
-#define	ECONNRESET	54		/* Connection reset by peer */
-#define	ENOBUFS		55		/* No buffer space available */
-#define	EISCONN		56		/* Socket is already connected */
-#define	ENOTCONN	57		/* Socket is not connected */
-#define	ESHUTDOWN	58		/* Can't send after socket shutdown */
-#define	ETOOMANYREFS	59		/* Too many references: can't splice */
-#define	ETIMEDOUT	60		/* Operation timed out */
-#define	ECONNREFUSED	61		/* Connection refused */
-
-#define	ELOOP		62		/* Too many levels of symbolic links */
-#endif /* _POSIX_SOURCE */
-#define	ENAMETOOLONG	63		/* File name too long */
-
-/* should be rearranged */
-#ifndef _POSIX_SOURCE
-#define	EHOSTDOWN	64		/* Host is down */
-#define	EHOSTUNREACH	65		/* No route to host */
-#endif /* _POSIX_SOURCE */
-#define	ENOTEMPTY	66		/* Directory not empty */
-
-/* quotas & mush */
-#ifndef _POSIX_SOURCE
-#define	EPROCLIM	67		/* Too many processes */
-#define	EUSERS		68		/* Too many users */
-#define	EDQUOT		69		/* Disc quota exceeded */
-
-/* Network File System */
-#define	ESTALE		70		/* Stale NFS file handle */
-#define	EREMOTE		71		/* Too many levels of remote in path */
-#define	EBADRPC		72		/* RPC struct is bad */
-#define	ERPCMISMATCH	73		/* RPC version wrong */
-#define	EPROGUNAVAIL	74		/* RPC prog. not avail */
-#define	EPROGMISMATCH	75		/* Program version wrong */
-#define	EPROCUNAVAIL	76		/* Bad procedure for program */
-#endif /* _POSIX_SOURCE */
-
-#define	ENOLCK		77		/* No locks available */
-#define	ENOSYS		78		/* Function not implemented */
-
-#ifndef _POSIX_SOURCE
-#define	EFTYPE		79		/* Inappropriate file type or format */
-#define	EAUTH		80		/* Authentication error */
-#define	ENEEDAUTH	81		/* Need authenticator */
-#define	EIDRM		82		/* Identifier removed */
-#define	ENOMSG		83		/* No message of desired type */
-#define	EOVERFLOW	84		/* Value too large to be stored in data type */
-#define	ECANCELED	85		/* Operation canceled */
-#define	EILSEQ		86		/* Illegal byte sequence */
-#define	ENOATTR		87		/* Attribute not found */
-
-#define EDOOFUS		88		/* Programming error */
-#endif /* _POSIX_SOURCE */
-
-#define	EBADMSG		89		/* Bad message */
-#define	EMULTIHOP	90		/* Multihop attempted */
-#define	ENOLINK		91		/* Link has been severed */
-#define	EPROTO		92		/* Protocol error */
-
-#ifndef _POSIX_SOURCE
-#define	ELAST		92		/* Must be equal largest errno */
-#endif /* _POSIX_SOURCE */
-
-#ifdef _KERNEL
-/* pseudo-errors returned inside kernel to modify return to process */
-#define	ERESTART	(-1)		/* restart syscall */
-#define	EJUSTRETURN	(-2)		/* don't modify regs, just return */
-#define	ENOIOCTL	(-3)		/* ioctl not handled by this layer */
-#define	EDIRIOCTL	(-4)		/* do direct ioctl in GEOM */
-#endif
-
-#endif
-/*-
- * Copyright (c) 1982, 1986, 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- * (c) UNIX System Laboratories, Inc.
- * All or some portions of this file are derived from material licensed
- * to the University of California by American Telephone and Telegraph
- * Co. or Unix System Laboratories, Inc. and are reproduced herein with
- * the permission of UNIX System Laboratories, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)errno.h	8.5 (Berkeley) 1/21/94
- * $FreeBSD: src/sys/sys/errno.h,v 1.28 2005/04/02 12:33:28 das Exp $
- */
-
-#ifndef _SYS_ERRNO_H_
-#define _SYS_ERRNO_H_
-
-#ifndef _KERNEL
-#include <sys/cdefs.h>
-__BEGIN_DECLS
-int *	__error(void);
-__END_DECLS
-#define	errno		(* __error())
-#endif
-
-#define	EPERM		1		/* Operation not permitted */
-#define	ENOENT		2		/* No such file or directory */
-#define	ESRCH		3		/* No such process */
-#define	EINTR		4		/* Interrupted system call */
-#define	EIO		5		/* Input/output error */
-#define	ENXIO		6		/* Device not configured */
-#define	E2BIG		7		/* Argument list too long */
-#define	ENOEXEC		8		/* Exec format error */
-#define	EBADF		9		/* Bad file descriptor */
-#define	ECHILD		10		/* No child processes */
-#define	EDEADLK		11		/* Resource deadlock avoided */
-					/* 11 was EAGAIN */
-#define	ENOMEM		12		/* Cannot allocate memory */
-#define	EACCES		13		/* Permission denied */
-#define	EFAULT		14		/* Bad address */
-#ifndef _POSIX_SOURCE
-#define	ENOTBLK		15		/* Block device required */
-#endif
-#define	EBUSY		16		/* Device busy */
-#define	EEXIST		17		/* File exists */
-#define	EXDEV		18		/* Cross-device link */
-#define	ENODEV		19		/* Operation not supported by device */
-#define	ENOTDIR		20		/* Not a directory */
-#define	EISDIR		21		/* Is a directory */
-#define	EINVAL		22		/* Invalid argument */
-#define	ENFILE		23		/* Too many open files in system */
-#define	EMFILE		24		/* Too many open files */
-#define	ENOTTY		25		/* Inappropriate ioctl for device */
-#ifndef _POSIX_SOURCE
-#define	ETXTBSY		26		/* Text file busy */
-#endif
-#define	EFBIG		27		/* File too large */
-#define	ENOSPC		28		/* No space left on device */
-#define	ESPIPE		29		/* Illegal seek */
-#define	EROFS		30		/* Read-only filesystem */
-#define	EMLINK		31		/* Too many links */
-#define	EPIPE		32		/* Broken pipe */
-
-/* math software */
-#define	EDOM		33		/* Numerical argument out of domain */
-#define	ERANGE		34		/* Result too large */
-
-/* non-blocking and interrupt i/o */
-#define	EAGAIN		35		/* Resource temporarily unavailable */
-#ifndef _POSIX_SOURCE
-#define	EWOULDBLOCK	EAGAIN		/* Operation would block */
-#define	EINPROGRESS	36		/* Operation now in progress */
-#define	EALREADY	37		/* Operation already in progress */
-
-/* ipc/network software -- argument errors */
-#define	ENOTSOCK	38		/* Socket operation on non-socket */
-#define	EDESTADDRREQ	39		/* Destination address required */
-#define	EMSGSIZE	40		/* Message too long */
-#define	EPROTOTYPE	41		/* Protocol wrong type for socket */
-#define	ENOPROTOOPT	42		/* Protocol not available */
-#define	EPROTONOSUPPORT	43		/* Protocol not supported */
-#define	ESOCKTNOSUPPORT	44		/* Socket type not supported */
-#define	EOPNOTSUPP	45		/* Operation not supported */
-#define	ENOTSUP		EOPNOTSUPP	/* Operation not supported */
-#define	EPFNOSUPPORT	46		/* Protocol family not supported */
-#define	EAFNOSUPPORT	47		/* Address family not supported by protocol family */
-#define	EADDRINUSE	48		/* Address already in use */
-#define	EADDRNOTAVAIL	49		/* Can't assign requested address */
-
-/* ipc/network software -- operational errors */
-#define	ENETDOWN	50		/* Network is down */
-#define	ENETUNREACH	51		/* Network is unreachable */
-#define	ENETRESET	52		/* Network dropped connection on reset */
-#define	ECONNABORTED	53		/* Software caused connection abort */
-#define	ECONNRESET	54		/* Connection reset by peer */
-#define	ENOBUFS		55		/* No buffer space available */
-#define	EISCONN		56		/* Socket is already connected */
-#define	ENOTCONN	57		/* Socket is not connected */
-#define	ESHUTDOWN	58		/* Can't send after socket shutdown */
-#define	ETOOMANYREFS	59		/* Too many references: can't splice */
-#define	ETIMEDOUT	60		/* Operation timed out */
-#define	ECONNREFUSED	61		/* Connection refused */
-
-#define	ELOOP		62		/* Too many levels of symbolic links */
-#endif /* _POSIX_SOURCE */
-#define	ENAMETOOLONG	63		/* File name too long */
-
-/* should be rearranged */
-#ifndef _POSIX_SOURCE
-#define	EHOSTDOWN	64		/* Host is down */
-#define	EHOSTUNREACH	65		/* No route to host */
-#endif /* _POSIX_SOURCE */
-#define	ENOTEMPTY	66		/* Directory not empty */
-
-/* quotas & mush */
-#ifndef _POSIX_SOURCE
-#define	EPROCLIM	67		/* Too many processes */
-#define	EUSERS		68		/* Too many users */
-#define	EDQUOT		69		/* Disc quota exceeded */
-
-/* Network File System */
-#define	ESTALE		70		/* Stale NFS file handle */
-#define	EREMOTE		71		/* Too many levels of remote in path */
-#define	EBADRPC		72		/* RPC struct is bad */
-#define	ERPCMISMATCH	73		/* RPC version wrong */
-#define	EPROGUNAVAIL	74		/* RPC prog. not avail */
-#define	EPROGMISMATCH	75		/* Program version wrong */
-#define	EPROCUNAVAIL	76		/* Bad procedure for program */
-#endif /* _POSIX_SOURCE */
-
-#define	ENOLCK		77		/* No locks available */
-#define	ENOSYS		78		/* Function not implemented */
-
-#ifndef _POSIX_SOURCE
-#define	EFTYPE		79		/* Inappropriate file type or format */
-#define	EAUTH		80		/* Authentication error */
-#define	ENEEDAUTH	81		/* Need authenticator */
-#define	EIDRM		82		/* Identifier removed */
-#define	ENOMSG		83		/* No message of desired type */
-#define	EOVERFLOW	84		/* Value too large to be stored in data type */
-#define	ECANCELED	85		/* Operation canceled */
-#define	EILSEQ		86		/* Illegal byte sequence */
-#define	ENOATTR		87		/* Attribute not found */
-
-#define EDOOFUS		88		/* Programming error */
-#endif /* _POSIX_SOURCE */
-
-#define	EBADMSG		89		/* Bad message */
-#define	EMULTIHOP	90		/* Multihop attempted */
-#define	ENOLINK		91		/* Link has been severed */
-#define	EPROTO		92		/* Protocol error */
-
-#ifndef _POSIX_SOURCE
-#define	ELAST		92		/* Must be equal largest errno */
-#endif /* _POSIX_SOURCE */
-
-#ifdef _KERNEL
-/* pseudo-errors returned inside kernel to modify return to process */
-#define	ERESTART	(-1)		/* restart syscall */
-#define	EJUSTRETURN	(-2)		/* don't modify regs, just return */
-#define	ENOIOCTL	(-3)		/* ioctl not handled by this layer */
-#define	EDIRIOCTL	(-4)		/* do direct ioctl in GEOM */
-#endif
-
-#endif
diff --git a/doc/errno.list.linux.txt b/doc/errno.list.linux.txt
deleted file mode 100644
index baa50792d4d..00000000000
--- a/doc/errno.list.linux.txt
+++ /dev/null
@@ -1,1586 +0,0 @@
-#define ICONV_SUPPORTS_ERRNO 1
-#include <errno.h>
-/* Error constants.  Linux specific version.
-   Copyright (C) 1996, 1997, 1998, 1999, 2005 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#ifdef _ERRNO_H
-
-# undef EDOM
-# undef EILSEQ
-# undef ERANGE
-# include <linux/errno.h>
-
-/* Linux has no ENOTSUP error code.  */
-# define ENOTSUP EOPNOTSUPP
-
-/* Older Linux versions also had no ECANCELED error code.  */
-# ifndef ECANCELED
-#  define ECANCELED	125
-# endif
-
-/* Support for error codes to support robust mutexes was added later, too.  */
-# ifndef EOWNERDEAD
-#  define EOWNERDEAD		130
-#  define ENOTRECOVERABLE	131
-# endif
-
-# ifndef __ASSEMBLER__
-/* Function to get address of global `errno' variable.  */
-extern int *__errno_location (void) __THROW __attribute__ ((__const__));
-
-#  if !defined _LIBC || defined _LIBC_REENTRANT
-/* When using threads, errno is a per-thread value.  */
-#   define errno (*__errno_location ())
-#  endif
-# endif /* !__ASSEMBLER__ */
-#endif /* _ERRNO_H */
-
-#if !defined _ERRNO_H && defined __need_Emath
-/* This is ugly but the kernel header is not clean enough.  We must
-   define only the values EDOM, EILSEQ and ERANGE in case __need_Emath is
-   defined.  */
-# define EDOM	33	/* Math argument out of domain of function.  */
-# define EILSEQ	84	/* Illegal byte sequence.  */
-# define ERANGE	34	/* Math result not representable.  */
-#endif /* !_ERRNO_H && __need_Emath */
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef APR_ERRNO_H
-#define APR_ERRNO_H
-
-/**
- * @file apr_errno.h
- * @brief APR Error Codes
- */
-
-#include "apr.h"
-
-#if APR_HAVE_ERRNO_H
-#include <errno.h>
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-
-/**
- * @defgroup apr_errno Error Codes
- * @ingroup APR 
- * @{
- */
-
-/**
- * Type for specifying an error or status code.
- */
-typedef int apr_status_t;
-
-/**
- * Return a human readable string describing the specified error.
- * @param statcode The error code the get a string for.
- * @param buf A buffer to hold the error string.
- * @param bufsize Size of the buffer to hold the string.
- */
-APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, 
-                                 apr_size_t bufsize);
-
-#if defined(DOXYGEN)
-/**
- * @def APR_FROM_OS_ERROR(os_err_type syserr)
- * Fold a platform specific error into an apr_status_t code.
- * @return apr_status_t
- * @param e The platform os error code.
- * @warning  macro implementation; the syserr argument may be evaluated
- *      multiple times.
- */
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-
-/**
- * @def APR_TO_OS_ERROR(apr_status_t statcode)
- * @return os_err_type
- * Fold an apr_status_t code back to the native platform defined error.
- * @param e The apr_status_t folded platform os error code.
- * @warning  macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error 
- *      or APR_FROM_OS_ERROR, the results are undefined.
- */
-#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-/** @def apr_get_os_error()
- * @return apr_status_t the last platform error, folded into apr_status_t, on most platforms
- * @remark This retrieves errno, or calls a GetLastError() style function, and
- *      folds it with APR_FROM_OS_ERROR.  Some platforms (such as OS2) have no
- *      such mechanism, so this call may be unsupported.  Do NOT use this
- *      call for socket errors from socket, send, recv etc!
- */
-
-/** @def apr_set_os_error(e)
- * Reset the last platform error, unfolded from an apr_status_t, on some platforms
- * @param e The OS error folded in a prior call to APR_FROM_OS_ERROR()
- * @warning This is a macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error
- *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a SetLastError() style function, unfolding statcode
- *      with APR_TO_OS_ERROR.  Some platforms (such as OS2) have no such
- *      mechanism, so this call may be unsupported.
- */
-
-/** @def apr_get_netos_error()
- * Return the last socket error, folded into apr_status_t, on all platforms
- * @remark This retrieves errno or calls a GetLastSocketError() style function,
- *      and folds it with APR_FROM_OS_ERROR.
- */
-
-/** @def apr_set_netos_error(e)
- * Reset the last socket error, unfolded from an apr_status_t
- * @param e The socket error folded in a prior call to APR_FROM_OS_ERROR()
- * @warning This is a macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error
- *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a WSASetLastError() style function, unfolding 
- *      socketcode with APR_TO_OS_ERROR.
- */
-
-#endif /* defined(DOXYGEN) */
-
-/**
- * APR_OS_START_ERROR is where the APR specific error values start.
- */
-#define APR_OS_START_ERROR     20000
-/**
- * APR_OS_ERRSPACE_SIZE is the maximum number of errors you can fit
- *    into one of the error/status ranges below -- except for
- *    APR_OS_START_USERERR, which see.
- */
-#define APR_OS_ERRSPACE_SIZE 50000
-/**
- * APR_OS_START_STATUS is where the APR specific status codes start.
- */
-#define APR_OS_START_STATUS    (APR_OS_START_ERROR + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_USERERR are reserved for applications that use APR that
- *     layer their own error codes along with APR's.  Note that the
- *     error immediately following this one is set ten times farther
- *     away than usual, so that users of apr have a lot of room in
- *     which to declare custom error codes.
- */
-#define APR_OS_START_USERERR    (APR_OS_START_STATUS + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_USEERR is obsolete, defined for compatibility only.
- * Use APR_OS_START_USERERR instead.
- */
-#define APR_OS_START_USEERR     APR_OS_START_USERERR
-/**
- * APR_OS_START_CANONERR is where APR versions of errno values are defined
- *     on systems which don't have the corresponding errno.
- */
-#define APR_OS_START_CANONERR  (APR_OS_START_USERERR \
-                                 + (APR_OS_ERRSPACE_SIZE * 10))
-/**
- * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into 
- *     apr_status_t values.
- */
-#define APR_OS_START_EAIERR    (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_SYSERR folds platform-specific system error values into 
- *     apr_status_t values.
- */
-#define APR_OS_START_SYSERR    (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE)
-
-/** no error. */
-#define APR_SUCCESS 0
-
-/** 
- * @defgroup APR_Error APR Error Values
- * <PRE>
- * <b>APR ERROR VALUES</b>
- * APR_ENOSTAT      APR was unable to perform a stat on the file 
- * APR_ENOPOOL      APR was not provided a pool with which to allocate memory
- * APR_EBADDATE     APR was given an invalid date 
- * APR_EINVALSOCK   APR was given an invalid socket
- * APR_ENOPROC      APR was not given a process structure
- * APR_ENOTIME      APR was not given a time structure
- * APR_ENODIR       APR was not given a directory structure
- * APR_ENOLOCK      APR was not given a lock structure
- * APR_ENOPOLL      APR was not given a poll structure
- * APR_ENOSOCKET    APR was not given a socket
- * APR_ENOTHREAD    APR was not given a thread structure
- * APR_ENOTHDKEY    APR was not given a thread key structure
- * APR_ENOSHMAVAIL  There is no more shared memory available
- * APR_EDSOOPEN     APR was unable to open the dso object.  For more 
- *                  information call apr_dso_error().
- * APR_EGENERAL     General failure (specific information not available)
- * APR_EBADIP       The specified IP address is invalid
- * APR_EBADMASK     The specified netmask is invalid
- * APR_ESYMNOTFOUND Could not find the requested symbol
- * </PRE>
- *
- * <PRE>
- * <b>APR STATUS VALUES</b>
- * APR_INCHILD        Program is currently executing in the child
- * APR_INPARENT       Program is currently executing in the parent
- * APR_DETACH         The thread is detached
- * APR_NOTDETACH      The thread is not detached
- * APR_CHILD_DONE     The child has finished executing
- * APR_CHILD_NOTDONE  The child has not finished executing
- * APR_TIMEUP         The operation did not finish before the timeout
- * APR_INCOMPLETE     The operation was incomplete although some processing
- *                    was performed and the results are partially valid
- * APR_BADCH          Getopt found an option not in the option string
- * APR_BADARG         Getopt found an option that is missing an argument 
- *                    and an argument was specified in the option string
- * APR_EOF            APR has encountered the end of the file
- * APR_NOTFOUND       APR was unable to find the socket in the poll structure
- * APR_ANONYMOUS      APR is using anonymous shared memory
- * APR_FILEBASED      APR is using a file name as the key to the shared memory
- * APR_KEYBASED       APR is using a shared key as the key to the shared memory
- * APR_EINIT          Ininitalizer value.  If no option has been found, but 
- *                    the status variable requires a value, this should be used
- * APR_ENOTIMPL       The APR function has not been implemented on this 
- *                    platform, either because nobody has gotten to it yet, 
- *                    or the function is impossible on this platform.
- * APR_EMISMATCH      Two passwords do not match.
- * APR_EABSOLUTE      The given path was absolute.
- * APR_ERELATIVE      The given path was relative.
- * APR_EINCOMPLETE    The given path was neither relative nor absolute.
- * APR_EABOVEROOT     The given path was above the root path.
- * APR_EBUSY          The given lock was busy.
- * APR_EPROC_UNKNOWN  The given process wasn't recognized by APR
- * </PRE>
- * @{
- */
-/** @see APR_STATUS_IS_ENOSTAT */
-#define APR_ENOSTAT        (APR_OS_START_ERROR + 1)
-/** @see APR_STATUS_IS_ENOPOOL */
-#define APR_ENOPOOL        (APR_OS_START_ERROR + 2)
-/* empty slot: +3 */
-/** @see APR_STATUS_IS_EBADDATE */
-#define APR_EBADDATE       (APR_OS_START_ERROR + 4)
-/** @see APR_STATUS_IS_EINVALSOCK */
-#define APR_EINVALSOCK     (APR_OS_START_ERROR + 5)
-/** @see APR_STATUS_IS_ENOPROC */
-#define APR_ENOPROC        (APR_OS_START_ERROR + 6)
-/** @see APR_STATUS_IS_ENOTIME */
-#define APR_ENOTIME        (APR_OS_START_ERROR + 7)
-/** @see APR_STATUS_IS_ENODIR */
-#define APR_ENODIR         (APR_OS_START_ERROR + 8)
-/** @see APR_STATUS_IS_ENOLOCK */
-#define APR_ENOLOCK        (APR_OS_START_ERROR + 9)
-/** @see APR_STATUS_IS_ENOPOLL */
-#define APR_ENOPOLL        (APR_OS_START_ERROR + 10)
-/** @see APR_STATUS_IS_ENOSOCKET */
-#define APR_ENOSOCKET      (APR_OS_START_ERROR + 11)
-/** @see APR_STATUS_IS_ENOTHREAD */
-#define APR_ENOTHREAD      (APR_OS_START_ERROR + 12)
-/** @see APR_STATUS_IS_ENOTHDKEY */
-#define APR_ENOTHDKEY      (APR_OS_START_ERROR + 13)
-/** @see APR_STATUS_IS_EGENERAL */
-#define APR_EGENERAL       (APR_OS_START_ERROR + 14)
-/** @see APR_STATUS_IS_ENOSHMAVAIL */
-#define APR_ENOSHMAVAIL    (APR_OS_START_ERROR + 15)
-/** @see APR_STATUS_IS_EBADIP */
-#define APR_EBADIP         (APR_OS_START_ERROR + 16)
-/** @see APR_STATUS_IS_EBADMASK */
-#define APR_EBADMASK       (APR_OS_START_ERROR + 17)
-/* empty slot: +18 */
-/** @see APR_STATUS_IS_EDSOPEN */
-#define APR_EDSOOPEN       (APR_OS_START_ERROR + 19)
-/** @see APR_STATUS_IS_EABSOLUTE */
-#define APR_EABSOLUTE      (APR_OS_START_ERROR + 20)
-/** @see APR_STATUS_IS_ERELATIVE */
-#define APR_ERELATIVE      (APR_OS_START_ERROR + 21)
-/** @see APR_STATUS_IS_EINCOMPLETE */
-#define APR_EINCOMPLETE    (APR_OS_START_ERROR + 22)
-/** @see APR_STATUS_IS_EABOVEROOT */
-#define APR_EABOVEROOT     (APR_OS_START_ERROR + 23)
-/** @see APR_STATUS_IS_EBADPATH */
-#define APR_EBADPATH       (APR_OS_START_ERROR + 24)
-/** @see APR_STATUS_IS_EPATHWILD */
-#define APR_EPATHWILD      (APR_OS_START_ERROR + 25)
-/** @see APR_STATUS_IS_ESYMNOTFOUND */
-#define APR_ESYMNOTFOUND   (APR_OS_START_ERROR + 26)
-/** @see APR_STATUS_IS_EPROC_UNKNOWN */
-#define APR_EPROC_UNKNOWN  (APR_OS_START_ERROR + 27)
-/** @see APR_STATUS_IS_ENOTENOUGHENTROPY */
-#define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28)
-/** @} */
-
-/** 
- * @defgroup APR_STATUS_IS Status Value Tests
- * @warning For any particular error condition, more than one of these tests
- *      may match. This is because platform-specific error codes may not
- *      always match the semantics of the POSIX codes these tests (and the
- *      corresponding APR error codes) are named after. A notable example
- *      are the APR_STATUS_IS_ENOENT and APR_STATUS_IS_ENOTDIR tests on
- *      Win32 platforms. The programmer should always be aware of this and
- *      adjust the order of the tests accordingly.
- * @{
- */
-/** 
- * APR was unable to perform a stat on the file 
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOSTAT(s)        ((s) == APR_ENOSTAT)
-/** 
- * APR was not provided a pool with which to allocate memory 
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOPOOL(s)        ((s) == APR_ENOPOOL)
-/** APR was given an invalid date  */
-#define APR_STATUS_IS_EBADDATE(s)       ((s) == APR_EBADDATE)
-/** APR was given an invalid socket */
-#define APR_STATUS_IS_EINVALSOCK(s)     ((s) == APR_EINVALSOCK)
-/** APR was not given a process structure */
-#define APR_STATUS_IS_ENOPROC(s)        ((s) == APR_ENOPROC)
-/** APR was not given a time structure */
-#define APR_STATUS_IS_ENOTIME(s)        ((s) == APR_ENOTIME)
-/** APR was not given a directory structure */
-#define APR_STATUS_IS_ENODIR(s)         ((s) == APR_ENODIR)
-/** APR was not given a lock structure */
-#define APR_STATUS_IS_ENOLOCK(s)        ((s) == APR_ENOLOCK)
-/** APR was not given a poll structure */
-#define APR_STATUS_IS_ENOPOLL(s)        ((s) == APR_ENOPOLL)
-/** APR was not given a socket */
-#define APR_STATUS_IS_ENOSOCKET(s)      ((s) == APR_ENOSOCKET)
-/** APR was not given a thread structure */
-#define APR_STATUS_IS_ENOTHREAD(s)      ((s) == APR_ENOTHREAD)
-/** APR was not given a thread key structure */
-#define APR_STATUS_IS_ENOTHDKEY(s)      ((s) == APR_ENOTHDKEY)
-/** Generic Error which can not be put into another spot */
-#define APR_STATUS_IS_EGENERAL(s)       ((s) == APR_EGENERAL)
-/** There is no more shared memory available */
-#define APR_STATUS_IS_ENOSHMAVAIL(s)    ((s) == APR_ENOSHMAVAIL)
-/** The specified IP address is invalid */
-#define APR_STATUS_IS_EBADIP(s)         ((s) == APR_EBADIP)
-/** The specified netmask is invalid */
-#define APR_STATUS_IS_EBADMASK(s)       ((s) == APR_EBADMASK)
-/* empty slot: +18 */
-/** 
- * APR was unable to open the dso object.  
- * For more information call apr_dso_error().
- */
-#if defined(WIN32)
-#define APR_STATUS_IS_EDSOOPEN(s)       ((s) == APR_EDSOOPEN \
-                       || APR_TO_OS_ERROR(s) == ERROR_MOD_NOT_FOUND)
-#else
-#define APR_STATUS_IS_EDSOOPEN(s)       ((s) == APR_EDSOOPEN)
-#endif
-/** The given path was absolute. */
-#define APR_STATUS_IS_EABSOLUTE(s)      ((s) == APR_EABSOLUTE)
-/** The given path was relative. */
-#define APR_STATUS_IS_ERELATIVE(s)      ((s) == APR_ERELATIVE)
-/** The given path was neither relative nor absolute. */
-#define APR_STATUS_IS_EINCOMPLETE(s)    ((s) == APR_EINCOMPLETE)
-/** The given path was above the root path. */
-#define APR_STATUS_IS_EABOVEROOT(s)     ((s) == APR_EABOVEROOT)
-/** The given path was bad. */
-#define APR_STATUS_IS_EBADPATH(s)       ((s) == APR_EBADPATH)
-/** The given path contained wildcards. */
-#define APR_STATUS_IS_EPATHWILD(s)      ((s) == APR_EPATHWILD)
-/** Could not find the requested symbol.
- * For more information call apr_dso_error().
- */
-#if defined(WIN32)
-#define APR_STATUS_IS_ESYMNOTFOUND(s)   ((s) == APR_ESYMNOTFOUND \
-                       || APR_TO_OS_ERROR(s) == ERROR_PROC_NOT_FOUND)
-#else
-#define APR_STATUS_IS_ESYMNOTFOUND(s)   ((s) == APR_ESYMNOTFOUND)
-#endif
-/** The given process was not recognized by APR. */
-#define APR_STATUS_IS_EPROC_UNKNOWN(s)  ((s) == APR_EPROC_UNKNOWN)
-
-/** APR could not gather enough entropy to continue. */
-#define APR_STATUS_IS_ENOTENOUGHENTROPY(s) ((s) == APR_ENOTENOUGHENTROPY)
-
-/** @} */
-
-/** 
- * @addtogroup APR_Error
- * @{
- */
-/** @see APR_STATUS_IS_INCHILD */
-#define APR_INCHILD        (APR_OS_START_STATUS + 1)
-/** @see APR_STATUS_IS_INPARENT */
-#define APR_INPARENT       (APR_OS_START_STATUS + 2)
-/** @see APR_STATUS_IS_DETACH */
-#define APR_DETACH         (APR_OS_START_STATUS + 3)
-/** @see APR_STATUS_IS_NOTDETACH */
-#define APR_NOTDETACH      (APR_OS_START_STATUS + 4)
-/** @see APR_STATUS_IS_CHILD_DONE */
-#define APR_CHILD_DONE     (APR_OS_START_STATUS + 5)
-/** @see APR_STATUS_IS_CHILD_NOTDONE */
-#define APR_CHILD_NOTDONE  (APR_OS_START_STATUS + 6)
-/** @see APR_STATUS_IS_TIMEUP */
-#define APR_TIMEUP         (APR_OS_START_STATUS + 7)
-/** @see APR_STATUS_IS_INCOMPLETE */
-#define APR_INCOMPLETE     (APR_OS_START_STATUS + 8)
-/* empty slot: +9 */
-/* empty slot: +10 */
-/* empty slot: +11 */
-/** @see APR_STATUS_IS_BADCH */
-#define APR_BADCH          (APR_OS_START_STATUS + 12)
-/** @see APR_STATUS_IS_BADARG */
-#define APR_BADARG         (APR_OS_START_STATUS + 13)
-/** @see APR_STATUS_IS_EOF */
-#define APR_EOF            (APR_OS_START_STATUS + 14)
-/** @see APR_STATUS_IS_NOTFOUND */
-#define APR_NOTFOUND       (APR_OS_START_STATUS + 15)
-/* empty slot: +16 */
-/* empty slot: +17 */
-/* empty slot: +18 */
-/** @see APR_STATUS_IS_ANONYMOUS */
-#define APR_ANONYMOUS      (APR_OS_START_STATUS + 19)
-/** @see APR_STATUS_IS_FILEBASED */
-#define APR_FILEBASED      (APR_OS_START_STATUS + 20)
-/** @see APR_STATUS_IS_KEYBASED */
-#define APR_KEYBASED       (APR_OS_START_STATUS + 21)
-/** @see APR_STATUS_IS_EINIT */
-#define APR_EINIT          (APR_OS_START_STATUS + 22)  
-/** @see APR_STATUS_IS_ENOTIMPL */
-#define APR_ENOTIMPL       (APR_OS_START_STATUS + 23)
-/** @see APR_STATUS_IS_EMISMATCH */
-#define APR_EMISMATCH      (APR_OS_START_STATUS + 24)
-/** @see APR_STATUS_IS_EBUSY */
-#define APR_EBUSY          (APR_OS_START_STATUS + 25)
-/** @} */
-
-/** 
- * @addtogroup APR_STATUS_IS
- * @{
- */
-/** 
- * Program is currently executing in the child 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code */
-#define APR_STATUS_IS_INCHILD(s)        ((s) == APR_INCHILD)
-/** 
- * Program is currently executing in the parent 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_INPARENT(s)       ((s) == APR_INPARENT)
-/** 
- * The thread is detached 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_DETACH(s)         ((s) == APR_DETACH)
-/** 
- * The thread is not detached 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_NOTDETACH(s)      ((s) == APR_NOTDETACH)
-/** 
- * The child has finished executing
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_CHILD_DONE(s)     ((s) == APR_CHILD_DONE)
-/** 
- * The child has not finished executing
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_CHILD_NOTDONE(s)  ((s) == APR_CHILD_NOTDONE)
-/** 
- * The operation did not finish before the timeout
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP)
-/** 
- * The operation was incomplete although some processing was performed
- * and the results are partially valid.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_INCOMPLETE(s)     ((s) == APR_INCOMPLETE)
-/* empty slot: +9 */
-/* empty slot: +10 */
-/* empty slot: +11 */
-/** 
- * Getopt found an option not in the option string
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_BADCH(s)          ((s) == APR_BADCH)
-/** 
- * Getopt found an option not in the option string and an argument was 
- * specified in the option string
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_BADARG(s)         ((s) == APR_BADARG)
-/** 
- * APR has encountered the end of the file
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EOF(s)            ((s) == APR_EOF)
-/** 
- * APR was unable to find the socket in the poll structure
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_NOTFOUND(s)       ((s) == APR_NOTFOUND)
-/* empty slot: +16 */
-/* empty slot: +17 */
-/* empty slot: +18 */
-/** 
- * APR is using anonymous shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ANONYMOUS(s)      ((s) == APR_ANONYMOUS)
-/** 
- * APR is using a file name as the key to the shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_FILEBASED(s)      ((s) == APR_FILEBASED)
-/** 
- * APR is using a shared key as the key to the shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_KEYBASED(s)       ((s) == APR_KEYBASED)
-/** 
- * Ininitalizer value.  If no option has been found, but 
- * the status variable requires a value, this should be used
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EINIT(s)          ((s) == APR_EINIT)
-/** 
- * The APR function has not been implemented on this 
- * platform, either because nobody has gotten to it yet, 
- * or the function is impossible on this platform.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOTIMPL(s)       ((s) == APR_ENOTIMPL)
-/** 
- * Two passwords do not match.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EMISMATCH(s)      ((s) == APR_EMISMATCH)
-/** 
- * The given lock was busy
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EBUSY(s)          ((s) == APR_EBUSY)
-
-/** @} */
-
-/** 
- * @addtogroup APR_Error APR Error Values
- * @{
- */
-/* APR CANONICAL ERROR VALUES */
-/** @see APR_STATUS_IS_EACCES */
-#ifdef EACCES
-#define APR_EACCES EACCES
-#else
-#define APR_EACCES         (APR_OS_START_CANONERR + 1)
-#endif
-
-/** @see APR_STATUS_IS_EXIST */
-#ifdef EEXIST
-#define APR_EEXIST EEXIST
-#else
-#define APR_EEXIST         (APR_OS_START_CANONERR + 2)
-#endif
-
-/** @see APR_STATUS_IS_ENAMETOOLONG */
-#ifdef ENAMETOOLONG
-#define APR_ENAMETOOLONG ENAMETOOLONG
-#else
-#define APR_ENAMETOOLONG   (APR_OS_START_CANONERR + 3)
-#endif
-
-/** @see APR_STATUS_IS_ENOENT */
-#ifdef ENOENT
-#define APR_ENOENT ENOENT
-#else
-#define APR_ENOENT         (APR_OS_START_CANONERR + 4)
-#endif
-
-/** @see APR_STATUS_IS_ENOTDIR */
-#ifdef ENOTDIR
-#define APR_ENOTDIR ENOTDIR
-#else
-#define APR_ENOTDIR        (APR_OS_START_CANONERR + 5)
-#endif
-
-/** @see APR_STATUS_IS_ENOSPC */
-#ifdef ENOSPC
-#define APR_ENOSPC ENOSPC
-#else
-#define APR_ENOSPC         (APR_OS_START_CANONERR + 6)
-#endif
-
-/** @see APR_STATUS_IS_ENOMEM */
-#ifdef ENOMEM
-#define APR_ENOMEM ENOMEM
-#else
-#define APR_ENOMEM         (APR_OS_START_CANONERR + 7)
-#endif
-
-/** @see APR_STATUS_IS_EMFILE */
-#ifdef EMFILE
-#define APR_EMFILE EMFILE
-#else
-#define APR_EMFILE         (APR_OS_START_CANONERR + 8)
-#endif
-
-/** @see APR_STATUS_IS_ENFILE */
-#ifdef ENFILE
-#define APR_ENFILE ENFILE
-#else
-#define APR_ENFILE         (APR_OS_START_CANONERR + 9)
-#endif
-
-/** @see APR_STATUS_IS_EBADF */
-#ifdef EBADF
-#define APR_EBADF EBADF
-#else
-#define APR_EBADF          (APR_OS_START_CANONERR + 10)
-#endif
-
-/** @see APR_STATUS_IS_EINVAL */
-#ifdef EINVAL
-#define APR_EINVAL EINVAL
-#else
-#define APR_EINVAL         (APR_OS_START_CANONERR + 11)
-#endif
-
-/** @see APR_STATUS_IS_ESPIPE */
-#ifdef ESPIPE
-#define APR_ESPIPE ESPIPE
-#else
-#define APR_ESPIPE         (APR_OS_START_CANONERR + 12)
-#endif
-
-/** 
- * @see APR_STATUS_IS_EAGAIN 
- * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value
- */
-#ifdef EAGAIN
-#define APR_EAGAIN EAGAIN
-#elif defined(EWOULDBLOCK)
-#define APR_EAGAIN EWOULDBLOCK
-#else
-#define APR_EAGAIN         (APR_OS_START_CANONERR + 13)
-#endif
-
-/** @see APR_STATUS_IS_EINTR */
-#ifdef EINTR
-#define APR_EINTR EINTR
-#else
-#define APR_EINTR          (APR_OS_START_CANONERR + 14)
-#endif
-
-/** @see APR_STATUS_IS_ENOTSOCK */
-#ifdef ENOTSOCK
-#define APR_ENOTSOCK ENOTSOCK
-#else
-#define APR_ENOTSOCK       (APR_OS_START_CANONERR + 15)
-#endif
-
-/** @see APR_STATUS_IS_ECONNREFUSED */
-#ifdef ECONNREFUSED
-#define APR_ECONNREFUSED ECONNREFUSED
-#else
-#define APR_ECONNREFUSED   (APR_OS_START_CANONERR + 16)
-#endif
-
-/** @see APR_STATUS_IS_EINPROGRESS */
-#ifdef EINPROGRESS
-#define APR_EINPROGRESS EINPROGRESS
-#else
-#define APR_EINPROGRESS    (APR_OS_START_CANONERR + 17)
-#endif
-
-/** 
- * @see APR_STATUS_IS_ECONNABORTED
- * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value
- */
-
-#ifdef ECONNABORTED
-#define APR_ECONNABORTED ECONNABORTED
-#else
-#define APR_ECONNABORTED   (APR_OS_START_CANONERR + 18)
-#endif
-
-/** @see APR_STATUS_IS_ECONNRESET */
-#ifdef ECONNRESET
-#define APR_ECONNRESET ECONNRESET
-#else
-#define APR_ECONNRESET     (APR_OS_START_CANONERR + 19)
-#endif
-
-/** @see APR_STATUS_IS_ETIMEDOUT 
- *  @deprecated */
-#ifdef ETIMEDOUT
-#define APR_ETIMEDOUT ETIMEDOUT
-#else
-#define APR_ETIMEDOUT      (APR_OS_START_CANONERR + 20)
-#endif
-
-/** @see APR_STATUS_IS_EHOSTUNREACH */
-#ifdef EHOSTUNREACH
-#define APR_EHOSTUNREACH EHOSTUNREACH
-#else
-#define APR_EHOSTUNREACH   (APR_OS_START_CANONERR + 21)
-#endif
-
-/** @see APR_STATUS_IS_ENETUNREACH */
-#ifdef ENETUNREACH
-#define APR_ENETUNREACH ENETUNREACH
-#else
-#define APR_ENETUNREACH    (APR_OS_START_CANONERR + 22)
-#endif
-
-/** @see APR_STATUS_IS_EFTYPE */
-#ifdef EFTYPE
-#define APR_EFTYPE EFTYPE
-#else
-#define APR_EFTYPE        (APR_OS_START_CANONERR + 23)
-#endif
-
-/** @see APR_STATUS_IS_EPIPE */
-#ifdef EPIPE
-#define APR_EPIPE EPIPE
-#else
-#define APR_EPIPE         (APR_OS_START_CANONERR + 24)
-#endif
-
-/** @see APR_STATUS_IS_EXDEV */
-#ifdef EXDEV
-#define APR_EXDEV EXDEV
-#else
-#define APR_EXDEV         (APR_OS_START_CANONERR + 25)
-#endif
-
-/** @see APR_STATUS_IS_ENOTEMPTY */
-#ifdef ENOTEMPTY
-#define APR_ENOTEMPTY ENOTEMPTY
-#else
-#define APR_ENOTEMPTY     (APR_OS_START_CANONERR + 26)
-#endif
-
-/** @} */
-
-#if defined(OS2) && !defined(DOXYGEN)
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define INCL_DOSERRORS
-#define INCL_DOS
-
-/* Leave these undefined.
- * OS2 doesn't rely on the errno concept.
- * The API calls always return a result codes which
- * should be filtered through APR_FROM_OS_ERROR().
- *
- * #define apr_get_os_error()   (APR_FROM_OS_ERROR(GetLastError()))
- * #define apr_set_os_error(e)  (SetLastError(APR_TO_OS_ERROR(e)))
- */
-
-/* A special case, only socket calls require this;
- */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(errno))
-#define apr_set_netos_error(e)  (errno = APR_TO_OS_ERROR(e))
-
-/* And this needs to be greped away for good:
- */
-#define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e))
-
-/* These can't sit in a private header, so in spite of the extra size, 
- * they need to be made available here.
- */
-#define SOCBASEERR              10000
-#define SOCEPERM                (SOCBASEERR+1)             /* Not owner */
-#define SOCESRCH                (SOCBASEERR+3)             /* No such process */
-#define SOCEINTR                (SOCBASEERR+4)             /* Interrupted system call */
-#define SOCENXIO                (SOCBASEERR+6)             /* No such device or address */
-#define SOCEBADF                (SOCBASEERR+9)             /* Bad file number */
-#define SOCEACCES               (SOCBASEERR+13)            /* Permission denied */
-#define SOCEFAULT               (SOCBASEERR+14)            /* Bad address */
-#define SOCEINVAL               (SOCBASEERR+22)            /* Invalid argument */
-#define SOCEMFILE               (SOCBASEERR+24)            /* Too many open files */
-#define SOCEPIPE                (SOCBASEERR+32)            /* Broken pipe */
-#define SOCEOS2ERR              (SOCBASEERR+100)           /* OS/2 Error */
-#define SOCEWOULDBLOCK          (SOCBASEERR+35)            /* Operation would block */
-#define SOCEINPROGRESS          (SOCBASEERR+36)            /* Operation now in progress */
-#define SOCEALREADY             (SOCBASEERR+37)            /* Operation already in progress */
-#define SOCENOTSOCK             (SOCBASEERR+38)            /* Socket operation on non-socket */
-#define SOCEDESTADDRREQ         (SOCBASEERR+39)            /* Destination address required */
-#define SOCEMSGSIZE             (SOCBASEERR+40)            /* Message too long */
-#define SOCEPROTOTYPE           (SOCBASEERR+41)            /* Protocol wrong type for socket */
-#define SOCENOPROTOOPT          (SOCBASEERR+42)            /* Protocol not available */
-#define SOCEPROTONOSUPPORT      (SOCBASEERR+43)            /* Protocol not supported */
-#define SOCESOCKTNOSUPPORT      (SOCBASEERR+44)            /* Socket type not supported */
-#define SOCEOPNOTSUPP           (SOCBASEERR+45)            /* Operation not supported on socket */
-#define SOCEPFNOSUPPORT         (SOCBASEERR+46)            /* Protocol family not supported */
-#define SOCEAFNOSUPPORT         (SOCBASEERR+47)            /* Address family not supported by protocol family */
-#define SOCEADDRINUSE           (SOCBASEERR+48)            /* Address already in use */
-#define SOCEADDRNOTAVAIL        (SOCBASEERR+49)            /* Can't assign requested address */
-#define SOCENETDOWN             (SOCBASEERR+50)            /* Network is down */
-#define SOCENETUNREACH          (SOCBASEERR+51)            /* Network is unreachable */
-#define SOCENETRESET            (SOCBASEERR+52)            /* Network dropped connection on reset */
-#define SOCECONNABORTED         (SOCBASEERR+53)            /* Software caused connection abort */
-#define SOCECONNRESET           (SOCBASEERR+54)            /* Connection reset by peer */
-#define SOCENOBUFS              (SOCBASEERR+55)            /* No buffer space available */
-#define SOCEISCONN              (SOCBASEERR+56)            /* Socket is already connected */
-#define SOCENOTCONN             (SOCBASEERR+57)            /* Socket is not connected */
-#define SOCESHUTDOWN            (SOCBASEERR+58)            /* Can't send after socket shutdown */
-#define SOCETOOMANYREFS         (SOCBASEERR+59)            /* Too many references: can't splice */
-#define SOCETIMEDOUT            (SOCBASEERR+60)            /* Connection timed out */
-#define SOCECONNREFUSED         (SOCBASEERR+61)            /* Connection refused */
-#define SOCELOOP                (SOCBASEERR+62)            /* Too many levels of symbolic links */
-#define SOCENAMETOOLONG         (SOCBASEERR+63)            /* File name too long */
-#define SOCEHOSTDOWN            (SOCBASEERR+64)            /* Host is down */
-#define SOCEHOSTUNREACH         (SOCBASEERR+65)            /* No route to host */
-#define SOCENOTEMPTY            (SOCBASEERR+66)            /* Directory not empty */
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \
-                || (s) == APR_OS_START_SYSERR + SOCENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE \
-                || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \
-                || (s) == APR_OS_START_SYSERR + SOCEWOULDBLOCK \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + SOCEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + SOCENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + SOCECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + SOCEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + SOCECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + SOCECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + SOCENETUNREACH)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE \
-                || (s) == APR_OS_START_SYSERR + SOCEPIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED)
-
-/*
-    Sorry, too tired to wrap this up for OS2... feel free to
-    fit the following into their best matches.
-
-    { ERROR_NO_SIGNAL_SENT,     ESRCH           },
-    { SOCEALREADY,              EALREADY        },
-    { SOCEDESTADDRREQ,          EDESTADDRREQ    },
-    { SOCEMSGSIZE,              EMSGSIZE        },
-    { SOCEPROTOTYPE,            EPROTOTYPE      },
-    { SOCENOPROTOOPT,           ENOPROTOOPT     },
-    { SOCEPROTONOSUPPORT,       EPROTONOSUPPORT },
-    { SOCESOCKTNOSUPPORT,       ESOCKTNOSUPPORT },
-    { SOCEOPNOTSUPP,            EOPNOTSUPP      },
-    { SOCEPFNOSUPPORT,          EPFNOSUPPORT    },
-    { SOCEAFNOSUPPORT,          EAFNOSUPPORT    },
-    { SOCEADDRINUSE,            EADDRINUSE      },
-    { SOCEADDRNOTAVAIL,         EADDRNOTAVAIL   },
-    { SOCENETDOWN,              ENETDOWN        },
-    { SOCENETRESET,             ENETRESET       },
-    { SOCENOBUFS,               ENOBUFS         },
-    { SOCEISCONN,               EISCONN         },
-    { SOCENOTCONN,              ENOTCONN        },
-    { SOCESHUTDOWN,             ESHUTDOWN       },
-    { SOCETOOMANYREFS,          ETOOMANYREFS    },
-    { SOCELOOP,                 ELOOP           },
-    { SOCEHOSTDOWN,             EHOSTDOWN       },
-    { SOCENOTEMPTY,             ENOTEMPTY       },
-    { SOCEPIPE,                 EPIPE           }
-*/
-
-#elif defined(WIN32) && !defined(DOXYGEN) /* !defined(OS2) */
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define apr_get_os_error()   (APR_FROM_OS_ERROR(GetLastError()))
-#define apr_set_os_error(e)  (SetLastError(APR_TO_OS_ERROR(e)))
-
-/* A special case, only socket calls require this:
- */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(WSAGetLastError()))
-#define apr_set_netos_error(e)   (WSASetLastError(APR_TO_OS_ERROR(e)))
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_CANNOT_MAKE \
-                || (s) == APR_OS_START_SYSERR + ERROR_CURRENT_DIRECTORY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DRIVE_LOCKED \
-                || (s) == APR_OS_START_SYSERR + ERROR_FAIL_I24 \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_LOCKED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NETWORK_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \
-                || (s) == APR_OS_START_SYSERR + WSAENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_NETPATH \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_NET_NAME \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_PATHNAME \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DRIVE)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM \
-                || (s) == APR_OS_START_SYSERR + ERROR_ARENA_TRASHED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_MEMORY \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_BLOCK \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_QUOTA \
-                || (s) == APR_OS_START_SYSERR + ERROR_OUTOFMEMORY)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE \
-                || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_TARGET_HANDLE)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_ACCESS \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DATA \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_SEEK_ON_DEVICE \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_PROC_SLOTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_NESTING_NOT_ALLOWED \
-                || (s) == APR_OS_START_SYSERR + ERROR_MAX_THRDS_REACHED \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \
-                || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + WSAEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + WSAENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + ERROR_NETNAME_DELETED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAENETUNREACH)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_EXE_MACHINE_TYPE_MISMATCH \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DLL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_MODULETYPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_EXE_FORMAT \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_EXE_SIGNATURE \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_CORRUPT \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_FORMAT)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY)
-
-#elif defined(NETWARE) && defined(USE_WINSOCK) && !defined(DOXYGEN) /* !defined(OS2) && !defined(WIN32) */
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define apr_get_os_error()    (errno)
-#define apr_set_os_error(e)   (errno = (e))
-
-/* A special case, only socket calls require this: */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(WSAGetLastError()))
-#define apr_set_netos_error(e)  (WSASetLastError(APR_TO_OS_ERROR(e)))
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE)
-
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) ==                       EWOULDBLOCK \
-                || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + WSAEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + WSAENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + WSAECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)       ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAENETUNREACH)
-#define APR_STATUS_IS_ENETDOWN(s)       ((s) == APR_OS_START_SYSERR + WSAENETDOWN)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY)
-
-#else /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */
-
-/*
- *  os error codes are clib error codes
- */
-#define APR_FROM_OS_ERROR(e)  (e)
-#define APR_TO_OS_ERROR(e)    (e)
-
-#define apr_get_os_error()    (errno)
-#define apr_set_os_error(e)   (errno = (e))
-
-/* A special case, only socket calls require this:
- */
-#define apr_get_netos_error() (errno)
-#define apr_set_netos_error(e) (errno = (e))
-
-/** 
- * @addtogroup APR_STATUS_IS
- * @{
- */
-
-/** permission denied */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES)
-/** file exists */
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST)
-/** path name is too long */
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG)
-/**
- * no such file or directory
- * @remark
- * EMVSCATLG can be returned by the automounter on z/OS for
- * paths which do not exist.
- */
-#ifdef EMVSCATLG
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                                      || (s) == EMVSCATLG)
-#else
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT)
-#endif
-/** not a directory */
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-/** no space left on device */
-#ifdef EDQUOT
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                                      || (s) == EDQUOT)
-#else
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC)
-#endif
-/** not enough memory */
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-/** too many open files */
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE)
-/** file table overflow */
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-/** bad file # */
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF)
-/** invalid argument */
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL)
-/** illegal seek */
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE)
-
-/** operation would block */
-#if !defined(EWOULDBLOCK) || !defined(EAGAIN)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN)
-#elif (EWOULDBLOCK == EAGAIN)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN)
-#else
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                                      || (s) == EWOULDBLOCK)
-#endif
-
-/** interrupted system call */
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR)
-/** socket operation on a non-socket */
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK)
-/** Connection Refused */
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED)
-/** operation now in progress */
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS)
-
-/** 
- * Software caused connection abort 
- * @remark
- * EPROTO on certain older kernels really means ECONNABORTED, so we need to 
- * ignore it for them.  See discussion in new-httpd archives nh.9701 & nh.9603
- *
- * There is potentially a bug in Solaris 2.x x<6, and other boxes that 
- * implement tcp sockets in userland (i.e. on top of STREAMS).  On these
- * systems, EPROTO can actually result in a fatal loop.  See PR#981 for 
- * example.  It's hard to handle both uses of EPROTO.
- */
-#ifdef EPROTO
-#define APR_STATUS_IS_ECONNABORTED(s)    ((s) == APR_ECONNABORTED \
-                                       || (s) == EPROTO)
-#else
-#define APR_STATUS_IS_ECONNABORTED(s)    ((s) == APR_ECONNABORTED)
-#endif
-
-/** Connection Reset by peer */
-#define APR_STATUS_IS_ECONNRESET(s)      ((s) == APR_ECONNRESET)
-/** Operation timed out
- *  @deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)      ((s) == APR_ETIMEDOUT)
-/** no route to host */
-#define APR_STATUS_IS_EHOSTUNREACH(s)    ((s) == APR_EHOSTUNREACH)
-/** network is unreachable */
-#define APR_STATUS_IS_ENETUNREACH(s)     ((s) == APR_ENETUNREACH)
-/** inappropiate file type or format */
-#define APR_STATUS_IS_EFTYPE(s)          ((s) == APR_EFTYPE)
-/** broken pipe */
-#define APR_STATUS_IS_EPIPE(s)           ((s) == APR_EPIPE)
-/** cross device link */
-#define APR_STATUS_IS_EXDEV(s)           ((s) == APR_EXDEV)
-/** Directory Not Empty */
-#define APR_STATUS_IS_ENOTEMPTY(s)       ((s) == APR_ENOTEMPTY || \
-                                          (s) == APR_EEXIST)
-/** @} */
-
-#endif /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */
-
-/** @} */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  /* ! APR_ERRNO_H */
-#ifndef _LINUX_ERRNO_H
-#define _LINUX_ERRNO_H
-
-#include <asm/errno.h>
-
-#ifdef __KERNEL__
-
-/* Should never be seen by user programs */
-#define ERESTARTSYS	512
-#define ERESTARTNOINTR	513
-#define ERESTARTNOHAND	514	/* restart if no handler.. */
-#define ENOIOCTLCMD	515	/* No ioctl command */
-#define ERESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */
-
-/* Defined for the NFSv3 protocol */
-#define EBADHANDLE	521	/* Illegal NFS file handle */
-#define ENOTSYNC	522	/* Update synchronization mismatch */
-#define EBADCOOKIE	523	/* Cookie is stale */
-#define ENOTSUPP	524	/* Operation is not supported */
-#define ETOOSMALL	525	/* Buffer or request is too small */
-#define ESERVERFAULT	526	/* An untranslatable error occurred */
-#define EBADTYPE	527	/* Type not supported by server */
-#define EJUKEBOX	528	/* Request initiated, but will not complete before timeout */
-#define EIOCBQUEUED	529	/* iocb queued, will get completion event */
-#define EIOCBRETRY	530	/* iocb queued, will trigger a retry */
-
-#endif
-
-#endif
-// Copyright (c) 1994 James Clark
-// See the file COPYING for copying permission.
-
-#ifndef ErrnoMessageArg_INCLUDED
-#define ErrnoMessageArg_INCLUDED 1
-
-#include "MessageArg.h"
-#include "rtti.h"
-
-#ifdef SP_NAMESPACE
-namespace SP_NAMESPACE {
-#endif
-
-class SP_API ErrnoMessageArg : public OtherMessageArg {
-  RTTI_CLASS
-public:
-  ErrnoMessageArg(int errnum) : errno_(errnum) { }
-  MessageArg *copy() const;
-  // errno might be a macro so we must use a different name
-  int errnum() const;
-private:
-  int errno_;
-};
-
-inline
-int ErrnoMessageArg::errnum() const
-{
-  return errno_;
-}
-
-#ifdef SP_NAMESPACE
-}
-#endif
-
-#endif /* not ErrnoMessageArg_INCLUDED */
-/* Copyright (C) 1991,92,93,94,95,96,97,2002 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-/*
- *	ISO C99 Standard: 7.5 Errors	<errno.h>
- */
-
-#ifndef	_ERRNO_H
-
-/* The includer defined __need_Emath if he wants only the definitions
-   of EDOM and ERANGE, and not everything else.  */
-#ifndef	__need_Emath
-# define _ERRNO_H	1
-# include <features.h>
-#endif
-
-__BEGIN_DECLS
-
-/* Get the error number constants from the system-specific file.
-   This file will test __need_Emath and _ERRNO_H.  */
-#include <bits/errno.h>
-#undef	__need_Emath
-
-#ifdef	_ERRNO_H
-
-/* Declare the `errno' variable, unless it's defined as a macro by
-   bits/errno.h.  This is the case in GNU, where it is a per-thread
-   variable.  This redeclaration using the macro still works, but it
-   will be a function declaration without a prototype and may trigger
-   a -Wstrict-prototypes warning.  */
-#ifndef	errno
-extern int errno;
-#endif
-
-#ifdef __USE_GNU
-
-/* The full and simple forms of the name with which the program was
-   invoked.  These variables are set up automatically at startup based on
-   the value of ARGV[0] (this works only if you use GNU ld).  */
-extern char *program_invocation_name, *program_invocation_short_name;
-#endif /* __USE_GNU */
-#endif /* _ERRNO_H */
-
-__END_DECLS
-
-#endif /* _ERRNO_H */
-
-/* The Hurd <bits/errno.h> defines `error_t' as an enumerated type so
-   that printing `error_t' values in the debugger shows the names.  We
-   might need this definition sometimes even if this file was included
-   before.  */
-#if defined __USE_GNU || defined __need_error_t
-# ifndef __error_t_defined
-typedef int error_t;
-#  define __error_t_defined	1
-# endif
-# undef __need_error_t
-#endif
-#ifndef _I386_ERRNO_H
-#define _I386_ERRNO_H
-
-#include <asm-generic/errno.h>
-
-#endif
-#ifndef _ASM_GENERIC_ERRNO_BASE_H
-#define _ASM_GENERIC_ERRNO_BASE_H
-
-#define	EPERM		 1	/* Operation not permitted */
-#define	ENOENT		 2	/* No such file or directory */
-#define	ESRCH		 3	/* No such process */
-#define	EINTR		 4	/* Interrupted system call */
-#define	EIO		 5	/* I/O error */
-#define	ENXIO		 6	/* No such device or address */
-#define	E2BIG		 7	/* Argument list too long */
-#define	ENOEXEC		 8	/* Exec format error */
-#define	EBADF		 9	/* Bad file number */
-#define	ECHILD		10	/* No child processes */
-#define	EAGAIN		11	/* Try again */
-#define	ENOMEM		12	/* Out of memory */
-#define	EACCES		13	/* Permission denied */
-#define	EFAULT		14	/* Bad address */
-#define	ENOTBLK		15	/* Block device required */
-#define	EBUSY		16	/* Device or resource busy */
-#define	EEXIST		17	/* File exists */
-#define	EXDEV		18	/* Cross-device link */
-#define	ENODEV		19	/* No such device */
-#define	ENOTDIR		20	/* Not a directory */
-#define	EISDIR		21	/* Is a directory */
-#define	EINVAL		22	/* Invalid argument */
-#define	ENFILE		23	/* File table overflow */
-#define	EMFILE		24	/* Too many open files */
-#define	ENOTTY		25	/* Not a typewriter */
-#define	ETXTBSY		26	/* Text file busy */
-#define	EFBIG		27	/* File too large */
-#define	ENOSPC		28	/* No space left on device */
-#define	ESPIPE		29	/* Illegal seek */
-#define	EROFS		30	/* Read-only file system */
-#define	EMLINK		31	/* Too many links */
-#define	EPIPE		32	/* Broken pipe */
-#define	EDOM		33	/* Math argument out of domain of func */
-#define	ERANGE		34	/* Math result not representable */
-
-#endif
-#ifndef _ASM_GENERIC_ERRNO_H
-#define _ASM_GENERIC_ERRNO_H
-
-#include <asm-generic/errno-base.h>
-
-#define	EDEADLK		35	/* Resource deadlock would occur */
-#define	ENAMETOOLONG	36	/* File name too long */
-#define	ENOLCK		37	/* No record locks available */
-#define	ENOSYS		38	/* Function not implemented */
-#define	ENOTEMPTY	39	/* Directory not empty */
-#define	ELOOP		40	/* Too many symbolic links encountered */
-#define	EWOULDBLOCK	EAGAIN	/* Operation would block */
-#define	ENOMSG		42	/* No message of desired type */
-#define	EIDRM		43	/* Identifier removed */
-#define	ECHRNG		44	/* Channel number out of range */
-#define	EL2NSYNC	45	/* Level 2 not synchronized */
-#define	EL3HLT		46	/* Level 3 halted */
-#define	EL3RST		47	/* Level 3 reset */
-#define	ELNRNG		48	/* Link number out of range */
-#define	EUNATCH		49	/* Protocol driver not attached */
-#define	ENOCSI		50	/* No CSI structure available */
-#define	EL2HLT		51	/* Level 2 halted */
-#define	EBADE		52	/* Invalid exchange */
-#define	EBADR		53	/* Invalid request descriptor */
-#define	EXFULL		54	/* Exchange full */
-#define	ENOANO		55	/* No anode */
-#define	EBADRQC		56	/* Invalid request code */
-#define	EBADSLT		57	/* Invalid slot */
-
-#define	EDEADLOCK	EDEADLK
-
-#define	EBFONT		59	/* Bad font file format */
-#define	ENOSTR		60	/* Device not a stream */
-#define	ENODATA		61	/* No data available */
-#define	ETIME		62	/* Timer expired */
-#define	ENOSR		63	/* Out of streams resources */
-#define	ENONET		64	/* Machine is not on the network */
-#define	ENOPKG		65	/* Package not installed */
-#define	EREMOTE		66	/* Object is remote */
-#define	ENOLINK		67	/* Link has been severed */
-#define	EADV		68	/* Advertise error */
-#define	ESRMNT		69	/* Srmount error */
-#define	ECOMM		70	/* Communication error on send */
-#define	EPROTO		71	/* Protocol error */
-#define	EMULTIHOP	72	/* Multihop attempted */
-#define	EDOTDOT		73	/* RFS specific error */
-#define	EBADMSG		74	/* Not a data message */
-#define	EOVERFLOW	75	/* Value too large for defined data type */
-#define	ENOTUNIQ	76	/* Name not unique on network */
-#define	EBADFD		77	/* File descriptor in bad state */
-#define	EREMCHG		78	/* Remote address changed */
-#define	ELIBACC		79	/* Can not access a needed shared library */
-#define	ELIBBAD		80	/* Accessing a corrupted shared library */
-#define	ELIBSCN		81	/* .lib section in a.out corrupted */
-#define	ELIBMAX		82	/* Attempting to link in too many shared libraries */
-#define	ELIBEXEC	83	/* Cannot exec a shared library directly */
-#define	EILSEQ		84	/* Illegal byte sequence */
-#define	ERESTART	85	/* Interrupted system call should be restarted */
-#define	ESTRPIPE	86	/* Streams pipe error */
-#define	EUSERS		87	/* Too many users */
-#define	ENOTSOCK	88	/* Socket operation on non-socket */
-#define	EDESTADDRREQ	89	/* Destination address required */
-#define	EMSGSIZE	90	/* Message too long */
-#define	EPROTOTYPE	91	/* Protocol wrong type for socket */
-#define	ENOPROTOOPT	92	/* Protocol not available */
-#define	EPROTONOSUPPORT	93	/* Protocol not supported */
-#define	ESOCKTNOSUPPORT	94	/* Socket type not supported */
-#define	EOPNOTSUPP	95	/* Operation not supported on transport endpoint */
-#define	EPFNOSUPPORT	96	/* Protocol family not supported */
-#define	EAFNOSUPPORT	97	/* Address family not supported by protocol */
-#define	EADDRINUSE	98	/* Address already in use */
-#define	EADDRNOTAVAIL	99	/* Cannot assign requested address */
-#define	ENETDOWN	100	/* Network is down */
-#define	ENETUNREACH	101	/* Network is unreachable */
-#define	ENETRESET	102	/* Network dropped connection because of reset */
-#define	ECONNABORTED	103	/* Software caused connection abort */
-#define	ECONNRESET	104	/* Connection reset by peer */
-#define	ENOBUFS		105	/* No buffer space available */
-#define	EISCONN		106	/* Transport endpoint is already connected */
-#define	ENOTCONN	107	/* Transport endpoint is not connected */
-#define	ESHUTDOWN	108	/* Cannot send after transport endpoint shutdown */
-#define	ETOOMANYREFS	109	/* Too many references: cannot splice */
-#define	ETIMEDOUT	110	/* Connection timed out */
-#define	ECONNREFUSED	111	/* Connection refused */
-#define	EHOSTDOWN	112	/* Host is down */
-#define	EHOSTUNREACH	113	/* No route to host */
-#define	EALREADY	114	/* Operation already in progress */
-#define	EINPROGRESS	115	/* Operation now in progress */
-#define	ESTALE		116	/* Stale NFS file handle */
-#define	EUCLEAN		117	/* Structure needs cleaning */
-#define	ENOTNAM		118	/* Not a XENIX named type file */
-#define	ENAVAIL		119	/* No XENIX semaphores available */
-#define	EISNAM		120	/* Is a named type file */
-#define	EREMOTEIO	121	/* Remote I/O error */
-#define	EDQUOT		122	/* Quota exceeded */
-
-#define	ENOMEDIUM	123	/* No medium found */
-#define	EMEDIUMTYPE	124	/* Wrong medium type */
-#define	ECANCELED	125	/* Operation Canceled */
-#define	ENOKEY		126	/* Required key not available */
-#define	EKEYEXPIRED	127	/* Key has expired */
-#define	EKEYREVOKED	128	/* Key has been revoked */
-#define	EKEYREJECTED	129	/* Key was rejected by service */
-
-/* for robust mutexes */
-#define	EOWNERDEAD	130	/* Owner died */
-#define	ENOTRECOVERABLE	131	/* State not recoverable */
-
-#endif
diff --git a/doc/errno.list.macosx.txt b/doc/errno.list.macosx.txt
deleted file mode 100644
index 728753ac793..00000000000
--- a/doc/errno.list.macosx.txt
+++ /dev/null
@@ -1,1513 +0,0 @@
-/* Copyright 2000-2005 The Apache Software Foundation or its licensors, as
- * applicable.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef APR_ERRNO_H
-#define APR_ERRNO_H
-
-/**
- * @file apr_errno.h
- * @brief APR Error Codes
- */
-
-#include "apr.h"
-
-#if APR_HAVE_ERRNO_H
-#include <errno.h>
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-
-/**
- * @defgroup apr_errno Error Codes
- * @ingroup APR 
- * @{
- */
-
-/**
- * Type for specifying an error or status code.
- */
-typedef int apr_status_t;
-
-/**
- * Return a human readable string describing the specified error.
- * @param statcode The error code the get a string for.
- * @param buf A buffer to hold the error string.
- * @param bufsize Size of the buffer to hold the string.
- */
-APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, 
-                                 apr_size_t bufsize);
-
-#if defined(DOXYGEN)
-/**
- * @def APR_FROM_OS_ERROR(os_err_type syserr)
- * Fold a platform specific error into an apr_status_t code.
- * @return apr_status_t
- * @param e The platform os error code.
- * @warning  macro implementation; the syserr argument may be evaluated
- *      multiple times.
- */
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-
-/**
- * @def APR_TO_OS_ERROR(apr_status_t statcode)
- * @return os_err_type
- * Fold an apr_status_t code back to the native platform defined error.
- * @param e The apr_status_t folded platform os error code.
- * @warning  macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error 
- *      or APR_FROM_OS_ERROR, the results are undefined.
- */
-#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-/** @def apr_get_os_error()
- * @return apr_status_t the last platform error, folded into apr_status_t, on most platforms
- * @remark This retrieves errno, or calls a GetLastError() style function, and
- *      folds it with APR_FROM_OS_ERROR.  Some platforms (such as OS2) have no
- *      such mechanism, so this call may be unsupported.  Do NOT use this
- *      call for socket errors from socket, send, recv etc!
- */
-
-/** @def apr_set_os_error(e)
- * Reset the last platform error, unfolded from an apr_status_t, on some platforms
- * @param e The OS error folded in a prior call to APR_FROM_OS_ERROR()
- * @warning This is a macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error
- *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a SetLastError() style function, unfolding statcode
- *      with APR_TO_OS_ERROR.  Some platforms (such as OS2) have no such
- *      mechanism, so this call may be unsupported.
- */
-
-/** @def apr_get_netos_error()
- * Return the last socket error, folded into apr_status_t, on all platforms
- * @remark This retrieves errno or calls a GetLastSocketError() style function,
- *      and folds it with APR_FROM_OS_ERROR.
- */
-
-/** @def apr_set_netos_error(e)
- * Reset the last socket error, unfolded from an apr_status_t
- * @param e The socket error folded in a prior call to APR_FROM_OS_ERROR()
- * @warning This is a macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error
- *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a WSASetLastError() style function, unfolding 
- *      socketcode with APR_TO_OS_ERROR.
- */
-
-#endif /* defined(DOXYGEN) */
-
-/**
- * APR_OS_START_ERROR is where the APR specific error values start.
- */
-#define APR_OS_START_ERROR     20000
-/**
- * APR_OS_ERRSPACE_SIZE is the maximum number of errors you can fit
- *    into one of the error/status ranges below -- except for
- *    APR_OS_START_USERERR, which see.
- */
-#define APR_OS_ERRSPACE_SIZE 50000
-/**
- * APR_OS_START_STATUS is where the APR specific status codes start.
- */
-#define APR_OS_START_STATUS    (APR_OS_START_ERROR + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_USERERR are reserved for applications that use APR that
- *     layer their own error codes along with APR's.  Note that the
- *     error immediately following this one is set ten times farther
- *     away than usual, so that users of apr have a lot of room in
- *     which to declare custom error codes.
- */
-#define APR_OS_START_USERERR    (APR_OS_START_STATUS + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_USEERR is obsolete, defined for compatibility only.
- * Use APR_OS_START_USERERR instead.
- */
-#define APR_OS_START_USEERR     APR_OS_START_USERERR
-/**
- * APR_OS_START_CANONERR is where APR versions of errno values are defined
- *     on systems which don't have the corresponding errno.
- */
-#define APR_OS_START_CANONERR  (APR_OS_START_USERERR \
-                                 + (APR_OS_ERRSPACE_SIZE * 10))
-/**
- * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into 
- *     apr_status_t values.
- */
-#define APR_OS_START_EAIERR    (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE)
-/**
- * APR_OS_START_SYSERR folds platform-specific system error values into 
- *     apr_status_t values.
- */
-#define APR_OS_START_SYSERR    (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE)
-
-/** no error. */
-#define APR_SUCCESS 0
-
-/** 
- * @defgroup APR_Error APR Error Values
- * <PRE>
- * <b>APR ERROR VALUES</b>
- * APR_ENOSTAT      APR was unable to perform a stat on the file 
- * APR_ENOPOOL      APR was not provided a pool with which to allocate memory
- * APR_EBADDATE     APR was given an invalid date 
- * APR_EINVALSOCK   APR was given an invalid socket
- * APR_ENOPROC      APR was not given a process structure
- * APR_ENOTIME      APR was not given a time structure
- * APR_ENODIR       APR was not given a directory structure
- * APR_ENOLOCK      APR was not given a lock structure
- * APR_ENOPOLL      APR was not given a poll structure
- * APR_ENOSOCKET    APR was not given a socket
- * APR_ENOTHREAD    APR was not given a thread structure
- * APR_ENOTHDKEY    APR was not given a thread key structure
- * APR_ENOSHMAVAIL  There is no more shared memory available
- * APR_EDSOOPEN     APR was unable to open the dso object.  For more 
- *                  information call apr_dso_error().
- * APR_EGENERAL     General failure (specific information not available)
- * APR_EBADIP       The specified IP address is invalid
- * APR_EBADMASK     The specified netmask is invalid
- * APR_ESYMNOTFOUND Could not find the requested symbol
- * </PRE>
- *
- * <PRE>
- * <b>APR STATUS VALUES</b>
- * APR_INCHILD        Program is currently executing in the child
- * APR_INPARENT       Program is currently executing in the parent
- * APR_DETACH         The thread is detached
- * APR_NOTDETACH      The thread is not detached
- * APR_CHILD_DONE     The child has finished executing
- * APR_CHILD_NOTDONE  The child has not finished executing
- * APR_TIMEUP         The operation did not finish before the timeout
- * APR_INCOMPLETE     The operation was incomplete although some processing
- *                    was performed and the results are partially valid
- * APR_BADCH          Getopt found an option not in the option string
- * APR_BADARG         Getopt found an option that is missing an argument 
- *                    and an argument was specified in the option string
- * APR_EOF            APR has encountered the end of the file
- * APR_NOTFOUND       APR was unable to find the socket in the poll structure
- * APR_ANONYMOUS      APR is using anonymous shared memory
- * APR_FILEBASED      APR is using a file name as the key to the shared memory
- * APR_KEYBASED       APR is using a shared key as the key to the shared memory
- * APR_EINIT          Ininitalizer value.  If no option has been found, but 
- *                    the status variable requires a value, this should be used
- * APR_ENOTIMPL       The APR function has not been implemented on this 
- *                    platform, either because nobody has gotten to it yet, 
- *                    or the function is impossible on this platform.
- * APR_EMISMATCH      Two passwords do not match.
- * APR_EABSOLUTE      The given path was absolute.
- * APR_ERELATIVE      The given path was relative.
- * APR_EINCOMPLETE    The given path was neither relative nor absolute.
- * APR_EABOVEROOT     The given path was above the root path.
- * APR_EBUSY          The given lock was busy.
- * APR_EPROC_UNKNOWN  The given process wasn't recognized by APR
- * </PRE>
- * @{
- */
-/** @see APR_STATUS_IS_ENOSTAT */
-#define APR_ENOSTAT        (APR_OS_START_ERROR + 1)
-/** @see APR_STATUS_IS_ENOPOOL */
-#define APR_ENOPOOL        (APR_OS_START_ERROR + 2)
-/* empty slot: +3 */
-/** @see APR_STATUS_IS_EBADDATE */
-#define APR_EBADDATE       (APR_OS_START_ERROR + 4)
-/** @see APR_STATUS_IS_EINVALSOCK */
-#define APR_EINVALSOCK     (APR_OS_START_ERROR + 5)
-/** @see APR_STATUS_IS_ENOPROC */
-#define APR_ENOPROC        (APR_OS_START_ERROR + 6)
-/** @see APR_STATUS_IS_ENOTIME */
-#define APR_ENOTIME        (APR_OS_START_ERROR + 7)
-/** @see APR_STATUS_IS_ENODIR */
-#define APR_ENODIR         (APR_OS_START_ERROR + 8)
-/** @see APR_STATUS_IS_ENOLOCK */
-#define APR_ENOLOCK        (APR_OS_START_ERROR + 9)
-/** @see APR_STATUS_IS_ENOPOLL */
-#define APR_ENOPOLL        (APR_OS_START_ERROR + 10)
-/** @see APR_STATUS_IS_ENOSOCKET */
-#define APR_ENOSOCKET      (APR_OS_START_ERROR + 11)
-/** @see APR_STATUS_IS_ENOTHREAD */
-#define APR_ENOTHREAD      (APR_OS_START_ERROR + 12)
-/** @see APR_STATUS_IS_ENOTHDKEY */
-#define APR_ENOTHDKEY      (APR_OS_START_ERROR + 13)
-/** @see APR_STATUS_IS_EGENERAL */
-#define APR_EGENERAL       (APR_OS_START_ERROR + 14)
-/** @see APR_STATUS_IS_ENOSHMAVAIL */
-#define APR_ENOSHMAVAIL    (APR_OS_START_ERROR + 15)
-/** @see APR_STATUS_IS_EBADIP */
-#define APR_EBADIP         (APR_OS_START_ERROR + 16)
-/** @see APR_STATUS_IS_EBADMASK */
-#define APR_EBADMASK       (APR_OS_START_ERROR + 17)
-/* empty slot: +18 */
-/** @see APR_STATUS_IS_EDSOPEN */
-#define APR_EDSOOPEN       (APR_OS_START_ERROR + 19)
-/** @see APR_STATUS_IS_EABSOLUTE */
-#define APR_EABSOLUTE      (APR_OS_START_ERROR + 20)
-/** @see APR_STATUS_IS_ERELATIVE */
-#define APR_ERELATIVE      (APR_OS_START_ERROR + 21)
-/** @see APR_STATUS_IS_EINCOMPLETE */
-#define APR_EINCOMPLETE    (APR_OS_START_ERROR + 22)
-/** @see APR_STATUS_IS_EABOVEROOT */
-#define APR_EABOVEROOT     (APR_OS_START_ERROR + 23)
-/** @see APR_STATUS_IS_EBADPATH */
-#define APR_EBADPATH       (APR_OS_START_ERROR + 24)
-/** @see APR_STATUS_IS_EPATHWILD */
-#define APR_EPATHWILD      (APR_OS_START_ERROR + 25)
-/** @see APR_STATUS_IS_ESYMNOTFOUND */
-#define APR_ESYMNOTFOUND   (APR_OS_START_ERROR + 26)
-/** @see APR_STATUS_IS_EPROC_UNKNOWN */
-#define APR_EPROC_UNKNOWN  (APR_OS_START_ERROR + 27)
-/** @see APR_STATUS_IS_ENOTENOUGHENTROPY */
-#define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28)
-/** @} */
-
-/** 
- * @defgroup APR_STATUS_IS Status Value Tests
- * @warning For any particular error condition, more than one of these tests
- *      may match. This is because platform-specific error codes may not
- *      always match the semantics of the POSIX codes these tests (and the
- *      corresponding APR error codes) are named after. A notable example
- *      are the APR_STATUS_IS_ENOENT and APR_STATUS_IS_ENOTDIR tests on
- *      Win32 platforms. The programmer should always be aware of this and
- *      adjust the order of the tests accordingly.
- * @{
- */
-/** 
- * APR was unable to perform a stat on the file 
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOSTAT(s)        ((s) == APR_ENOSTAT)
-/** 
- * APR was not provided a pool with which to allocate memory 
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOPOOL(s)        ((s) == APR_ENOPOOL)
-/** APR was given an invalid date  */
-#define APR_STATUS_IS_EBADDATE(s)       ((s) == APR_EBADDATE)
-/** APR was given an invalid socket */
-#define APR_STATUS_IS_EINVALSOCK(s)     ((s) == APR_EINVALSOCK)
-/** APR was not given a process structure */
-#define APR_STATUS_IS_ENOPROC(s)        ((s) == APR_ENOPROC)
-/** APR was not given a time structure */
-#define APR_STATUS_IS_ENOTIME(s)        ((s) == APR_ENOTIME)
-/** APR was not given a directory structure */
-#define APR_STATUS_IS_ENODIR(s)         ((s) == APR_ENODIR)
-/** APR was not given a lock structure */
-#define APR_STATUS_IS_ENOLOCK(s)        ((s) == APR_ENOLOCK)
-/** APR was not given a poll structure */
-#define APR_STATUS_IS_ENOPOLL(s)        ((s) == APR_ENOPOLL)
-/** APR was not given a socket */
-#define APR_STATUS_IS_ENOSOCKET(s)      ((s) == APR_ENOSOCKET)
-/** APR was not given a thread structure */
-#define APR_STATUS_IS_ENOTHREAD(s)      ((s) == APR_ENOTHREAD)
-/** APR was not given a thread key structure */
-#define APR_STATUS_IS_ENOTHDKEY(s)      ((s) == APR_ENOTHDKEY)
-/** Generic Error which can not be put into another spot */
-#define APR_STATUS_IS_EGENERAL(s)       ((s) == APR_EGENERAL)
-/** There is no more shared memory available */
-#define APR_STATUS_IS_ENOSHMAVAIL(s)    ((s) == APR_ENOSHMAVAIL)
-/** The specified IP address is invalid */
-#define APR_STATUS_IS_EBADIP(s)         ((s) == APR_EBADIP)
-/** The specified netmask is invalid */
-#define APR_STATUS_IS_EBADMASK(s)       ((s) == APR_EBADMASK)
-/* empty slot: +18 */
-/** 
- * APR was unable to open the dso object.  
- * For more information call apr_dso_error().
- */
-#if defined(WIN32)
-#define APR_STATUS_IS_EDSOOPEN(s)       ((s) == APR_EDSOOPEN \
-                       || APR_TO_OS_ERROR(s) == ERROR_MOD_NOT_FOUND)
-#else
-#define APR_STATUS_IS_EDSOOPEN(s)       ((s) == APR_EDSOOPEN)
-#endif
-/** The given path was absolute. */
-#define APR_STATUS_IS_EABSOLUTE(s)      ((s) == APR_EABSOLUTE)
-/** The given path was relative. */
-#define APR_STATUS_IS_ERELATIVE(s)      ((s) == APR_ERELATIVE)
-/** The given path was neither relative nor absolute. */
-#define APR_STATUS_IS_EINCOMPLETE(s)    ((s) == APR_EINCOMPLETE)
-/** The given path was above the root path. */
-#define APR_STATUS_IS_EABOVEROOT(s)     ((s) == APR_EABOVEROOT)
-/** The given path was bad. */
-#define APR_STATUS_IS_EBADPATH(s)       ((s) == APR_EBADPATH)
-/** The given path contained wildcards. */
-#define APR_STATUS_IS_EPATHWILD(s)      ((s) == APR_EPATHWILD)
-/** Could not find the requested symbol.
- * For more information call apr_dso_error().
- */
-#if defined(WIN32)
-#define APR_STATUS_IS_ESYMNOTFOUND(s)   ((s) == APR_ESYMNOTFOUND \
-                       || APR_TO_OS_ERROR(s) == ERROR_PROC_NOT_FOUND)
-#else
-#define APR_STATUS_IS_ESYMNOTFOUND(s)   ((s) == APR_ESYMNOTFOUND)
-#endif
-/** The given process was not recognized by APR. */
-#define APR_STATUS_IS_EPROC_UNKNOWN(s)  ((s) == APR_EPROC_UNKNOWN)
-
-/** APR could not gather enough entropy to continue. */
-#define APR_STATUS_IS_ENOTENOUGHENTROPY(s) ((s) == APR_ENOTENOUGHENTROPY)
-
-/** @} */
-
-/** 
- * @addtogroup APR_Error
- * @{
- */
-/** @see APR_STATUS_IS_INCHILD */
-#define APR_INCHILD        (APR_OS_START_STATUS + 1)
-/** @see APR_STATUS_IS_INPARENT */
-#define APR_INPARENT       (APR_OS_START_STATUS + 2)
-/** @see APR_STATUS_IS_DETACH */
-#define APR_DETACH         (APR_OS_START_STATUS + 3)
-/** @see APR_STATUS_IS_NOTDETACH */
-#define APR_NOTDETACH      (APR_OS_START_STATUS + 4)
-/** @see APR_STATUS_IS_CHILD_DONE */
-#define APR_CHILD_DONE     (APR_OS_START_STATUS + 5)
-/** @see APR_STATUS_IS_CHILD_NOTDONE */
-#define APR_CHILD_NOTDONE  (APR_OS_START_STATUS + 6)
-/** @see APR_STATUS_IS_TIMEUP */
-#define APR_TIMEUP         (APR_OS_START_STATUS + 7)
-/** @see APR_STATUS_IS_INCOMPLETE */
-#define APR_INCOMPLETE     (APR_OS_START_STATUS + 8)
-/* empty slot: +9 */
-/* empty slot: +10 */
-/* empty slot: +11 */
-/** @see APR_STATUS_IS_BADCH */
-#define APR_BADCH          (APR_OS_START_STATUS + 12)
-/** @see APR_STATUS_IS_BADARG */
-#define APR_BADARG         (APR_OS_START_STATUS + 13)
-/** @see APR_STATUS_IS_EOF */
-#define APR_EOF            (APR_OS_START_STATUS + 14)
-/** @see APR_STATUS_IS_NOTFOUND */
-#define APR_NOTFOUND       (APR_OS_START_STATUS + 15)
-/* empty slot: +16 */
-/* empty slot: +17 */
-/* empty slot: +18 */
-/** @see APR_STATUS_IS_ANONYMOUS */
-#define APR_ANONYMOUS      (APR_OS_START_STATUS + 19)
-/** @see APR_STATUS_IS_FILEBASED */
-#define APR_FILEBASED      (APR_OS_START_STATUS + 20)
-/** @see APR_STATUS_IS_KEYBASED */
-#define APR_KEYBASED       (APR_OS_START_STATUS + 21)
-/** @see APR_STATUS_IS_EINIT */
-#define APR_EINIT          (APR_OS_START_STATUS + 22)  
-/** @see APR_STATUS_IS_ENOTIMPL */
-#define APR_ENOTIMPL       (APR_OS_START_STATUS + 23)
-/** @see APR_STATUS_IS_EMISMATCH */
-#define APR_EMISMATCH      (APR_OS_START_STATUS + 24)
-/** @see APR_STATUS_IS_EBUSY */
-#define APR_EBUSY          (APR_OS_START_STATUS + 25)
-/** @} */
-
-/** 
- * @addtogroup APR_STATUS_IS
- * @{
- */
-/** 
- * Program is currently executing in the child 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code */
-#define APR_STATUS_IS_INCHILD(s)        ((s) == APR_INCHILD)
-/** 
- * Program is currently executing in the parent 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_INPARENT(s)       ((s) == APR_INPARENT)
-/** 
- * The thread is detached 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_DETACH(s)         ((s) == APR_DETACH)
-/** 
- * The thread is not detached 
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_NOTDETACH(s)      ((s) == APR_NOTDETACH)
-/** 
- * The child has finished executing
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_CHILD_DONE(s)     ((s) == APR_CHILD_DONE)
-/** 
- * The child has not finished executing
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_CHILD_NOTDONE(s)  ((s) == APR_CHILD_NOTDONE)
-/** 
- * The operation did not finish before the timeout
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP)
-/** 
- * The operation was incomplete although some processing was performed
- * and the results are partially valid.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_INCOMPLETE(s)     ((s) == APR_INCOMPLETE)
-/* empty slot: +9 */
-/* empty slot: +10 */
-/* empty slot: +11 */
-/** 
- * Getopt found an option not in the option string
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_BADCH(s)          ((s) == APR_BADCH)
-/** 
- * Getopt found an option not in the option string and an argument was 
- * specified in the option string
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_BADARG(s)         ((s) == APR_BADARG)
-/** 
- * APR has encountered the end of the file
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EOF(s)            ((s) == APR_EOF)
-/** 
- * APR was unable to find the socket in the poll structure
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_NOTFOUND(s)       ((s) == APR_NOTFOUND)
-/* empty slot: +16 */
-/* empty slot: +17 */
-/* empty slot: +18 */
-/** 
- * APR is using anonymous shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ANONYMOUS(s)      ((s) == APR_ANONYMOUS)
-/** 
- * APR is using a file name as the key to the shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_FILEBASED(s)      ((s) == APR_FILEBASED)
-/** 
- * APR is using a shared key as the key to the shared memory
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_KEYBASED(s)       ((s) == APR_KEYBASED)
-/** 
- * Ininitalizer value.  If no option has been found, but 
- * the status variable requires a value, this should be used
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EINIT(s)          ((s) == APR_EINIT)
-/** 
- * The APR function has not been implemented on this 
- * platform, either because nobody has gotten to it yet, 
- * or the function is impossible on this platform.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_ENOTIMPL(s)       ((s) == APR_ENOTIMPL)
-/** 
- * Two passwords do not match.
- * @warning
- * always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EMISMATCH(s)      ((s) == APR_EMISMATCH)
-/** 
- * The given lock was busy
- * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
- */
-#define APR_STATUS_IS_EBUSY(s)          ((s) == APR_EBUSY)
-
-/** @} */
-
-/** 
- * @addtogroup APR_Error APR Error Values
- * @{
- */
-/* APR CANONICAL ERROR VALUES */
-/** @see APR_STATUS_IS_EACCES */
-#ifdef EACCES
-#define APR_EACCES EACCES
-#else
-#define APR_EACCES         (APR_OS_START_CANONERR + 1)
-#endif
-
-/** @see APR_STATUS_IS_EXIST */
-#ifdef EEXIST
-#define APR_EEXIST EEXIST
-#else
-#define APR_EEXIST         (APR_OS_START_CANONERR + 2)
-#endif
-
-/** @see APR_STATUS_IS_ENAMETOOLONG */
-#ifdef ENAMETOOLONG
-#define APR_ENAMETOOLONG ENAMETOOLONG
-#else
-#define APR_ENAMETOOLONG   (APR_OS_START_CANONERR + 3)
-#endif
-
-/** @see APR_STATUS_IS_ENOENT */
-#ifdef ENOENT
-#define APR_ENOENT ENOENT
-#else
-#define APR_ENOENT         (APR_OS_START_CANONERR + 4)
-#endif
-
-/** @see APR_STATUS_IS_ENOTDIR */
-#ifdef ENOTDIR
-#define APR_ENOTDIR ENOTDIR
-#else
-#define APR_ENOTDIR        (APR_OS_START_CANONERR + 5)
-#endif
-
-/** @see APR_STATUS_IS_ENOSPC */
-#ifdef ENOSPC
-#define APR_ENOSPC ENOSPC
-#else
-#define APR_ENOSPC         (APR_OS_START_CANONERR + 6)
-#endif
-
-/** @see APR_STATUS_IS_ENOMEM */
-#ifdef ENOMEM
-#define APR_ENOMEM ENOMEM
-#else
-#define APR_ENOMEM         (APR_OS_START_CANONERR + 7)
-#endif
-
-/** @see APR_STATUS_IS_EMFILE */
-#ifdef EMFILE
-#define APR_EMFILE EMFILE
-#else
-#define APR_EMFILE         (APR_OS_START_CANONERR + 8)
-#endif
-
-/** @see APR_STATUS_IS_ENFILE */
-#ifdef ENFILE
-#define APR_ENFILE ENFILE
-#else
-#define APR_ENFILE         (APR_OS_START_CANONERR + 9)
-#endif
-
-/** @see APR_STATUS_IS_EBADF */
-#ifdef EBADF
-#define APR_EBADF EBADF
-#else
-#define APR_EBADF          (APR_OS_START_CANONERR + 10)
-#endif
-
-/** @see APR_STATUS_IS_EINVAL */
-#ifdef EINVAL
-#define APR_EINVAL EINVAL
-#else
-#define APR_EINVAL         (APR_OS_START_CANONERR + 11)
-#endif
-
-/** @see APR_STATUS_IS_ESPIPE */
-#ifdef ESPIPE
-#define APR_ESPIPE ESPIPE
-#else
-#define APR_ESPIPE         (APR_OS_START_CANONERR + 12)
-#endif
-
-/** 
- * @see APR_STATUS_IS_EAGAIN 
- * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value
- */
-#ifdef EAGAIN
-#define APR_EAGAIN EAGAIN
-#elif defined(EWOULDBLOCK)
-#define APR_EAGAIN EWOULDBLOCK
-#else
-#define APR_EAGAIN         (APR_OS_START_CANONERR + 13)
-#endif
-
-/** @see APR_STATUS_IS_EINTR */
-#ifdef EINTR
-#define APR_EINTR EINTR
-#else
-#define APR_EINTR          (APR_OS_START_CANONERR + 14)
-#endif
-
-/** @see APR_STATUS_IS_ENOTSOCK */
-#ifdef ENOTSOCK
-#define APR_ENOTSOCK ENOTSOCK
-#else
-#define APR_ENOTSOCK       (APR_OS_START_CANONERR + 15)
-#endif
-
-/** @see APR_STATUS_IS_ECONNREFUSED */
-#ifdef ECONNREFUSED
-#define APR_ECONNREFUSED ECONNREFUSED
-#else
-#define APR_ECONNREFUSED   (APR_OS_START_CANONERR + 16)
-#endif
-
-/** @see APR_STATUS_IS_EINPROGRESS */
-#ifdef EINPROGRESS
-#define APR_EINPROGRESS EINPROGRESS
-#else
-#define APR_EINPROGRESS    (APR_OS_START_CANONERR + 17)
-#endif
-
-/** 
- * @see APR_STATUS_IS_ECONNABORTED
- * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value
- */
-
-#ifdef ECONNABORTED
-#define APR_ECONNABORTED ECONNABORTED
-#else
-#define APR_ECONNABORTED   (APR_OS_START_CANONERR + 18)
-#endif
-
-/** @see APR_STATUS_IS_ECONNRESET */
-#ifdef ECONNRESET
-#define APR_ECONNRESET ECONNRESET
-#else
-#define APR_ECONNRESET     (APR_OS_START_CANONERR + 19)
-#endif
-
-/** @see APR_STATUS_IS_ETIMEDOUT 
- *  @deprecated */
-#ifdef ETIMEDOUT
-#define APR_ETIMEDOUT ETIMEDOUT
-#else
-#define APR_ETIMEDOUT      (APR_OS_START_CANONERR + 20)
-#endif
-
-/** @see APR_STATUS_IS_EHOSTUNREACH */
-#ifdef EHOSTUNREACH
-#define APR_EHOSTUNREACH EHOSTUNREACH
-#else
-#define APR_EHOSTUNREACH   (APR_OS_START_CANONERR + 21)
-#endif
-
-/** @see APR_STATUS_IS_ENETUNREACH */
-#ifdef ENETUNREACH
-#define APR_ENETUNREACH ENETUNREACH
-#else
-#define APR_ENETUNREACH    (APR_OS_START_CANONERR + 22)
-#endif
-
-/** @see APR_STATUS_IS_EFTYPE */
-#ifdef EFTYPE
-#define APR_EFTYPE EFTYPE
-#else
-#define APR_EFTYPE        (APR_OS_START_CANONERR + 23)
-#endif
-
-/** @see APR_STATUS_IS_EPIPE */
-#ifdef EPIPE
-#define APR_EPIPE EPIPE
-#else
-#define APR_EPIPE         (APR_OS_START_CANONERR + 24)
-#endif
-
-/** @see APR_STATUS_IS_EXDEV */
-#ifdef EXDEV
-#define APR_EXDEV EXDEV
-#else
-#define APR_EXDEV         (APR_OS_START_CANONERR + 25)
-#endif
-
-/** @see APR_STATUS_IS_ENOTEMPTY */
-#ifdef ENOTEMPTY
-#define APR_ENOTEMPTY ENOTEMPTY
-#else
-#define APR_ENOTEMPTY     (APR_OS_START_CANONERR + 26)
-#endif
-
-/** @} */
-
-#if defined(OS2) && !defined(DOXYGEN)
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define INCL_DOSERRORS
-#define INCL_DOS
-
-/* Leave these undefined.
- * OS2 doesn't rely on the errno concept.
- * The API calls always return a result codes which
- * should be filtered through APR_FROM_OS_ERROR().
- *
- * #define apr_get_os_error()   (APR_FROM_OS_ERROR(GetLastError()))
- * #define apr_set_os_error(e)  (SetLastError(APR_TO_OS_ERROR(e)))
- */
-
-/* A special case, only socket calls require this;
- */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(errno))
-#define apr_set_netos_error(e)  (errno = APR_TO_OS_ERROR(e))
-
-/* And this needs to be greped away for good:
- */
-#define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e))
-
-/* These can't sit in a private header, so in spite of the extra size, 
- * they need to be made available here.
- */
-#define SOCBASEERR              10000
-#define SOCEPERM                (SOCBASEERR+1)             /* Not owner */
-#define SOCESRCH                (SOCBASEERR+3)             /* No such process */
-#define SOCEINTR                (SOCBASEERR+4)             /* Interrupted system call */
-#define SOCENXIO                (SOCBASEERR+6)             /* No such device or address */
-#define SOCEBADF                (SOCBASEERR+9)             /* Bad file number */
-#define SOCEACCES               (SOCBASEERR+13)            /* Permission denied */
-#define SOCEFAULT               (SOCBASEERR+14)            /* Bad address */
-#define SOCEINVAL               (SOCBASEERR+22)            /* Invalid argument */
-#define SOCEMFILE               (SOCBASEERR+24)            /* Too many open files */
-#define SOCEPIPE                (SOCBASEERR+32)            /* Broken pipe */
-#define SOCEOS2ERR              (SOCBASEERR+100)           /* OS/2 Error */
-#define SOCEWOULDBLOCK          (SOCBASEERR+35)            /* Operation would block */
-#define SOCEINPROGRESS          (SOCBASEERR+36)            /* Operation now in progress */
-#define SOCEALREADY             (SOCBASEERR+37)            /* Operation already in progress */
-#define SOCENOTSOCK             (SOCBASEERR+38)            /* Socket operation on non-socket */
-#define SOCEDESTADDRREQ         (SOCBASEERR+39)            /* Destination address required */
-#define SOCEMSGSIZE             (SOCBASEERR+40)            /* Message too long */
-#define SOCEPROTOTYPE           (SOCBASEERR+41)            /* Protocol wrong type for socket */
-#define SOCENOPROTOOPT          (SOCBASEERR+42)            /* Protocol not available */
-#define SOCEPROTONOSUPPORT      (SOCBASEERR+43)            /* Protocol not supported */
-#define SOCESOCKTNOSUPPORT      (SOCBASEERR+44)            /* Socket type not supported */
-#define SOCEOPNOTSUPP           (SOCBASEERR+45)            /* Operation not supported on socket */
-#define SOCEPFNOSUPPORT         (SOCBASEERR+46)            /* Protocol family not supported */
-#define SOCEAFNOSUPPORT         (SOCBASEERR+47)            /* Address family not supported by protocol family */
-#define SOCEADDRINUSE           (SOCBASEERR+48)            /* Address already in use */
-#define SOCEADDRNOTAVAIL        (SOCBASEERR+49)            /* Can't assign requested address */
-#define SOCENETDOWN             (SOCBASEERR+50)            /* Network is down */
-#define SOCENETUNREACH          (SOCBASEERR+51)            /* Network is unreachable */
-#define SOCENETRESET            (SOCBASEERR+52)            /* Network dropped connection on reset */
-#define SOCECONNABORTED         (SOCBASEERR+53)            /* Software caused connection abort */
-#define SOCECONNRESET           (SOCBASEERR+54)            /* Connection reset by peer */
-#define SOCENOBUFS              (SOCBASEERR+55)            /* No buffer space available */
-#define SOCEISCONN              (SOCBASEERR+56)            /* Socket is already connected */
-#define SOCENOTCONN             (SOCBASEERR+57)            /* Socket is not connected */
-#define SOCESHUTDOWN            (SOCBASEERR+58)            /* Can't send after socket shutdown */
-#define SOCETOOMANYREFS         (SOCBASEERR+59)            /* Too many references: can't splice */
-#define SOCETIMEDOUT            (SOCBASEERR+60)            /* Connection timed out */
-#define SOCECONNREFUSED         (SOCBASEERR+61)            /* Connection refused */
-#define SOCELOOP                (SOCBASEERR+62)            /* Too many levels of symbolic links */
-#define SOCENAMETOOLONG         (SOCBASEERR+63)            /* File name too long */
-#define SOCEHOSTDOWN            (SOCBASEERR+64)            /* Host is down */
-#define SOCEHOSTUNREACH         (SOCBASEERR+65)            /* No route to host */
-#define SOCENOTEMPTY            (SOCBASEERR+66)            /* Directory not empty */
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \
-                || (s) == APR_OS_START_SYSERR + SOCENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE \
-                || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \
-                || (s) == APR_OS_START_SYSERR + SOCEWOULDBLOCK \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + SOCEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + SOCENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + SOCECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + SOCEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + SOCECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + SOCECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + SOCENETUNREACH)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE \
-                || (s) == APR_OS_START_SYSERR + SOCEPIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED)
-
-/*
-    Sorry, too tired to wrap this up for OS2... feel free to
-    fit the following into their best matches.
-
-    { ERROR_NO_SIGNAL_SENT,     ESRCH           },
-    { SOCEALREADY,              EALREADY        },
-    { SOCEDESTADDRREQ,          EDESTADDRREQ    },
-    { SOCEMSGSIZE,              EMSGSIZE        },
-    { SOCEPROTOTYPE,            EPROTOTYPE      },
-    { SOCENOPROTOOPT,           ENOPROTOOPT     },
-    { SOCEPROTONOSUPPORT,       EPROTONOSUPPORT },
-    { SOCESOCKTNOSUPPORT,       ESOCKTNOSUPPORT },
-    { SOCEOPNOTSUPP,            EOPNOTSUPP      },
-    { SOCEPFNOSUPPORT,          EPFNOSUPPORT    },
-    { SOCEAFNOSUPPORT,          EAFNOSUPPORT    },
-    { SOCEADDRINUSE,            EADDRINUSE      },
-    { SOCEADDRNOTAVAIL,         EADDRNOTAVAIL   },
-    { SOCENETDOWN,              ENETDOWN        },
-    { SOCENETRESET,             ENETRESET       },
-    { SOCENOBUFS,               ENOBUFS         },
-    { SOCEISCONN,               EISCONN         },
-    { SOCENOTCONN,              ENOTCONN        },
-    { SOCESHUTDOWN,             ESHUTDOWN       },
-    { SOCETOOMANYREFS,          ETOOMANYREFS    },
-    { SOCELOOP,                 ELOOP           },
-    { SOCEHOSTDOWN,             EHOSTDOWN       },
-    { SOCENOTEMPTY,             ENOTEMPTY       },
-    { SOCEPIPE,                 EPIPE           }
-*/
-
-#elif defined(WIN32) && !defined(DOXYGEN) /* !defined(OS2) */
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define apr_get_os_error()   (APR_FROM_OS_ERROR(GetLastError()))
-#define apr_set_os_error(e)  (SetLastError(APR_TO_OS_ERROR(e)))
-
-/* A special case, only socket calls require this:
- */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(WSAGetLastError()))
-#define apr_set_netos_error(e)   (WSASetLastError(APR_TO_OS_ERROR(e)))
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES \
-                || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_CANNOT_MAKE \
-                || (s) == APR_OS_START_SYSERR + ERROR_CURRENT_DIRECTORY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DRIVE_LOCKED \
-                || (s) == APR_OS_START_SYSERR + ERROR_FAIL_I24 \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_LOCKED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NETWORK_ACCESS_DENIED \
-                || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \
-                || (s) == APR_OS_START_SYSERR + WSAENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR \
-                || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_NETPATH \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_NET_NAME \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_PATHNAME \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DRIVE)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM \
-                || (s) == APR_OS_START_SYSERR + ERROR_ARENA_TRASHED \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_MEMORY \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_BLOCK \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_QUOTA \
-                || (s) == APR_OS_START_SYSERR + ERROR_OUTOFMEMORY)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE \
-                || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_TARGET_HANDLE)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_ACCESS \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DATA \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_SEEK_ON_DEVICE \
-                || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \
-                || (s) == APR_OS_START_SYSERR + ERROR_NO_PROC_SLOTS \
-                || (s) == APR_OS_START_SYSERR + ERROR_NESTING_NOT_ALLOWED \
-                || (s) == APR_OS_START_SYSERR + ERROR_MAX_THRDS_REACHED \
-                || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \
-                || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + WSAEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + WSAENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + ERROR_NETNAME_DELETED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAENETUNREACH)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_EXE_MACHINE_TYPE_MISMATCH \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DLL \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_MODULETYPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_EXE_FORMAT \
-                || (s) == APR_OS_START_SYSERR + ERROR_INVALID_EXE_SIGNATURE \
-                || (s) == APR_OS_START_SYSERR + ERROR_FILE_CORRUPT \
-                || (s) == APR_OS_START_SYSERR + ERROR_BAD_FORMAT)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE \
-                || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV \
-                || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY \
-                || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY)
-
-#elif defined(NETWARE) && defined(USE_WINSOCK) && !defined(DOXYGEN) /* !defined(OS2) && !defined(WIN32) */
-
-#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR)
-#define APR_TO_OS_ERROR(e)   (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
-
-#define apr_get_os_error()    (errno)
-#define apr_set_os_error(e)   (errno = (e))
-
-/* A special case, only socket calls require this: */
-#define apr_get_netos_error()   (APR_FROM_OS_ERROR(WSAGetLastError()))
-#define apr_set_netos_error(e)  (WSASetLastError(APR_TO_OS_ERROR(e)))
-
-/* APR CANONICAL ERROR TESTS */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES)
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST)
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG)
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT)
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC)
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE)
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF)
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL)
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE)
-
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                || (s) ==                       EWOULDBLOCK \
-                || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK)
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR \
-                || (s) == APR_OS_START_SYSERR + WSAEINTR)
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK \
-                || (s) == APR_OS_START_SYSERR + WSAENOTSOCK)
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED)
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS \
-                || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS)
-#define APR_STATUS_IS_ECONNABORTED(s)   ((s) == APR_ECONNABORTED \
-                || (s) == APR_OS_START_SYSERR + WSAECONNABORTED)
-#define APR_STATUS_IS_ECONNRESET(s)     ((s) == APR_ECONNRESET \
-                || (s) == APR_OS_START_SYSERR + WSAECONNRESET)
-/* XXX deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)       ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#undef APR_STATUS_IS_TIMEUP
-#define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT)
-#define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH)
-#define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
-                || (s) == APR_OS_START_SYSERR + WSAENETUNREACH)
-#define APR_STATUS_IS_ENETDOWN(s)       ((s) == APR_OS_START_SYSERR + WSAENETDOWN)
-#define APR_STATUS_IS_EFTYPE(s)         ((s) == APR_EFTYPE)
-#define APR_STATUS_IS_EPIPE(s)          ((s) == APR_EPIPE)
-#define APR_STATUS_IS_EXDEV(s)          ((s) == APR_EXDEV)
-#define APR_STATUS_IS_ENOTEMPTY(s)      ((s) == APR_ENOTEMPTY)
-
-#else /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */
-
-/*
- *  os error codes are clib error codes
- */
-#define APR_FROM_OS_ERROR(e)  (e)
-#define APR_TO_OS_ERROR(e)    (e)
-
-#define apr_get_os_error()    (errno)
-#define apr_set_os_error(e)   (errno = (e))
-
-/* A special case, only socket calls require this:
- */
-#define apr_get_netos_error() (errno)
-#define apr_set_netos_error(e) (errno = (e))
-
-/** 
- * @addtogroup APR_STATUS_IS
- * @{
- */
-
-/** permission denied */
-#define APR_STATUS_IS_EACCES(s)         ((s) == APR_EACCES)
-/** file exists */
-#define APR_STATUS_IS_EEXIST(s)         ((s) == APR_EEXIST)
-/** path name is too long */
-#define APR_STATUS_IS_ENAMETOOLONG(s)   ((s) == APR_ENAMETOOLONG)
-/**
- * no such file or directory
- * @remark
- * EMVSCATLG can be returned by the automounter on z/OS for
- * paths which do not exist.
- */
-#ifdef EMVSCATLG
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT \
-                                      || (s) == EMVSCATLG)
-#else
-#define APR_STATUS_IS_ENOENT(s)         ((s) == APR_ENOENT)
-#endif
-/** not a directory */
-#define APR_STATUS_IS_ENOTDIR(s)        ((s) == APR_ENOTDIR)
-/** no space left on device */
-#ifdef EDQUOT
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC \
-                                      || (s) == EDQUOT)
-#else
-#define APR_STATUS_IS_ENOSPC(s)         ((s) == APR_ENOSPC)
-#endif
-/** not enough memory */
-#define APR_STATUS_IS_ENOMEM(s)         ((s) == APR_ENOMEM)
-/** too many open files */
-#define APR_STATUS_IS_EMFILE(s)         ((s) == APR_EMFILE)
-/** file table overflow */
-#define APR_STATUS_IS_ENFILE(s)         ((s) == APR_ENFILE)
-/** bad file # */
-#define APR_STATUS_IS_EBADF(s)          ((s) == APR_EBADF)
-/** invalid argument */
-#define APR_STATUS_IS_EINVAL(s)         ((s) == APR_EINVAL)
-/** illegal seek */
-#define APR_STATUS_IS_ESPIPE(s)         ((s) == APR_ESPIPE)
-
-/** operation would block */
-#if !defined(EWOULDBLOCK) || !defined(EAGAIN)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN)
-#elif (EWOULDBLOCK == EAGAIN)
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN)
-#else
-#define APR_STATUS_IS_EAGAIN(s)         ((s) == APR_EAGAIN \
-                                      || (s) == EWOULDBLOCK)
-#endif
-
-/** interrupted system call */
-#define APR_STATUS_IS_EINTR(s)          ((s) == APR_EINTR)
-/** socket operation on a non-socket */
-#define APR_STATUS_IS_ENOTSOCK(s)       ((s) == APR_ENOTSOCK)
-/** Connection Refused */
-#define APR_STATUS_IS_ECONNREFUSED(s)   ((s) == APR_ECONNREFUSED)
-/** operation now in progress */
-#define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS)
-
-/** 
- * Software caused connection abort 
- * @remark
- * EPROTO on certain older kernels really means ECONNABORTED, so we need to 
- * ignore it for them.  See discussion in new-httpd archives nh.9701 & nh.9603
- *
- * There is potentially a bug in Solaris 2.x x<6, and other boxes that 
- * implement tcp sockets in userland (i.e. on top of STREAMS).  On these
- * systems, EPROTO can actually result in a fatal loop.  See PR#981 for 
- * example.  It's hard to handle both uses of EPROTO.
- */
-#ifdef EPROTO
-#define APR_STATUS_IS_ECONNABORTED(s)    ((s) == APR_ECONNABORTED \
-                                       || (s) == EPROTO)
-#else
-#define APR_STATUS_IS_ECONNABORTED(s)    ((s) == APR_ECONNABORTED)
-#endif
-
-/** Connection Reset by peer */
-#define APR_STATUS_IS_ECONNRESET(s)      ((s) == APR_ECONNRESET)
-/** Operation timed out
- *  @deprecated */
-#define APR_STATUS_IS_ETIMEDOUT(s)      ((s) == APR_ETIMEDOUT)
-/** no route to host */
-#define APR_STATUS_IS_EHOSTUNREACH(s)    ((s) == APR_EHOSTUNREACH)
-/** network is unreachable */
-#define APR_STATUS_IS_ENETUNREACH(s)     ((s) == APR_ENETUNREACH)
-/** inappropiate file type or format */
-#define APR_STATUS_IS_EFTYPE(s)          ((s) == APR_EFTYPE)
-/** broken pipe */
-#define APR_STATUS_IS_EPIPE(s)           ((s) == APR_EPIPE)
-/** cross device link */
-#define APR_STATUS_IS_EXDEV(s)           ((s) == APR_EXDEV)
-/** Directory Not Empty */
-#define APR_STATUS_IS_ENOTEMPTY(s)       ((s) == APR_ENOTEMPTY || \
-                                          (s) == APR_EEXIST)
-/** @} */
-
-#endif /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */
-
-/** @} */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  /* ! APR_ERRNO_H */
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <sys/errno.h>
-
-
-/*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-/*
- * Copyright (c) 1982, 1986, 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- * (c) UNIX System Laboratories, Inc.
- * All or some portions of this file are derived from material licensed
- * to the University of California by American Telephone and Telegraph
- * Co. or Unix System Laboratories, Inc. and are reproduced herein with
- * the permission of UNIX System Laboratories, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)errno.h	8.5 (Berkeley) 1/21/94
- */
-
-#ifndef	_SYS_ERRNO_H_
-#define	_SYS_ERRNO_H_
-
-#include <sys/cdefs.h>
-__BEGIN_DECLS
-extern int * __error(void);
-#define errno (*__error())
-__END_DECLS
-
-/*
- * Error codes
- */
-
-#define	EPERM		1		/* Operation not permitted */
-#define	ENOENT		2		/* No such file or directory */
-#define	ESRCH		3		/* No such process */
-#define	EINTR		4		/* Interrupted system call */
-#define	EIO		5		/* Input/output error */
-#define	ENXIO		6		/* Device not configured */
-#define	E2BIG		7		/* Argument list too long */
-#define	ENOEXEC		8		/* Exec format error */
-#define	EBADF		9		/* Bad file descriptor */
-#define	ECHILD		10		/* No child processes */
-#define	EDEADLK		11		/* Resource deadlock avoided */
-					/* 11 was EAGAIN */
-#define	ENOMEM		12		/* Cannot allocate memory */
-#define	EACCES		13		/* Permission denied */
-#define	EFAULT		14		/* Bad address */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	ENOTBLK		15		/* Block device required */
-#endif
-#define	EBUSY		16		/* Device / Resource busy */
-#define	EEXIST		17		/* File exists */
-#define	EXDEV		18		/* Cross-device link */
-#define	ENODEV		19		/* Operation not supported by device */
-#define	ENOTDIR		20		/* Not a directory */
-#define	EISDIR		21		/* Is a directory */
-#define	EINVAL		22		/* Invalid argument */
-#define	ENFILE		23		/* Too many open files in system */
-#define	EMFILE		24		/* Too many open files */
-#define	ENOTTY		25		/* Inappropriate ioctl for device */
-#define	ETXTBSY		26		/* Text file busy */
-#define	EFBIG		27		/* File too large */
-#define	ENOSPC		28		/* No space left on device */
-#define	ESPIPE		29		/* Illegal seek */
-#define	EROFS		30		/* Read-only file system */
-#define	EMLINK		31		/* Too many links */
-#define	EPIPE		32		/* Broken pipe */
-
-/* math software */
-#define	EDOM		33		/* Numerical argument out of domain */
-#define	ERANGE		34		/* Result too large */
-
-/* non-blocking and interrupt i/o */
-#define	EAGAIN		35		/* Resource temporarily unavailable */
-#define	EWOULDBLOCK	EAGAIN		/* Operation would block */
-#define	EINPROGRESS	36		/* Operation now in progress */
-#define	EALREADY	37		/* Operation already in progress */
-
-/* ipc/network software -- argument errors */
-#define	ENOTSOCK	38		/* Socket operation on non-socket */
-#define	EDESTADDRREQ	39		/* Destination address required */
-#define	EMSGSIZE	40		/* Message too long */
-#define	EPROTOTYPE	41		/* Protocol wrong type for socket */
-#define	ENOPROTOOPT	42		/* Protocol not available */
-#define	EPROTONOSUPPORT	43		/* Protocol not supported */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	ESOCKTNOSUPPORT	44		/* Socket type not supported */
-#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
-#define ENOTSUP		45		/* Operation not supported */
-#if !__DARWIN_UNIX03 && !defined(KERNEL)
-/*
- * This is the same for binary and source copmpatability, unless compiling
- * the kernel itself, or compiling __DARWIN_UNIX03; if compiling for the
- * kernel, the correct value will be returned.  If compiling non-POSIX
- * source, the kernel return value will be converted by a stub in libc, and
- * if compiling source with __DARWIN_UNIX03, the conversion in libc is not
- * done, and the caller gets the expected (discrete) value.
- */
-#define	EOPNOTSUPP	 ENOTSUP	/* Operation not supported on socket */
-#endif /* !__DARWIN_UNIX03 && !KERNEL */
-
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	EPFNOSUPPORT	46		/* Protocol family not supported */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-#define	EAFNOSUPPORT	47		/* Address family not supported by protocol family */
-#define	EADDRINUSE	48		/* Address already in use */
-#define	EADDRNOTAVAIL	49		/* Can't assign requested address */
-
-/* ipc/network software -- operational errors */
-#define	ENETDOWN	50		/* Network is down */
-#define	ENETUNREACH	51		/* Network is unreachable */
-#define	ENETRESET	52		/* Network dropped connection on reset */
-#define	ECONNABORTED	53		/* Software caused connection abort */
-#define	ECONNRESET	54		/* Connection reset by peer */
-#define	ENOBUFS		55		/* No buffer space available */
-#define	EISCONN		56		/* Socket is already connected */
-#define	ENOTCONN	57		/* Socket is not connected */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	ESHUTDOWN	58		/* Can't send after socket shutdown */
-#define	ETOOMANYREFS	59		/* Too many references: can't splice */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-#define	ETIMEDOUT	60		/* Operation timed out */
-#define	ECONNREFUSED	61		/* Connection refused */
-
-#define	ELOOP		62		/* Too many levels of symbolic links */
-#define	ENAMETOOLONG	63		/* File name too long */
-
-/* should be rearranged */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	EHOSTDOWN	64		/* Host is down */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-#define	EHOSTUNREACH	65		/* No route to host */
-#define	ENOTEMPTY	66		/* Directory not empty */
-
-/* quotas & mush */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	EPROCLIM	67		/* Too many processes */
-#define	EUSERS		68		/* Too many users */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-#define	EDQUOT		69		/* Disc quota exceeded */
-
-/* Network File System */
-#define	ESTALE		70		/* Stale NFS file handle */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	EREMOTE		71		/* Too many levels of remote in path */
-#define	EBADRPC		72		/* RPC struct is bad */
-#define	ERPCMISMATCH	73		/* RPC version wrong */
-#define	EPROGUNAVAIL	74		/* RPC prog. not avail */
-#define	EPROGMISMATCH	75		/* Program version wrong */
-#define	EPROCUNAVAIL	76		/* Bad procedure for program */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-
-#define	ENOLCK		77		/* No locks available */
-#define	ENOSYS		78		/* Function not implemented */
-
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	EFTYPE		79		/* Inappropriate file type or format */
-#define	EAUTH		80		/* Authentication error */
-#define	ENEEDAUTH	81		/* Need authenticator */
-
-/* Intelligent device errors */
-#define	EPWROFF		82	/* Device power is off */
-#define	EDEVERR		83	/* Device error, e.g. paper out */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-
-#define	EOVERFLOW	84		/* Value too large to be stored in data type */
-
-/* Program loading errors */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define EBADEXEC	85	/* Bad executable */
-#define EBADARCH	86	/* Bad CPU type in executable */
-#define ESHLIBVERS	87	/* Shared library version mismatch */
-#define EBADMACHO	88	/* Malformed Macho file */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-
-#define	ECANCELED	89		/* Operation canceled */
-
-#define EIDRM		90		/* Identifier removed */
-#define ENOMSG		91		/* No message of desired type */   
-#define EILSEQ		92		/* Illegal byte sequence */
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define ENOATTR		93		/* Attribute not found */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-
-#define EBADMSG		94		/* Bad message */
-#define EMULTIHOP	95		/* Reserved */
-#define	ENODATA		96		/* No message available on STREAM */
-#define ENOLINK		97		/* Reserved */
-#define ENOSR		98		/* No STREAM resources */
-#define ENOSTR		99		/* Not a STREAM */
-#define	EPROTO		100		/* Protocol error */
-#define ETIME		101		/* STREAM ioctl timeout */
-
-#if __DARWIN_UNIX03 || defined(KERNEL)
-/* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */
-#define	EOPNOTSUPP	102		/* Operation not supported on socket */
-#endif /* __DARWIN_UNIX03 || KERNEL */
-
-#define ENOPOLICY	103		/* No such policy registered */
-
-#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
-#define	ELAST		103		/* Must be equal largest errno */
-#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
-
-#endif /* _SYS_ERRNO_H_ */
diff --git a/doc/errno.list.solaris.txt b/doc/errno.list.solaris.txt
deleted file mode 100644
index 23601e9d374..00000000000
--- a/doc/errno.list.solaris.txt
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright 2000 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
- */
-
-/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
-/*	  All Rights Reserved  	*/
-
-/*
- * University Copyright- Copyright (c) 1982, 1986, 1988
- * The Regents of the University of California
- * All Rights Reserved
- *
- * University Acknowledgment- Portions of this document are derived from
- * software developed by the University of California, Berkeley, and its
- * contributors.
- */
-
-#ifndef _SYS_ERRNO_H
-#define	_SYS_ERRNO_H
-
-#pragma ident	"@(#)errno.h	1.22	05/06/08 SMI"
-
-#ifdef	__cplusplus
-extern "C" {
-#endif
-
-/*
- * Error codes
- */
-
-#define	EPERM	1	/* Not super-user			*/
-#define	ENOENT	2	/* No such file or directory		*/
-#define	ESRCH	3	/* No such process			*/
-#define	EINTR	4	/* interrupted system call		*/
-#define	EIO	5	/* I/O error				*/
-#define	ENXIO	6	/* No such device or address		*/
-#define	E2BIG	7	/* Arg list too long			*/
-#define	ENOEXEC	8	/* Exec format error			*/
-#define	EBADF	9	/* Bad file number			*/
-#define	ECHILD	10	/* No children				*/
-#define	EAGAIN	11	/* Resource temporarily unavailable	*/
-#define	ENOMEM	12	/* Not enough core			*/
-#define	EACCES	13	/* Permission denied			*/
-#define	EFAULT	14	/* Bad address				*/
-#define	ENOTBLK	15	/* Block device required		*/
-#define	EBUSY	16	/* Mount device busy			*/
-#define	EEXIST	17	/* File exists				*/
-#define	EXDEV	18	/* Cross-device link			*/
-#define	ENODEV	19	/* No such device			*/
-#define	ENOTDIR	20	/* Not a directory			*/
-#define	EISDIR	21	/* Is a directory			*/
-#define	EINVAL	22	/* Invalid argument			*/
-#define	ENFILE	23	/* File table overflow			*/
-#define	EMFILE	24	/* Too many open files			*/
-#define	ENOTTY	25	/* Inappropriate ioctl for device	*/
-#define	ETXTBSY	26	/* Text file busy			*/
-#define	EFBIG	27	/* File too large			*/
-#define	ENOSPC	28	/* No space left on device		*/
-#define	ESPIPE	29	/* Illegal seek				*/
-#define	EROFS	30	/* Read only file system		*/
-#define	EMLINK	31	/* Too many links			*/
-#define	EPIPE	32	/* Broken pipe				*/
-#define	EDOM	33	/* Math arg out of domain of func	*/
-#define	ERANGE	34	/* Math result not representable	*/
-#define	ENOMSG	35	/* No message of desired type		*/
-#define	EIDRM	36	/* Identifier removed			*/
-#define	ECHRNG	37	/* Channel number out of range		*/
-#define	EL2NSYNC 38	/* Level 2 not synchronized		*/
-#define	EL3HLT	39	/* Level 3 halted			*/
-#define	EL3RST	40	/* Level 3 reset			*/
-#define	ELNRNG	41	/* Link number out of range		*/
-#define	EUNATCH 42	/* Protocol driver not attached		*/
-#define	ENOCSI	43	/* No CSI structure available		*/
-#define	EL2HLT	44	/* Level 2 halted			*/
-#define	EDEADLK	45	/* Deadlock condition.			*/
-#define	ENOLCK	46	/* No record locks available.		*/
-#define	ECANCELED 47	/* Operation canceled			*/
-#define	ENOTSUP	48	/* Operation not supported		*/
-
-/* Filesystem Quotas */
-#define	EDQUOT	49	/* Disc quota exceeded			*/
-
-/* Convergent Error Returns */
-#define	EBADE	50	/* invalid exchange			*/
-#define	EBADR	51	/* invalid request descriptor		*/
-#define	EXFULL	52	/* exchange full			*/
-#define	ENOANO	53	/* no anode				*/
-#define	EBADRQC	54	/* invalid request code			*/
-#define	EBADSLT	55	/* invalid slot				*/
-#define	EDEADLOCK 56	/* file locking deadlock error		*/
-
-#define	EBFONT	57	/* bad font file fmt			*/
-
-/* Interprocess Robust Locks */
-#define	EOWNERDEAD	58	/* process died with the lock */
-#define	ENOTRECOVERABLE	59	/* lock is not recoverable */
-
-/* stream problems */
-#define	ENOSTR	60	/* Device not a stream			*/
-#define	ENODATA	61	/* no data (for no delay io)		*/
-#define	ETIME	62	/* timer expired			*/
-#define	ENOSR	63	/* out of streams resources		*/
-
-#define	ENONET	64	/* Machine is not on the network	*/
-#define	ENOPKG	65	/* Package not installed		*/
-#define	EREMOTE	66	/* The object is remote			*/
-#define	ENOLINK	67	/* the link has been severed		*/
-#define	EADV	68	/* advertise error			*/
-#define	ESRMNT	69	/* srmount error			*/
-
-#define	ECOMM	70	/* Communication error on send		*/
-#define	EPROTO	71	/* Protocol error			*/
-
-/* Interprocess Robust Locks */
-#define	ELOCKUNMAPPED	72	/* locked lock was unmapped */
-
-#define	ENOTACTIVE 73	/* Facility is not active		*/
-#define	EMULTIHOP 74	/* multihop attempted			*/
-#define	EBADMSG 77	/* trying to read unreadable message	*/
-#define	ENAMETOOLONG 78	/* path name is too long		*/
-#define	EOVERFLOW 79	/* value too large to be stored in data type */
-#define	ENOTUNIQ 80	/* given log. name not unique		*/
-#define	EBADFD	81	/* f.d. invalid for this operation	*/
-#define	EREMCHG	82	/* Remote address changed		*/
-
-/* shared library problems */
-#define	ELIBACC	83	/* Can't access a needed shared lib.	*/
-#define	ELIBBAD	84	/* Accessing a corrupted shared lib.	*/
-#define	ELIBSCN	85	/* .lib section in a.out corrupted.	*/
-#define	ELIBMAX	86	/* Attempting to link in too many libs.	*/
-#define	ELIBEXEC 87	/* Attempting to exec a shared library.	*/
-#define	EILSEQ	88	/* Illegal byte sequence.		*/
-#define	ENOSYS	89	/* Unsupported file system operation	*/
-#define	ELOOP	90	/* Symbolic link loop			*/
-#define	ERESTART 91	/* Restartable system call		*/
-#define	ESTRPIPE 92	/* if pipe/FIFO, don't sleep in stream head */
-#define	ENOTEMPTY 93	/* directory not empty			*/
-#define	EUSERS	94	/* Too many users (for UFS)		*/
-
-/* BSD Networking Software */
-	/* argument errors */
-#define	ENOTSOCK	95	/* Socket operation on non-socket */
-#define	EDESTADDRREQ	96	/* Destination address required */
-#define	EMSGSIZE	97	/* Message too long */
-#define	EPROTOTYPE	98	/* Protocol wrong type for socket */
-#define	ENOPROTOOPT	99	/* Protocol not available */
-#define	EPROTONOSUPPORT	120	/* Protocol not supported */
-#define	ESOCKTNOSUPPORT	121	/* Socket type not supported */
-#define	EOPNOTSUPP	122	/* Operation not supported on socket */
-#define	EPFNOSUPPORT	123	/* Protocol family not supported */
-#define	EAFNOSUPPORT	124	/* Address family not supported by */
-				/* protocol family */
-#define	EADDRINUSE	125	/* Address already in use */
-#define	EADDRNOTAVAIL	126	/* Can't assign requested address */
-	/* operational errors */
-#define	ENETDOWN	127	/* Network is down */
-#define	ENETUNREACH	128	/* Network is unreachable */
-#define	ENETRESET	129	/* Network dropped connection because */
-				/* of reset */
-#define	ECONNABORTED	130	/* Software caused connection abort */
-#define	ECONNRESET	131	/* Connection reset by peer */
-#define	ENOBUFS		132	/* No buffer space available */
-#define	EISCONN		133	/* Socket is already connected */
-#define	ENOTCONN	134	/* Socket is not connected */
-/* XENIX has 135 - 142 */
-#define	ESHUTDOWN	143	/* Can't send after socket shutdown */
-#define	ETOOMANYREFS	144	/* Too many references: can't splice */
-#define	ETIMEDOUT	145	/* Connection timed out */
-#define	ECONNREFUSED	146	/* Connection refused */
-#define	EHOSTDOWN	147	/* Host is down */
-#define	EHOSTUNREACH	148	/* No route to host */
-#define	EWOULDBLOCK	EAGAIN
-#define	EALREADY	149	/* operation already in progress */
-#define	EINPROGRESS	150	/* operation now in progress */
-
-/* SUN Network File System */
-#define	ESTALE		151	/* Stale NFS file handle */
-
-
-#ifdef	__cplusplus
-}
-#endif
-
-#endif	/* _SYS_ERRNO_H */
diff --git a/doc/examples/Makefile.am b/doc/examples/Makefile.am
deleted file mode 100644
index b4c93f4c90b..00000000000
--- a/doc/examples/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-EXTRA = README unify.vol replicate.vol stripe.vol protocol-client.vol protocol-server.vol posix-locks.vol trash.vol write-behind.vol io-threads.vol io-cache.vol read-ahead.vol filter.vol trace.vol
-EXTRA_DIST = $(EXTRA)
-
-docdir = $(datadir)/doc/$(PACKAGE_NAME)
-Examplesdir = $(docdir)/examples
-Examples_DATA = $(EXTRA)
-
-CLEANFILES = 
diff --git a/doc/examples/README b/doc/examples/README
deleted file mode 100644
index 4d472ac086e..00000000000
--- a/doc/examples/README
+++ /dev/null
@@ -1,13 +0,0 @@
-GlusterFS's translator feature is very flexible and there are quite a lot of ways one 
-can configure their filesystem to behave like. 
-
-Volume Specification is a way in which GlusterFS understands how it has to work, based 
-on what is written there. 
-
-Going through the following URLs may give you more idea about all these.
-
-* http://www.gluster.org/docs/index.php/GlusterFS
-* http://www.gluster.org/docs/index.php/GlusterFS_Volume_Specification
-* http://www.gluster.org/docs/index.php/GlusterFS_Translators
-
-Mail us any doubts, suggestions on 'gluster-devel(at)nongnu.org'
diff --git a/doc/examples/filter.vol b/doc/examples/filter.vol
deleted file mode 100644
index ca5c5983779..00000000000
--- a/doc/examples/filter.vol
+++ /dev/null
@@ -1,23 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-# 
-# Definition of cluster translator (may be unify, afr, or unify over afr)
-# 
-
-### 'Filter' translator is used on client side (or server side according to needs). This traslator makes all the below translators, (or say volumes) as read-only. Hence if one wants a 'read-only' filesystem, using filter as the top most volume will make it really fast as the fops are returned from this level itself.
-
-volume filter-ro
- type features/filter
- option root-squashing enable
-# option completely-read-only yes
-# translate-uid 1-99=0
- subvolumes client
-end-volume
-\ No newline at end of file
diff --git a/doc/examples/io-cache.vol b/doc/examples/io-cache.vol
deleted file mode 100644
index 5f3eca4c527..00000000000
--- a/doc/examples/io-cache.vol
+++ /dev/null
@@ -1,25 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-# 
-# Definition of cluster translator (may be unify, replicate, or unify over replicate)
-# 
-
-### 'IO-Cache' translator is best used on client side when a filesystem has file which are not modified frequently but read several times. For example, while compiling a kernel, *.h files are read while compiling every *.c file, in these case, io-cache translator comes very handy, as it keeps the whole file content in the cache, and serves from the cache.
-# One can provide the priority of the cache too.
-
-volume ioc
-  type performance/io-cache
-  subvolumes client         # In this example it is 'client' you may have to change it according to your spec file.
-  option page-size 1MB      # 128KB is default
-  option cache-size 64MB    # 32MB is default
-  option force-revalidate-timeout 5 # 1second is default 
-  option priority *.html:2,*:1 # default is *:0
-end-volume
diff --git a/doc/examples/io-threads.vol b/doc/examples/io-threads.vol
deleted file mode 100644
index 9954724e1fe..00000000000
--- a/doc/examples/io-threads.vol
+++ /dev/null
@@ -1,21 +0,0 @@
-
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-###  'IO-threads' translator gives a threading behaviour to File I/O calls. All other normal fops are having default behaviour. Loading this on server side helps to reduce the contension of network. (Which is assumed as a GlusterFS hang). 
-# One can load it in client side to reduce the latency involved in case of a slow network, when loaded below write-behind.
-volume iot
-  type performance/io-threads
-  subvolumes brick
-  option thread-count 4 # default value is 1
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes iot brick
-  option transport-type tcp     # For TCP/IP transport
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-  option auth.addr.iot.allow 192.168.* # Allow access to "p-locks" volume
-end-volume
diff --git a/doc/examples/posix-locks.vol b/doc/examples/posix-locks.vol
deleted file mode 100644
index b9c9e7a6481..00000000000
--- a/doc/examples/posix-locks.vol
+++ /dev/null
@@ -1,20 +0,0 @@
-
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### 'Posix-locks' feature should be added on the server side (as posix volume as subvolume) because it contains the actual file. 
-volume p-locks
-  type features/posix-locks
-  subvolumes brick
-  option mandatory on
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes p-locks brick
-  option transport-type tcp
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-  option auth.addr.p-locks.allow 192.168.* # Allow access to "p-locks" volume
-end-volume
diff --git a/doc/examples/protocol-client.vol b/doc/examples/protocol-client.vol
deleted file mode 100644
index 43c43e02d54..00000000000
--- a/doc/examples/protocol-client.vol
+++ /dev/null
@@ -1,17 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-# option transport.socket.remote-port 24016              
-
-# option transport-type ib-verbs # for Infiniband verbs transport
-# option transport.ib-verbs.work-request-send-size  1048576
-# option transport.ib-verbs.work-request-send-count 16
-# option transport.ib-verbs.work-request-recv-size  1048576
-# option transport.ib-verbs.work-request-recv-count 16
-# option transport.ib-verbs.remote-port 24016              
-
-  option remote-subvolume brick        # name of the remote volume
-# option transport-timeout 30          # default value is 120seconds 
-end-volume
diff --git a/doc/examples/protocol-server.vol b/doc/examples/protocol-server.vol
deleted file mode 100644
index e8e4a464383..00000000000
--- a/doc/examples/protocol-server.vol
+++ /dev/null
@@ -1,25 +0,0 @@
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp      # For TCP/IP transport
-# option transport.socket.listen-port 24016              
-
-# option transport-type ib-verbs # For Infiniband Verbs transport
-# option transport.ib-verbs.work-request-send-size  131072
-# option transport.ib-verbs.work-request-send-count 64
-# option transport.ib-verbs.work-request-recv-size  131072
-# option transport.ib-verbs.work-request-recv-count 64
-# option transport.ib-verbs.listen-port 24016              
-
-# option bind-address 192.168.1.10     # Default is to listen on all interfaces
-# option client-volume-filename /etc/glusterfs/glusterfs-client.vol
-  subvolumes brick 
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-end-volume
diff --git a/doc/examples/read-ahead.vol b/doc/examples/read-ahead.vol
deleted file mode 100644
index 3ce0d95aca7..00000000000
--- a/doc/examples/read-ahead.vol
+++ /dev/null
@@ -1,22 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-# 
-# Definition of cluster translator (may be unify, replicate, or unify over replicate)
-# 
-
-### 'Read-Ahead' translator is best utilized on client side, as it prefetches the file contents when the first read() call is issued. 
-volume ra
-  type performance/read-ahead
-  subvolumes client            # In this example it is 'client' you may have to change it according to your spec file.
-  option page-size 1MB         # default is 256KB
-  option page-count 4          # default is 2
-  option force-atime-update no # defalut is 'no'
-end-volume
diff --git a/doc/examples/replicate.vol b/doc/examples/replicate.vol
deleted file mode 100644
index 333ba7de1aa..00000000000
--- a/doc/examples/replicate.vol
+++ /dev/null
@@ -1,119 +0,0 @@
-### 'NOTE'
-# This file has both server spec and client spec to get an understanding of stripe's spec file. Hence can't be used as it is, as a GlusterFS spec file. 
-# One need to seperate out server spec and client spec to get it working.
-
-#=========================================================================
-
-# **** server1 spec file ****
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume posix1
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export1        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick1
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix1
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                      # For TCP/IP transport
-  option transport.socket.listen-port 24016       
-# option client-volume-filename /etc/glusterfs/glusterfs-client.vol
-  subvolumes brick1
-  option auth.addr.brick1.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server2 spec file ****
-volume posix2
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export2        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick2
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix2
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                  # For TCP/IP transport
-  option transport.socket.listen-port 24017   
-  subvolumes brick2
-  option auth.addr.brick2.allow * # Allow access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server3 spec file ****
-
-volume posix3
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export3        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick3
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix3
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                 # For TCP/IP transport
-  option transport.socket.listen-port 24018  
-  subvolumes brick3
-  option auth.addr.brick3.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** Clustered Client config file ****
-
-### Add client feature and attach to remote subvolume of server1
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24016              
-  option remote-subvolume brick1        # name of the remote volume
-end-volume
-
-### Add client feature and attach to remote subvolume of server2
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24017              
-  option remote-subvolume brick2        # name of the remote volume
-end-volume
-
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24018              
-  option remote-subvolume brick3        # name of the remote volume
-end-volume
-
-## Add replicate feature.
-volume replicate
-  type cluster/replicate
-  subvolumes client1 client2 client3
-end-volume
-
diff --git a/doc/examples/stripe.vol b/doc/examples/stripe.vol
deleted file mode 100644
index 6055b66b922..00000000000
--- a/doc/examples/stripe.vol
+++ /dev/null
@@ -1,121 +0,0 @@
-
-### 'NOTE'
-# This file has both server spec and client spec to get an understanding of stripe's spec file. Hence can't be used as it is, as a GlusterFS spec file. 
-# One need to seperate out server spec and client spec to get it working.
-
-#=========================================================================
-
-# **** server1 spec file ****
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume posix1
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export1        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick1
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix1
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24016              
-# option client-volume-filename /etc/glusterfs/glusterfs-client.vol
-  subvolumes brick1
-  option auth.addr.brick1.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server2 spec file ****
-volume posix2
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export2        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick2
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix2
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24017              
-  subvolumes brick2
-  option auth.addr.brick2.allow * # Allow access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server3 spec file ****
-
-volume posix3
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export3        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick3
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix3
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24018              
-  subvolumes brick3
-  option auth.addr.brick3.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** Clustered Client config file ****
-
-### Add client feature and attach to remote subvolume of server1
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24016              
-  option remote-subvolume brick1        # name of the remote volume
-end-volume
-
-### Add client feature and attach to remote subvolume of server2
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24017              
-  option remote-subvolume brick2        # name of the remote volume
-end-volume
-
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24018              
-  option remote-subvolume brick3        # name of the remote volume
-end-volume
-
-## Add Stripe Feature.
-volume stripe
-  type cluster/stripe
-  subvolumes client1 client2 client3
-  option block-size 1MB
-end-volume
-
diff --git a/doc/examples/trace.vol b/doc/examples/trace.vol
deleted file mode 100644
index 3f4864db496..00000000000
--- a/doc/examples/trace.vol
+++ /dev/null
@@ -1,16 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-### 'Trace' translator is a very handy debug tool for GlusterFS, as it can be loaded between any of the two volumes without changing the behaviour of the filesystem. 
-# On client side it can be the top most volume in spec (like now) to understand what calls are made on FUSE filesystem, when a mounted filesystem is accessed.
-
-volume trace
-  type debug/trace
-  subvolumes client
-end-volume
-
-# 'NOTE:' By loading 'debug/trace' translator, filesystem will be very slow as it logs each and every calls to the log file.
diff --git a/doc/examples/trash.vol b/doc/examples/trash.vol
deleted file mode 100644
index 16e71be3232..00000000000
--- a/doc/examples/trash.vol
+++ /dev/null
@@ -1,20 +0,0 @@
-
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### 'Trash' translator is best used on server side as it just renames the deleted file inside 'trash-dir', and it makes 4 seperate fops for one unlink call.  
-volume trashcan
-  type features/trash
-  subvolumes brick
-  option trash-dir /.trashcan
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes trashcan brick
-  option transport-type tcp     # For TCP/IP transport
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-  option auth.addr.trashcan.allow 192.168.* # Allow access to "p-locks" volume
-end-volume
diff --git a/doc/examples/unify.vol b/doc/examples/unify.vol
deleted file mode 100644
index 3fb7e832086..00000000000
--- a/doc/examples/unify.vol
+++ /dev/null
@@ -1,178 +0,0 @@
-### 'NOTE'
-# This file has both server spec and client spec to get an understanding of stripe's spec file. Hence can't be used as it is, as a GlusterFS spec file. 
-# One need to seperate out server spec and client spec to get it working.
-
-
-#=========================================================================
-
-# **** server1 spec file ****
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume posix1
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export1        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick1
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix1
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24016              
-# option client-volume-filename /etc/glusterfs/glusterfs-client.vol
-  subvolumes brick1
-  option auth.addr.brick1.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server2 spec file ****
-volume posix2
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export2        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick2
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix2
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24017              
-  subvolumes brick2
-  option auth.addr.brick2.allow * # Allow access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server3 spec file ****
-
-volume posix3
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export3        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick3
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix3
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24018              
-  subvolumes brick3
-  option auth.addr.brick3.allow * 		# access to "brick" volume
-end-volume
-
-#=========================================================================
-
-# *** server for namespace *** 
-### Export volume "brick" with the contents of "/home/export" directory.
-volume brick-ns
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export-ns      # Export this directory
-end-volume
-
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24019              
-  subvolumes brick-ns
-  option auth.addr.brick-ns.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** Clustered Client config file ****
-
-### Add client feature and attach to remote subvolume of server1
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24016              
-  option remote-subvolume brick1        # name of the remote volume
-end-volume
-
-### Add client feature and attach to remote subvolume of server2
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24017              
-  option remote-subvolume brick2        # name of the remote volume
-end-volume
-
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24018              
-  option remote-subvolume brick3        # name of the remote volume
-end-volume
-
-
-volume client-ns
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-  option remote-host 127.0.0.1         # IP address of the remote brick
-  option transport.socket.remote-port 24019              
-  option remote-subvolume brick-ns     # name of the remote volume
-end-volume
-
-### Add unify feature to cluster the servers. Associate an
-### appropriate scheduler that matches your I/O demand.
-volume bricks
-  type cluster/unify
-  option namespace client-ns # this will not be storage child of unify.
-  subvolumes client1 client2 client3
-### ** ALU Scheduler Option **
-  option self-heal background # foreground off # default is foreground
-  option scheduler alu
-  option alu.limits.min-free-disk  5% #%
-  option alu.limits.max-open-files 10000
-  option alu.order disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
-  option alu.disk-usage.entry-threshold 2GB
-  option alu.disk-usage.exit-threshold  128MB
-  option alu.open-files-usage.entry-threshold 1024
-  option alu.open-files-usage.exit-threshold 32
-  option alu.read-usage.entry-threshold 20 #%
-  option alu.read-usage.exit-threshold 4 #%
-  option alu.write-usage.entry-threshold 20 #%
-  option alu.write-usage.exit-threshold 4 #%
-  option alu.disk-speed-usage.entry-threshold 0 # DO NOT SET IT. SPEED IS CONSTANT!!!.
-  option alu.disk-speed-usage.exit-threshold 0 # DO NOT SET IT. SPEED IS CONSTANT!!!.
-  option alu.stat-refresh.interval 10sec
-  option alu.stat-refresh.num-file-create 10
-### ** Random Scheduler **
-# option scheduler random
-### ** NUFA Scheduler **
-# option scheduler nufa
-# option nufa.local-volume-name posix1
-### ** Round Robin (RR) Scheduler **
-# option scheduler rr
-# option rr.limits.min-free-disk 5% #%
-end-volume
-
diff --git a/doc/examples/write-behind.vol b/doc/examples/write-behind.vol
deleted file mode 100644
index 9c6bae11c53..00000000000
--- a/doc/examples/write-behind.vol
+++ /dev/null
@@ -1,26 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-# 
-# Definition of cluster translator (may be unify, replicate, or unify over replicate)
-# 
-
-
-### 'Write-behind' translator is a performance booster for write operation. Best used on client side, as its main intension is to reduce the network latency caused for each write operation.
-
-volume wb
-  type performance/write-behind
-  subvolumes client         # In this example it is 'client' you may have to change it according to your spec file.
-  option flush-behind on    # default value is 'off'
-  option window-size 2MB
-  option aggregate-size 1MB # default value is 0
-  option enable_O_SYNC no  # default is no
-  option disable-for-first-nbytes 128KB #default is 1 
-end-volume
diff --git a/doc/features/ctime.md b/doc/features/ctime.md
new file mode 100644
index 00000000000..74a77abed4b
--- /dev/null
+++ b/doc/features/ctime.md
@@ -0,0 +1,68 @@
+# Consistent time attributes in gluster across replica/distribute
+
+
+#### Problem:
+Traditionally gluster has been using time attributes (ctime, atime, mtime) of files/dirs from bricks. The problem with this approach is that, it is not consisteant  across replica and distribute bricks. And applications which depend on it breaks as replica  might not  always return time attributes from same brick.
+
+Tar especially gives "file changed as we read it" whenever it detects ctime differences when stat is served from  different bricks. The way we have been trying to solve it is to serve  the stat structures from same brick in afr, max-time in dht. But it doesn't avoid the problem completely. Because there is no way to change ctime at the moment(lutimes() only allows mtime, atime), there is little we can do to make sure ctimes match after self-heals/xattr updates/rebalance.
+
+#### Solution Proposed:
+Store time attribues (ctime, mtime, atime) as an xattr of the file. The xattr is updated based
+on the fop. If a filesystem fop changes only mtime and ctime, update only those in xattr for
+that file.
+
+#### Design Overview:
+1) As part of each fop, top layer will generate a time stamp and pass it to the down along
+ with other information
+  - This will bring a dependency for NTP synced clients along with servers
+  - There can be a diff in time if the fop stuck in the xlator for various reason,
+for ex: because of locks.
+
+ 2)  On the server, posix layer stores the value in the memory (inode ctx) and will sync the data periodically to the disk as an extended attr
+  -  Of course sync call also will force it. And fop comes for an inode which is not linked, we do the sync immediately.
+
+ 3)  Each time when inodes are created or initialized it read the data from disk and store in inode ctx.
+
+ 4)  Before setting to inode_ctx we compare the timestamp stored and the timestamp received, and only store if the stored value is lesser than the current value.
+
+ 5)  So in best case data will be stored and retrieved from the memory. We replace the values in iatt with the values in inode_ctx.
+
+ 6)  File ops that changes the parent directory attr time need to be consistent across all the distributed directories across the subvolumes. (for eg: a create call will change ctime and mtime of parent dir)
+
+  - This has to handle separately because we only send the fop to the hashed subvolume.
+  - We can asynchronously send the timeupdate setattr fop to the other subvoumes and change the values for parent directory if the file fops is successful on hashed subvolume.
+  -  This will have a window where the times are inconsistent across dht subvolume (Please provide your suggestions)
+
+7)  Currently we have couple of mount options for time attributes like noatime, relatime , nodiratime etc. But we are not explicitly handled those options even if it is given as mount option when gluster mount.
+
+
+#### Implementation Overview:
+This features involves changes in following xlators.
+ - utime xlator
+ - posix xlator
+
+##### utime xlator:
+This is a new client side xlator which does following tasks.
+
+1. It will generate a time stamp and passes it down in frame->root->ctime  and over the network.
+2.  Based on fop, it also decides the time attributes to be updated and this passed using "frame->root->flags"
+
+    Patches:
+    1. https://review.gluster.org/#/c/19857/
+
+##### posix xlator:
+Following tasks are done in posix xlator:
+
+1. Provides APIs to set and get the xattr from backend. It also caches the xattr in inode context. During get, it updates time attributes stored in xattr into iatt structure.
+2. Based on the flags from utime xlator, relevant fops update the time attributes in the xattr.
+
+    Patches:
+    1. https://review.gluster.org/#/c/19267/
+    2. https://review.gluster.org/#/c/19795/
+    3. https://review.gluster.org/#/c/19796/
+
+#### Pending Work:
+1. Handling of time related mount options (noatime, realatime,etc)
+2. flag based create (depending on flags in open, create behaviour might change)
+3. Changes in dht for direcotory sync acrosss multiple subvolumes
+4. readdirp stat need to be worked on.
diff --git a/doc/features/ganesha-ha.md b/doc/features/ganesha-ha.md
new file mode 100644
index 00000000000..4b226a22ccf
--- /dev/null
+++ b/doc/features/ganesha-ha.md
@@ -0,0 +1,43 @@
+# Overview of Ganesha HA Resource Agents in GlusterFS 3.7
+
+The ganesha_mon RA monitors its ganesha.nfsd daemon. While the
+daemon is running, it creates two attributes: ganesha-active and
+grace-active. When the daemon stops for any reason, the attributes
+are deleted. Deleting the ganesha-active attribute triggers the
+failover of the virtual IP (the IPaddr RA) to another node —
+according to constraint location rules — where ganesha.nfsd is
+still running.
+
+The ganesha_grace RA monitors the grace-active attribute. When
+the grace-active attibute is deleted, the ganesha_grace RA stops,
+and will not restart. This triggers pacemaker to invoke the notify
+action in the ganesha_grace RAs on the other nodes in the cluster;
+which send a DBUS message to their respective ganesha.nfsd.
+
+(N.B. grace-active is a bit of a misnomer. while the grace-active
+attribute exists, everything is normal and healthy. Deleting the
+attribute triggers putting the surviving ganesha.nfsds into GRACE.)
+
+To ensure that the remaining/surviving ganesha.nfsds are put into
+ NFS-GRACE before the IPaddr (virtual IP) fails over there is a
+short delay (sleep) between deleting the grace-active attribute
+and the ganesha-active attribute. To summarize, e.g. in a four
+node cluster:
+
+1. on node 2 ganesha_mon::monitor notices that ganesha.nfsd has died
+
+2. on node 2 ganesha_mon::monitor deletes its grace-active attribute
+
+3. on node 2 ganesha_grace::monitor notices that grace-active is gone
+and returns OCF_ERR_GENERIC, a.k.a. new error. When pacemaker tries
+to (re)start ganesha_grace, its start action will return
+OCF_NOT_RUNNING, a.k.a. known error, don't attempt further restarts.
+
+4. on nodes 1, 3, and 4, ganesha_grace::notify receives a post-stop
+notification indicating that node 2 is gone, and sends a DBUS message
+to its ganesha.nfsd, putting it into NFS-GRACE.
+
+5. on node 2 ganesha_mon::monitor waits a short period, then deletes
+its ganesha-active attribute. This triggers the IPaddr (virt IP)
+failover according to constraint location rules.
+
diff --git a/doc/get_put_api_using_xattr.txt b/doc/get_put_api_using_xattr.txt
deleted file mode 100644
index 58951f5bf4f..00000000000
--- a/doc/get_put_api_using_xattr.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-GlusterFS get/put API interface provided through extended attributes:
-
-API usage:
-  int put(dirpath/filename, data): setfattr -n glusterfs.file.<filename> -v <data> <dirpath>
-  void *get(dirpath/filename):  getfattr -n glusterfs.file.<filename> <dirpath>
-
-
-internals:
-* unify handling setxattr/getxattr
-  - setxattr
-     unify's setxattr forwards setxattr call to all the child nodes with XATTR_REPLACE flag, except namespace. setxattr will succeeds only on the child node on which the file already exists. if the setxattr operation fails on all child nodes, it indicates that the file does not already exist on any of the child nodes. unify follows the same rules as it follows for create, but using setxattr call itself with XATTR_CREATE flag. unify sends a setxattr to namespace first, with zero length data. if namespace setxattr succeeds, unify schedules setxattr to one of the child nodes.
-
-   - getxattr
-      unify's getxattr forwards getxattr call to all the child nodes. wait for completion of operation on all the child nodes, and returns success if getxattr succeeded one child node.
-
-* posix handling setxattr/getxattr
-  - setxattr
-     posix setxattr does a open with O_CREAT|O_TRUNC on the <path>/<name>, writes value of the setxattr as data into the file and closes the file. when data is null, posix setxattr avoids doing write. file is closed after write.
-     
-  - getxattr
-     posix getxattr does open with O_RDONLY on the <path>/<name>, reads the complete content of the file. file is closed after read.
-
diff --git a/doc/gluster.8 b/doc/gluster.8
index 23910dc030b..ba595edca15 100644
--- a/doc/gluster.8
+++ b/doc/gluster.8
@@ -1,133 +1,360 @@
-.\"
-.\"  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+
+.\"  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is GF_FREE software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU Affero General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  Affero General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU Affero General Public License
-.\"  along with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
-.TH Gluster 8 "Gluster command line tool" "9 September 2010" "Gluster Inc."
+.TH Gluster 8 "Gluster command line utility" "07 March 2011" "Gluster Inc."
 .SH NAME
-Gluster \- Gluster command line tool.
+gluster - Gluster Console Manager (command line utility)
 .SH SYNOPSIS
 .B gluster
 .PP
-To enter gluster prompt 
+To run the program and display gluster prompt:
+.PP
+.B gluster [--remote-host=<gluster_node>] [--mode=script] [--xml]
 .PP
 (or)
-.pp
-
-.B gluster
-.I [commands] [options]
 .PP
-Gluster is a command line tool for dynamic volume management for glusterfs.
+To specify a command directly:
+.PP
+.B gluster
+.I  [commands] [options] [--remote-host=<gluster_node>] [--mode=script] [--xml]
 .SH DESCRIPTION
-Gluster command line tool is used for dynamic volume management. 
-The tool must be executed on any of the export servers. The cli gives an easy way to perform cloud operations such as volume create, volume expand, volume shrink, volume migrate, volume rebalance without the necessity of scheduling downtimes. 
+The Gluster Console Manager is a command line utility for elastic volume management. You can run the gluster command on any export server. The command enables administrators to perform cloud operations, such as creating, expanding, shrinking, rebalancing, and migrating volumes without needing to schedule server downtime.
 .SH COMMANDS
-.PP
-Mandatory or optional arguments to long options are also mandatory or optional
-for any corresponding short options.
-.SS "Basic commands"
+
+.SS "Volume Commands"
 .PP
 .TP
 
 \fB\ volume info [all|<VOLNAME>] \fR
-list information of all volumes
+Display information about all volumes, or the specified volume.
 .TP
-\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [transport <tcp|rdma>] <NEW-BRICK> ... \fR
-create a new volume of specified type with mentioned bricks, with specified transport-type [default:tcp]
+\fB\ volume list \fR
+List all volumes in cluster
+.TP
+\fB\ volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]] [detail|clients|mem|inode|fd|callpool|tasks|client-list] \fR
+Display status of all or specified volume(s)/brick
+.TP
+\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] [disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ... <TA-BRICK> \fR
+Create a new volume of the specified type using the specified bricks and transport type (the default transport type is tcp).
+To create a volume with both transports (tcp and rdma), give 'transport tcp,rdma' as an option.
 .TP
 \fB\ volume delete <VOLNAME> \fR
-delete volume specified by \fB\<VOLNAME>\fR
+Delete the specified volume.
 .TP
 \fB\ volume start <VOLNAME> \fR
-start volume specified by \fB\<VOLNAME>\fR
+Start the specified volume.
 .TP
 \fB\ volume stop <VOLNAME> [force] \fR
-stop volume specified by \fB\<VOLNAME>\fR
+Stop the specified volume.
+.TP
+\fB\ volume set <VOLNAME> <OPTION> <PARAMETER> [<OPTION> <PARAMETER>] ... \fR
+Set the volume options.
 .TP
-\fB\ volume rename <VOLNAME> <NEW-VOLNAME> \fR
-rename volume \fB\<VOLNAME>\fR to \fB\<NEW-VOLNAME>\fR
+\fB\ volume get <VOLNAME/all> <OPTION/all> \fR
+Get the value of the all options or given option for volume <VOLNAME> or all option. gluster volume get all all is to get all global options
 .TP
-\fB\ volume add-brick <VOLNAME> [(replica <COUNT>)|(stripe <COUNT>)] <NEW-BRICK> ... \fR
-add brick to volume \fB\<VOLNAME>\fR
+\fB\ volume reset <VOLNAME> [option] [force] \fR
+Reset all the reconfigured options
 .TP
-\fB\ volume remove-brick <VOLNAME> [(replica <COUNT>)|(stripe <COUNT>)] <BRICK> ... \fR
-remove brick from volume \fB\<VOLNAME>\fR
+\fB\ volume barrier <VOLNAME> {enable|disable} \fR
+Barrier/unbarrier file operations on a volume
+.TP
+\fB\ volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}{inode [range]|entry [basename]|posix [range]} \fR
+Clear locks held on path
+.TP
+\fB\ volume help \fR
+Display help for the volume command.
+.SS "Brick Commands"
+.PP
+.TP
+\fB\ volume add-brick <VOLNAME> <NEW-BRICK> ... \fR
+Add the specified brick to the specified volume.
+.TP
+\fB\ volume remove-brick <VOLNAME> <BRICK> ... \fR
+Remove the specified brick from the specified volume.
+.IP
+.B Note:
+If you remove the brick, the data stored in that brick will not be available. You can migrate data from one brick to another using
+.B replace-brick
+option.
+.TP
+\fB\ volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} | {<NEW-BRICK> commit}} \fR
+Brings down or replaces the specified source brick with the new brick.
+.TP
+\fB\ volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> commit force \fR
+Replace the specified source brick with a new brick.
 .TP
 \fB\ volume rebalance <VOLNAME> start \fR
-start rebalance of volume \fB\<VOLNAME>\fR
+Start rebalancing the specified volume.
 .TP
 \fB\ volume rebalance <VOLNAME> stop \fR
-stop rebalance of volume \fB\<VOLNAME>\fR
+Stop rebalancing the specified volume.
 .TP
 \fB\ volume rebalance <VOLNAME> status \fR
-rebalance status of volume \fB<VOLNAME>\fR
+Display the rebalance status of the specified volume.
+.SS "Log Commands"
 .TP
-\fB\ volume replace-brick <VOLNAME> (<BRICK> <NEW-BRICK>) start|pause|abort|status \fR
-replace-brick operations
+\fB\ volume log filename <VOLNAME> [BRICK] <DIRECTORY> \fB
+Set the log directory for the corresponding volume/brick.
 .TP
-\fB\ volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ... \fR
-set transport type for volume \fB\<VOLNAME>\fR
+\fB\ volume log locate <VOLNAME> [BRICK] \fB
+Locate the log file for corresponding volume/brick.
 .TP
-\fB\ volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ... \fR
-set transport type for volume \fB\<VOLNAME>\fR
-\fR
+\fB\ volume log rotate <VOLNAME> [BRICK] \fB
+Rotate the log file for corresponding volume/brick.
 .TP
-\fB\ volume help \fB
-display help for the volume command
+\fB\ volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs] \fR
+Profile operations on the volume. Once started, volume profile <volname> info provides cumulative statistics of the FOPs performed.
 .TP
-\fB\ volume log filename <VOLNAME> [BRICK] <PATH> \fB
-set the log file for corresponding volume/brick
+\fB\ volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] | {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>] \fR
+Generates a profile of a volume representing the performance and bottlenecks/hotspots of each brick.
 .TP
-\fB\ volume log locate <VOLNAME> [BRICK] \fB
-locate the log file for corresponding volume/brick
+\fB\ volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|priv|fd|inode|history]... | [client <hostname:process-id>]] \fR
+Dumps the in memory state of the specified process or the bricks of the volume.
 .TP
-\fB\ volume log rotate <VOLNAME> [BRICK] \fB
-rotate the log file for corresponding volume/brick
+\fB\ volume sync <HOSTNAME> [all|<VOLNAME>] \fR
+Sync the volume information from a peer
+.SS "Peer Commands"
 .TP
 \fB\ peer probe <HOSTNAME> \fR
-probe peer specified by \fB\<HOSTNAME>\fR
+Probe the specified peer. In case the <HOSTNAME> given belongs to an already probed peer, the peer probe command will add the hostname to the peer if required.
 .TP
 \fB\ peer detach <HOSTNAME> \fR
-detach peer specified by \fB\<HOSTNAME>\fR
+Detach the specified peer.
 .TP
 \fB\ peer status \fR
-list status of peers
+Display the status of peers.
+.TP
+\fB\ pool list \fR
+List all the nodes in the pool (including localhost)
 .TP
 \fB\ peer help \fR
-Help command for peer 
+Display help for the peer command.
+.SS "Quota Commands"
+.TP
+\fB\ volume quota <VOLNAME> enable \fR
+Enable quota on the specified volume. This will cause all the directories in the filesystem hierarchy to be accounted and updated thereafter on each operation in the the filesystem. To kick start this accounting, a crawl is done over the hierarchy with an auxiliary client.
+.TP
+\fB\ volume quota <VOLNAME> disable \fR
+Disable quota on the volume. This will disable enforcement and accounting in the filesystem. Any configured limits will be lost.
+.TP
+\fB\ volume quota <VOLNAME> limit-usage <PATH> <SIZE> [<PERCENT>] \fR
+Set a usage  limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit.
+.TP
+\fB\ volume quota <VOLNAME> limit-objects <PATH> <SIZE> [<PERCENT>] \fR
+Set an inode limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit.
+.TP
+NOTE: valid units of SIZE are : B, KB, MB, GB, TB, PB. If no unit is specified, the unit defaults to bytes.
+.TP
+\fB\ volume quota <VOLNAME> remove <PATH> \fR
+Remove any usage limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced.
+.TP
+\fB\ volume quota <VOLNAME> remove-objects <PATH> \fR
+Remove any inode limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced.
+.TP
+\fB\ volume quota <VOLNAME> list <PATH> \fR
+Lists the  usage and limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's usage. If no path is given, usage and limits are displayed for all directories that has limits configured.
+.TP
+\fB\ volume quota <VOLNAME> list-objects <PATH> \fR
+Lists the inode usage and inode limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's inode usage. If no path is given, usage and limits are displayed for all directories that has limits configured.
+.TP
+\fB\ volume quota <VOLNAME> default-soft-limit <PERCENT> \fR
+Set the percentage value for default soft limit for the volume.
+.TP
+\fB\ volume quota <VOLNAME> soft-timeout <TIME> \fR
+Set the soft timeout for the volume. The interval in which limits are retested before the soft limit is breached.
+.TP
+\fB\ volume quota <VOLNAME> hard-timeout <TIME> \fR
+Set the hard timeout for the volume. The interval in which limits are retested after the soft limit is breached.
+.TP
+\fB\ volume quota <VOLNAME> alert-time <TIME> \fR
+Set the frequency in which warning messages need to be logged (in the brick logs) once soft limit is breached.
+.TP
+\fB\ volume inode-quota <VOLNAME> enable/disable \fR
+Enable/disable inode-quota for <VOLNAME>
+.TP
+\fB\ volume quota help \fR
+Display help for volume quota commands
+.TP
+NOTE: valid units of time and their symbols are : hours(h/hr), minutes(m/min), seconds(s/sec), weeks(w/wk), Days(d/days).
+.SS "Geo-replication Commands"
+.TP
+\fI\ Note\fR: password-less ssh, from the master node (where these commands are executed) to the slave node <SLAVE_HOST>, is a prerequisite for the geo-replication commands.
+.TP
+\fB\ system:: execute gsec_create\fR
+Generates pem keys which are required for push-pem
+.TP
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> create [[ssh-port n][[no-verify]|[push-pem]]] [force]\fR
+Create a new geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+Use ssh-port n if custom SSH port is configured in slave nodes.
+Use no-verify if the rsa-keys of nodes in master volume is distributed to slave nodes through an external agent.
+Use push-pem to push the keys automatically.
+.TP
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> {start|stop} [force] \fR
+Start/stop the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+.TP
+\fB\ volume geo-replication [<MASTER_VOL> [<SLAVE_HOST>::<SLAVE_VOL>]] status [detail] \fR
+Query status of the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+.TP
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> {pause|resume} [force] \fR
+Pause/resume the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+.TP
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> delete [reset-sync-time]\fR
+Delete the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>.
+Optionally you can also reset the sync time in case you need to resync the entire volume on session recreate.
+.TP
+\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> config [[!]<options> [<value>]] \fR
+View (when no option provided) or set configuration for this geo-replication session.
+Use "!<OPTION>" to reset option <OPTION> to default value.
+.SS "Bitrot Commands"
+.TP
+\fB\ volume bitrot <VOLNAME> {enable|disable} \fR
+Enable/disable bitrot for volume <VOLNAME>
+.TP
+\fB\ volume bitrot <VOLNAME> signing-time <time-in-secs> \fR
+Waiting time for an object after last fd is closed to start signing process.
+.TP
+\fB\ volume bitrot <VOLNAME> signer-threads <count> \fR
+Number of signing process threads. Usually set to number of available cores.
+.TP
+\fB\ volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive} \fR
+Scrub-throttle value is a measure of how fast or slow the scrubber scrubs the filesystem for volume <VOLNAME>
+.TP
+\fB\ volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly|monthly} \fR
+Scrub frequency for volume <VOLNAME>
+.TP
+\fB\ volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand} \fR
+Pause/Resume scrub. Upon resume, scrubber continues where it left off. status option shows the statistics of scrubber. ondemand option starts the scrubbing immediately if the scrubber is not paused or already running.
+.TP
+\fB\ volume bitrot help \fR
+Display help for volume bitrot commands
+.TP
+.SS "Snapshot Commands"
+.TP
+\fB\ snapshot create <snapname> <volname> [no-timestamp] [description <description>] [force] \fR
+Creates a snapshot of a GlusterFS volume. User can provide a snap-name and a description to identify the snap. Snap will be created by appending timestamp in GMT. User can override this behaviour using "no-timestamp" option. The description cannot be more than 1024 characters. To be able to take a snapshot, volume should be present and it should be in started state.
+.TP
+\fB\ snapshot restore <snapname> \fR
+Restores an already taken snapshot of a GlusterFS volume. Snapshot restore is an offline activity therefore if the volume is online (in started state) then the restore operation will fail. Once the snapshot is restored it will not be available in the list of snapshots.
+.TP
+\fB\ snapshot clone <clonename> <snapname> \fR
+Create a clone of a snapshot volume, the resulting volume will be GlusterFS volume. User can provide a clone-name. To be able to take a clone, snapshot should be present and it should be in activated state.
+.TP
+\fB\ snapshot delete ( all | <snapname> | volume <volname> ) \fR
+If snapname is specified then mentioned snapshot is deleted. If volname is specified then all snapshots belonging to that particular volume is deleted. If keyword *all* is used then all snapshots belonging to the system is deleted.
+.TP
+\fB\ snapshot list [volname] \fR
+Lists all snapshots taken. If volname is provided, then only the snapshots belonging to that particular volume is listed.
+.TP
+\fB\ snapshot info [snapname | (volume <volname>)] \fR
+This command gives information such as snapshot name, snapshot UUID, time at which snapshot was created, and it lists down the snap-volume-name, number of snapshots already taken and number of snapshots still available for that particular volume, and the state of the snapshot. If snapname is specified then info of the  mentioned  snapshot is  displayed.  If volname is specified then info of all snapshots belonging to that volume is displayed.  If  both  snapname and  volname  is  not specified then info of all the snapshots present in the system are displayed.
+.TP
+\fB\ snapshot status [snapname | (volume <volname>)] \fR
+This command gives status of the snapshot. The details included are snapshot brick path, volume group(LVM details), status of the snapshot bricks, PID of the bricks, data percentage filled for that particular volume group to which the snapshots belong to, and total size of the logical volume.
+
+If snapname is specified then status of the mentioned snapshot is displayed. If volname is specified then status of all snapshots belonging to that volume is displayed. If both snapname and volname is not specified then status of all the snapshots present in the system are displayed.
+.TP
+\fB\ snapshot config [volname] ([snap-max-hard-limit <count>] [snap-max-soft-limit <percent>]) | ([auto-delete <enable|disable>]) | ([activate-on-create <enable|disable>])
+Displays and sets the snapshot config values.
+
+snapshot config without any keywords displays the snapshot config values of all volumes in the system. If volname is provided, then the snapshot config values of that volume is displayed.
+
+Snapshot config command along with keywords can be used to change the existing config values. If volname is provided then config value of that volume is changed, else it will set/change the system limit.
+
+snap-max-soft-limit and auto-delete are global options, that will be inherited by all volumes in the system and cannot be set to individual volumes.
+
+snap-max-hard-limit can be set globally, as well as per volume. The lowest limit between the global system limit and the volume specific limit, becomes the
+"Effective snap-max-hard-limit" for a volume.
+
+snap-max-soft-limit is a percentage value, which is applied on the "Effective snap-max-hard-limit" to get the "Effective snap-max-soft-limit".
+
+When auto-delete feature is enabled, then upon reaching the "Effective snap-max-soft-limit", with every successful snapshot creation, the oldest snapshot will be deleted.
+
+When auto-delete feature is disabled, then upon reaching the "Effective snap-max-soft-limit", the user gets a warning with every successful snapshot creation.
+
+When auto-delete feature is disabled, then upon reaching the "Effective snap-max-hard-limit", further  snapshot  creations  will not be allowed.
+
+activate-on-create is disabled by default. If you enable activate-on-create, then further snapshot will be activated during the time of snapshot creation.
+.TP
+\fB\ snapshot activate <snapname> \fR
+Activates the mentioned snapshot.
+
+Note : By default the snapshot is activated during snapshot creation.
+.TP
+\fB\ snapshot deactivate <snapname> \fR
+Deactivates the mentioned snapshot.
+.TP
+\fB\ snapshot help \fR
+Display help for the snapshot commands.
+.SS "Self-heal Commands"
+.TP
+\fB\ volume heal <VOLNAME>\fR
+Triggers index self heal for the files that need healing.
+
+.TP
+\fB\ volume heal  <VOLNAME> [enable | disable]\fR
+Enable/disable self-heal-daemon for volume <VOLNAME>.
+
+.TP
+\fB\ volume heal <VOLNAME> full\fR
+Triggers self heal on all the files.
+
+.TP
+\fB\ volume heal <VOLNAME> info \fR
+Lists the files that need healing.
+
+.TP
+\fB\ volume heal <VOLNAME> info split-brain \fR
+Lists the files which are in split-brain state.
+
+.TP
+\fB\ volume heal <VOLNAME> statistics \fR
+Lists the crawl statistics.
+
+.TP
+\fB\ volume heal <VOLNAME> statistics heal-count \fR
+Displays the count of files to be healed.
+
+.TP
+\fB\ volume heal <VOLNAME> statistics heal-count replica <HOSTNAME:BRICKNAME> \fR
+Displays the number of files to be healed from a particular replica subvolume to which the brick <HOSTNAME:BRICKNAME> belongs.
+
+.TP
+\fB\ volume heal <VOLNAME> split-brain bigger-file <FILE> \fR
+Performs healing of <FILE> which is in split-brain by choosing the bigger file in the replica as source.
+
+.TP
+\fB\ volume heal <VOLNAME> split-brain source-brick <HOSTNAME:BRICKNAME> \fR
+Selects <HOSTNAME:BRICKNAME> as the source for all the files that are in split-brain in that replica and heals them.
+
+.TP
+\fB\ volume heal <VOLNAME> split-brain source-brick <HOSTNAME:BRICKNAME> <FILE> \fR
+Selects the split-brained <FILE> present in <HOSTNAME:BRICKNAME> as source and completes heal.
+.SS "Other Commands"
+.TP
+\fB\ get-state [<daemon>] [[odir </path/to/output/dir/>] [file <filename>]] [detail|volumeoptions] \fR
+Get local state representation of mentioned daemon and store data in provided path information
 .TP
 \fB\ help \fR
-display command options
+Display the command options.
 .TP
 \fB\ quit \fR
-quit
-.TP
-.PP
-.SH FILES
-/etc/glusterd/*
+Exit the gluster command line interface.
 
+.SH FILES
+/var/lib/glusterd/*
 .SH SEE ALSO
 .nf
-\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs-volgen\fR(8)
-\fBglusterfs\fR(8), \fBglusterd\fR(8)
+\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs\fR(8), \fBglusterd\fR(8)
 \fR
 .fi
 .SH COPYRIGHT
 .nf
-Copyright(c) 2006-2010  Gluster, Inc.  <http://www.gluster.com>
-
-.fi
+Copyright(c) 2006-2011  Gluster, Inc.  <http://www.gluster.com>
diff --git a/doc/glusterd.8 b/doc/glusterd.8
index ba714b2d4e4..e3768c78761 100644
--- a/doc/glusterd.8
+++ b/doc/glusterd.8
@@ -1,85 +1,67 @@
 .\"
-.\"  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is GF_FREE software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU Affero General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  Affero General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU Affero General Public License
-.\"  along with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 
-.TH Glusterd 8 "Gluster dynamic volume management Daemon" "9 September 2010" "Gluster Inc." 
+.TH Glusterd 8 "Gluster elastic volume management daemon" "07 March 2011" "Gluster Inc."
 .SH NAME
-Glusterd \- Gluster dynamic volume management Daemon.
+Glusterd \- Gluster elastic volume management daemon
 .SH SYNOPSIS
 .B glusterd
-.PP
-(or)
-.B glusterd
 .I [OPTION...]
-.PP
-Glusterd is a daemon for dynamic volume management for glusterfs.
 .SH DESCRIPTION
-Gluster daemon is used for dynamic volume management.
-The daemon must be executed on all of the export servers.
+The glusterd daemon is used for elastic volume management. The daemon must be run on all export servers.
 
 .SH OPTIONS
-.PP
-Mandatory or optional arguments to long options are also mandatory or optional
-for any corresponding short options.
+
 .SS "Basic options"
 .PP
 .TP
 
-\fB\-l, \fB\-\-log\-file=LOGFILE\fR
-File to use for logging [default:/usr/local/var/log/glusterfs/glusterfs.log]
+\fB\-l <LOGFILE>, \fB\-\-log\-file=<LOGFILE>\fR
+File to use for logging.
+.TP
+\fB\-L <LOGLEVEL>, \fB\-\-log\-level=<LOGLEVEL>\fR
+Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
 .TP
-\fB\-L, \fB\-\-log\-level=LOGLEVEL\fR
-Logging severity.  Valid options are TRACE, DEBUG, NORMAL, WARNING,
-ERROR and CRITICAL [default: NORMAL]
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
 .TP
 \fB\-\-debug\fR
-Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
-and \fB\-\-log\-file\fR to console
+Run the program in debug mode. This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
+and \fB\-\-log\-file\fR to console.
 .TP
 \fB\-N, \fB\-\-no\-daemon\fR
-Run in foreground
-.TP
+Run the program in the foreground.
 
-.TP
-.SS  Miscellaneous Options:
+.SS "Miscellaneous Options:"
 .TP
 \fB\-?, \fB\-\-help\fR
-Give this help list
+Display this help.
 .TP
 \fB\-\-usage\fR
-Give a short usage message
+Display a short usage message.
 .TP
 \fB\-V, \fB\-\-version\fR
-Print program version
+Print the program version.
 
 .PP
 .SH FILES
-/etc/glusterd/*
+/var/lib/glusterd/*
 
 .SH SEE ALSO
 .nf
-\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs-volgen\fR(8)
-\fBglusterfs\fR(8), \fBgluster\fR(8)
+\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs\fR(8), \fBgluster\fR(8)
 \fR
 .fi
 .SH COPYRIGHT
 .nf
-Copyright(c) 2006-2010  Gluster, Inc.  <http://www.gluster.com>
+Copyright(c) 2006-2011  Gluster, Inc.  <http://www.gluster.com>
 \fR
 .fi
diff --git a/doc/glusterd.vol b/doc/glusterd.vol
deleted file mode 100644
index 4e43fb0dad2..00000000000
--- a/doc/glusterd.vol
+++ /dev/null
@@ -1,8 +0,0 @@
-volume management
-    type mgmt/glusterd
-    option working-directory /etc/glusterd
-    option transport-type socket,rdma
-    option transport.socket.keepalive-time 10
-    option transport.socket.keepalive-interval 2
-end-volume
-
diff --git a/doc/glusterfs-volgen.8 b/doc/glusterfs-volgen.8
deleted file mode 100644
index a47873783bd..00000000000
--- a/doc/glusterfs-volgen.8
+++ /dev/null
@@ -1,82 +0,0 @@
-.\"  Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com>
-.\"  This file is part of GlusterFS.
-.\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU Affero General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  Affero General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU Affero General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
-.\"
-.\"
-.\"
-.TH GlusterFS 8 "Cluster Filesystem" "19 March 2010" "Gluster Inc."
-.SH NAME
-glusterfs-volgen  \- Tool to generate GlusterFS volume files
-.SH SYNOPSIS
-.B glusterfs-volgen --name <volumename>
-.I [options] <server1:/export/directory> <server1:/export/directory> ...
-.PP
-.SH DESCRIPTION
-glusterfs-volgen tool is used to generate glusterfs volume files as per the
-options given.
-
-GlusterFS volume files are very important component of gluster filesystem,
-using which \fBglusterfs\fR(8) will behave as a client component or a server
-component. Each glusterfs modules (in technical terms, a translator) is defined
-in this volume files in certain order to achieve the required behavior from
-filesystem.
-
-glusterfs-volgen tool abstracts the complexities of volumefile logic into a
-simple command line tool, which based on user's requirement (by options
-provided) generates the corresponding error-free volume file.
-
-
-.SH OPTIONS
-.PP
-
-\fB\-n \-\-name\fR=VOLUME-NAME
-This is a mandatory argument required for glusterfs-volgen.
-.TP
-\fB\-r \-\-raid\fR=[0|1]
-Supported options are 0 (stripe), 1 (mirror). If this option is not used,
-distribute is used without replicate or stripe.
-.TP
-\fB\-t \-\-transport\fR=[tcp|ib-verbs|tcp,ib-verbs]
-Supported types are 'tcp' and 'ib-verbs'. This option can take both types
-together like 'tcp,ib-verbs'. [default is 'tcp']
-.TP
-\fB\-p \-\-port\fR=PORT
-The port number to use in volume files.
-[default is 24007 for tcp, (24007 + 1) for ib-verbs]
-.TP
-\fB\-a \-\-auth\fR=AUTH_PARAMS
-Authentication parameters (comma seperated) to use in server volume.
-[default is '*', ie allow everyone]
-.TP
-\fB\-c \-\-conf-dir\fR=CONF_DIR
-Directory where the output volume files are generated. [default is
-${prefix}/etc/glusterfs]
-
-.PP
-.SH FILES
-
-${CONF_DIR}/*.vol
-
-.SH SEE ALSO
-.nf
-\fB\fBmount.glusterfs\fR(8)
-\fR
-.fi
-.SH COPYRIGHT
-.nf
-Copyright(c) 2009-2010  Gluster, Inc.  <http://www.gluster.com>
-\fR
-.fi
diff --git a/doc/glusterfs.8 b/doc/glusterfs.8
index 082b8074a28..3d359ea85e4 100644
--- a/doc/glusterfs.8
+++ b/doc/glusterfs.8
@@ -1,137 +1,181 @@
-.\"  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU Affero General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  Affero General Public License for more details.
 .\"
-.\"  You should have received a copy of the GNU Affero General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
 .\"
-.\"
-.\"
-.TH GlusterFS 8 "Cluster Filesystem" "19 March 2010" "Gluster Inc."
+.TH GlusterFS 8 "Clustered File System" "07 March 2011" "Gluster Inc."
 .SH NAME
-GlusterFS \- Clustered Filesystem.
+GlusterFS \- clustered file system
 .SH SYNOPSIS
 .B glusterfs
 .I [options] [mountpoint]
 .PP
 .SH DESCRIPTION
-GlusterFS is a clustered file-system capable of scaling to several peta-bytes.
-It aggregates various storage bricks over Infiniband RDMA or TCP/IP
+GlusterFS is a clustered file system, capable of scaling to several peta-bytes.
+It aggregates various storage bricks over Infiniband RDMA or TCP/IP and
 interconnect into one large parallel network file system. Storage bricks can
-be made of any commodity hardware such as x86-64 server with SATA-II RAID and
+be made of any commodity hardware, such as x86-64 server with SATA-II RAID and
 Infiniband HBA.
 
-GlusterFS is fully POSIX compliant FileSystem. On client side, it has dependency
-on FUSE package, on server side, it works seemlessly on different OSes.
-(Currently supported on GNU/Linux, Solaris).
+GlusterFS is fully POSIX compliant file system. On client side, it has dependency
+on FUSE package, on server side, it works seemlessly on different operating systems.
+Currently supported on GNU/Linux and Solaris.
 
 .SH OPTIONS
-.PP
-Mandatory or optional arguments to long options are also mandatory or optional
-for any corresponding short options.
+
 .SS "Basic options"
 .PP
 .TP
-
 \fB\-f, \fB\-\-volfile=VOLUME-FILE\fR
-File to use as VOLUME-FILE [default:/etc/glusterfs/glusterfs.vol]
+File to use as VOLUME-FILE.
 .TP
 \fB\-l, \fB\-\-log\-file=LOGFILE\fR
-File to use for logging [default:/var/log/glusterfs/glusterfs.log]
+File to use for logging (the default is <INSTALL-DIR>/var/log/glusterfs/<MOUNT-POINT>.log).
 .TP
 \fB\-L, \fB\-\-log\-level=LOGLEVEL\fR
-Logging severity.  Valid options are TRACE, DEBUG, NORMAL, WARNING, ERROR and
-CRITICAL [default: WARNING]
+Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
 .TP
 \fB\-s, \fB\-\-volfile\-server=SERVER\fR
-Server to get the volume from.  This option overrides \fB\-\-volfile option
+Server to get the volume from.  This option overrides \fB\-\-volfile \fR option.
 .TP
-\fB\-\-log\-server=LOG\-SERVER\fR
-Server to use as the central log server.
+\fB\-\-volfile\-max\-fetch\-attempts=MAX\-ATTEMPTS\fR
+Maximum number of connect attempts to server. This option should be provided with
+\fB\-\-volfile\-server\fR option (the default is 1).
 
 .SS "Advanced options"
 .PP
 .TP
-
+\fB\-\-acl\fR
+Mount the filesystem with POSIX ACL support.
+.TP
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
+.TP
 \fB\-\-debug\fR
-Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
-and \fB\-\-log\-file\fR to console
+Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG,
+and \fB\-\-log\-file\fR to console.
+.TP
+\fB\-\-enable\-ino32=BOOL\fR
+Use 32-bit inodes when mounting to workaround application that doesn't support 64-bit inodes.
+.TP
+\fB\-\-fopen\-keep\-cache[=BOOL]\fR
+Do not purge the cache on file open (default: false).
+.TP
+\fB\-\-mac\-compat=BOOL\fR
+Provide stubs for attributes needed for seamless operation on Macs (the default is off).
 .TP
 \fB\-N, \fB\-\-no\-daemon\fR
-Run in foreground
+Run in the foreground.
+.TP
+\fB\-p, \fB\-\-pid\-file=PIDFILE\fR
+File to use as PID file.
 .TP
 \fB\-\-read\-only\fR
-Makes the filesystem read-only
+Mount the file system in 'read-only' mode.
 .TP
-\fB\-p, \fB\-\-pid\-file=PIDFILE\fR
-File to use as pid file
+\fB\-\-selinux\fR
+Enable SELinux label (extended attributes) support on inodes.
+.TP
+\fB\-S, \fB\-\-socket\-file=SOCKFILE\fR
+File to use as unix-socket.
 .TP
 \fB\-\-volfile\-id=KEY\fR
-KEY of the volume file to be fetched from server
+Key of the volume file to be fetched from the server.
 .TP
 \fB\-\-volfile\-server\-port=PORT\fR
-Port number of volfile server
+Port number of volfile server.
 .TP
 \fB\-\-volfile\-server\-transport=TRANSPORT\fR
-Transport type to get volume file from server [default: socket]
+Transport type to get volume file from server (the default is tcp).
 .TP
 \fB\-\-volume\-name=VOLUME\-NAME\fR
-Volume name to be used for MOUNT-POINT [default: top most volume in
-VOLUME-FILE]
+Volume name to be used for MOUNT-POINT (the default is top most volume in VOLUME-FILE).
+.TP
+\fB\-\-worm\fR
+Mount the filesystem in 'worm' mode.
 .TP
 \fB\-\-xlator\-option=VOLUME\-NAME.OPTION=VALUE\fR
-Add/override a translator option for a volume with the specified value
+Add/Override a translator option for a volume with the specified value.
 .TP
-\fB\-\-log\-server\-port=PORT\fR
-Listening port number of log server
+\fB\-\-subdir\-mount=SUBDIR\-MOUNT\-PATH\fR
+Mount subdirectory instead of the '/' of volume.
 
 .SS "Fuse options"
 .PP
 .TP
 
+\fB\-\-attr\-times\-granularity=NANOSECONDS\fR
+Declare supported granularity of file attribute times (default is 0 which kernel handles as unspecified; valid real values are between 1 and 1000000000).
+.TP
 \fB\-\-attribute\-timeout=SECONDS\fR
-Set attribute timeout to SECONDS for inodes in fuse kernel module [default: 1]
+Set attribute timeout to SECONDS for inodes in fuse kernel module (the default is 1).
+.TP
+\fB\-\-background\-qlen=N\fR
+Set fuse module's background queue length to N (the default is 64).
+.TP
+\fB\-\-congestion\-threshold=N\fR
+Set fuse module's congestion threshold to N (the default is 48).
+.TP
+\fB\-\-direct\-io\-mode=BOOL|auto\fR
+Specify fuse direct I/O strategy (the default is auto).
+.TP
+\fB\-\-dump-fuse=PATH\f\R
+Dump fuse traffic to PATH
 .TP
 \fB\-\-entry\-timeout=SECONDS\fR
-Set entry timeout to SECONDS in fuse kernel module [default: 1]
+Set entry timeout to SECONDS in fuse kernel module (the default is 1).
+.TP
+\fB\-\-gid\-timeout=SECONDS\fR
+Set auxiliary group list timeout to SECONDS for fuse translator (the default is 0).
+.TP
+\fB\-\-kernel-writeback-cache=BOOL\fR
+Enable fuse in-kernel writeback cache.
 .TP
-\fB\-\-disable\-direct\-io\-mode\fR
-Disable direct I/O mode in fuse kernel module
+\fB\-\-negative\-timeout=SECONDS\fR
+Set negative timeout to SECONDS in fuse kernel module (the default is 0).
+.TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on].
+.TP
+\fB\-\-volfile-check\fR
+Enable strict volume file checking.
 
 .SS "Miscellaneous Options"
 .PP
 .TP
 
 \fB\-?, \fB\-\-help\fR
-Give this help list
+Display this help.
 .TP
 \fB\-\-usage\fR
-Give a short usage message
+Display a short usage message.
 .TP
 \fB\-V, \fB\-\-version\fR
-Print program version
+Print the program version.
 
 .PP
 .SH FILES
-/etc/glusterfs/*.vol
+/var/lib/glusterd/vols/*/*.vol
+.SH EXAMPLES
+mount a volume named foo on server bar with log level DEBUG on mount point
+/mnt/foo
+
+# glusterfs \-\-log\-level=DEBUG \-\-volfile\-id=foo \-\-volfile\-server=bar /mnt/foo
 
 .SH SEE ALSO
 .nf
-\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs-volgen\fR(8)
+\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBgluster\fR(8)
 \fR
 .fi
 .SH COPYRIGHT
 .nf
-Copyright(c) 2006-2010  Gluster, Inc.  <http://www.gluster.com>
+Copyright(c) 2006-2011  Red Hat, Inc.  <http://www.redhat.com>
 \fR
 .fi
diff --git a/doc/glusterfs.vol.sample b/doc/glusterfs.vol.sample
deleted file mode 100644
index 3b1f1851722..00000000000
--- a/doc/glusterfs.vol.sample
+++ /dev/null
@@ -1,61 +0,0 @@
-### file: client-volume.vol.sample
-
-#####################################
-###  GlusterFS Client Volume File  ##
-#####################################
-
-#### CONFIG FILE RULES:
-### "#" is comment character.
-### - Config file is case sensitive
-### - Options within a volume block can be in any order.
-### - Spaces or tabs are used as delimitter within a line. 
-### - Each option should end within a line.
-### - Missing or commented fields will assume default values.
-### - Blank/commented lines are allowed.
-### - Sub-volumes should already be defined above before referring.
-
-### Add client feature and attach to remote subvolume
-volume client
-  type protocol/client
-  option transport-type tcp
-# option transport-type unix
-# option transport-type ib-sdp
-  option remote-host 127.0.0.1         # IP address of the remote brick
-# option transport.socket.remote-port 24016              
-
-# option transport-type ib-verbs
-# option transport.ib-verbs.remote-port 24016              
-# option transport.ib-verbs.work-request-send-size  1048576
-# option transport.ib-verbs.work-request-send-count 16
-# option transport.ib-verbs.work-request-recv-size  1048576
-# option transport.ib-verbs.work-request-recv-count 16
-
-# option transport-timeout 30          # seconds to wait for a reply
-                                       # from server for each request
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-### Add readahead feature
-#volume readahead
-#  type performance/read-ahead
-#  option page-size 1MB     # unit in bytes
-#  option page-count 2       # cache per file  = (page-count x page-size)
-#  subvolumes client
-#end-volume
-
-### Add IO-Cache feature
-#volume iocache
-#  type performance/io-cache
-#  option page-size 256KB
-#  option page-count 2
-#  subvolumes readahead
-#end-volume
-
-### Add writeback feature
-#volume writeback
-#  type performance/write-behind
-#  option aggregate-size 1MB
-#  option window-size 2MB
-#  option flush-behind off
-#  subvolumes iocache   
-#end-volume
diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
new file mode 100644
index 00000000000..bc1de2a8c80
--- /dev/null
+++ b/doc/glusterfsd.8
@@ -0,0 +1,147 @@
+.\"  Copyright (c) 20088888888-2012 Red Hat, Inc. <http://www.redhat.com>
+.\"  This file is part of GlusterFS.
+.\"
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
+.\"
+.\"
+.\"
+.TH GlusterFS 8 "Cluster Filesystem" "19 March 2010" "Gluster Inc."
+.SH NAME
+GlusterFS \- Clustered Filesystem.
+.SH SYNOPSIS
+.B glusterfsd
+.I [options] [mountpoint]
+.PP
+.SH DESCRIPTION
+GlusterFS is a clustered file-system capable of scaling to several peta-bytes.
+It aggregates various storage bricks over Infiniband RDMA or TCP/IP
+interconnect into one large parallel network file system. Storage bricks can
+be made of any commodity hardware such as x86-64 server with SATA-II RAID and
+Infiniband HBA.
+
+GlusterFS is fully POSIX compliant FileSystem. On client side, it has dependency
+on FUSE package, on server side, it works seemlessly on different OSes.
+(Currently supported on GNU/Linux, Solaris).
+
+.SH OPTIONS
+.PP
+Mandatory or optional arguments to long options are also mandatory or optional
+for any corresponding short options.
+.SS "Basic options"
+.PP
+.TP
+
+\fB\-f, \fB\-\-volfile=VOLUME-FILE\fR
+File to use as VOLUME-FILE [default:/etc/glusterfs/glusterfs.vol]
+.TP
+\fB\-l, \fB\-\-log\-file=LOGFILE\fR
+File to use for logging [default:/var/log/glusterfs/glusterfs.log]
+.TP
+\fB\-L, \fB\-\-log\-level=LOGLEVEL\fR
+Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and
+CRITICAL [default: WARNING]
+.TP
+\fB\-s, \fB\-\-volfile\-server=SERVER\fR
+Server to get the volume from.  This option overrides \fB\-\-volfile option
+
+.SS "Advanced options"
+.PP
+.TP
+
+\fB\-\-localtime\-logging\fR
+Enable localtime log timestamps.
+.TP
+\fB\-\-debug\fR
+Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG
+and \fB\-\-log\-file\fR to console
+.TP
+\fB\-N, \fB\-\-no\-daemon\fR
+Run in foreground
+.TP
+\fB\-\-read\-only\fR
+Makes the filesystem read-only
+.TP
+\fB\-p, \fB\-\-pid\-file=PIDFILE\fR
+File to use as pid file
+.TP
+\fB\-S SOCKFILE
+Socket file to used for inter-process communication
+.TP
+\fB\-\-brick\-name DIRECTORY
+Directory to be used as export directory for GlusterFS
+.TP
+\fB\-\-brick\-port PORT
+Brick Port to be registered with Gluster portmapper
+.TP
+\fB\-\-volfile\-id=KEY\fR
+KEY of the volume file to be fetched from server
+.TP
+\fB\-\-volfile\-server\-port=PORT\fR
+Port number of volfile server
+.TP
+\fB\-\-volfile\-server\-transport=TRANSPORT\fR
+Transport type to get volume file from server [default: tcp]
+.TP
+\fB\-\-volume\-name=VOLUME\-NAME\fR
+Volume name to be used for MOUNT-POINT [default: top most volume in
+VOLUME-FILE]
+.TP
+\fB\-\-xlator\-option=VOLUME\-NAME.OPTION=VALUE\fR
+Add/override a translator option for a volume with the specified value
+
+.SS "Fuse options"
+.PP
+.TP
+
+\fB\-\-attribute\-timeout=SECONDS\fR
+Set attribute timeout to SECONDS for inodes in fuse kernel module [default: 1]
+.TP
+\fB\-\-entry\-timeout=SECONDS\fR
+Set entry timeout to SECONDS in fuse kernel module [default: 1]
+.TP
+\fB\-\-direct\-io\-mode=BOOL\fR
+Enable/Disable direct-io mode in fuse module [default: enable]
+.TP
+\fB\-\-resolve-gids\fR
+Resolve all auxiliary groups in fuse translator (max 32 otherwise)
+.TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
+
+.SS "Miscellaneous Options"
+.PP
+.TP
+
+\fB\-?, \fB\-\-help\fR
+Give this help list
+.TP
+\fB\-\-usage\fR
+Give a short usage message
+.TP
+\fB\-V, \fB\-\-version\fR
+Print program version
+
+.PP
+.SH FILES
+/etc/glusterfs/*.vol
+
+.SH EXAMPLES
+Start a GlusterFS server on localhost with volume name foo
+
+glusterfsd \-s localhost \-\-volfile\-id foo.server.media-disk\-1 \-p /var/lib/glusterd/vols/foo/run/server\-media\-disk\-1.pid \-S /tmp/<uniqueid>.socket \-\-brick-name /media/disk\-1 \-l /var/log/glusterfs/bricks/media\-disk\-1.log \-\-brick\-port 24009 \-\-xlator\-option foo\-server.listen-port=24009
+
+.SH SEE ALSO
+.nf
+\fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBgluster\fR(8)
+\fR
+.fi
+.SH COPYRIGHT
+.nf
+Copyright(c) 2006-2011  Gluster, Inc.  <http://www.gluster.com>
+\fR
+.fi
diff --git a/doc/glusterfsd.vol.sample b/doc/glusterfsd.vol.sample
deleted file mode 100644
index e91df3290a9..00000000000
--- a/doc/glusterfsd.vol.sample
+++ /dev/null
@@ -1,47 +0,0 @@
-### file: server-volume.vol.sample
-
-#####################################
-###  GlusterFS Server Volume File  ##
-#####################################
-
-#### CONFIG FILE RULES:
-### "#" is comment character.
-### - Config file is case sensitive
-### - Options within a volume block can be in any order.
-### - Spaces or tabs are used as delimitter within a line. 
-### - Multiple values to options will be : delimitted.
-### - Each option should end within a line.
-### - Missing or commented fields will assume default values.
-### - Blank/commented lines are allowed.
-### - Sub-volumes should already be defined above before referring.
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp
-# option transport-type unix
-# option transport-type ib-sdp
-# option transport.socket.bind-address 192.168.1.10     # Default is to listen on all interfaces
-# option transport.socket.listen-port 24016              
-
-# option transport-type ib-verbs
-# option transport.ib-verbs.bind-address 192.168.1.10     # Default is to listen on all interfaces
-# option transport.ib-verbs.listen-port 24016              
-# option transport.ib-verbs.work-request-send-size  131072
-# option transport.ib-verbs.work-request-send-count 64
-# option transport.ib-verbs.work-request-recv-size  131072
-# option transport.ib-verbs.work-request-recv-count 64
-
-# option client-volume-filename /etc/glusterfs/glusterfs-client.vol
-  subvolumes brick
-# NOTE: Access to any volume through protocol/server is denied by
-# default. You need to explicitly grant access through # "auth"
-# option.
-  option auth.addr.brick.allow * # Allow access to "brick" volume
-end-volume
diff --git a/doc/hacker-guide/Makefile.am b/doc/hacker-guide/Makefile.am
deleted file mode 100644
index 65c92ac235e..00000000000
--- a/doc/hacker-guide/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-EXTRA_DIST = replicate.txt bdb.txt posix.txt call-stub.txt write-behind.txt
-
-#EXTRA_DIST = hacker-guide.tex afr.txt bdb.txt posix.txt call-stub.txt write-behind.txt
-#hacker_guidedir = $(docdir)
-#hacker_guide_DATA = hacker-guide.pdf
-
-#hacker-guide.pdf: $(EXTRA_DIST)
-#	pdflatex $(srcdir)/hacker-guide.tex
diff --git a/doc/hacker-guide/adding-fops.txt b/doc/hacker-guide/adding-fops.txt
deleted file mode 100644
index e70dbbdc829..00000000000
--- a/doc/hacker-guide/adding-fops.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-		  HOW TO ADD A NEW FOP TO GlusterFS
-		  =================================
-
-Steps to be followed when adding a new FOP to GlusterFS:
-
-1. Edit glusterfs.h and add a GF_FOP_* constant.
-
-2. Edit xlator.[ch] and:
-   2a. add the new prototype for fop and callback.
-   2b. edit xlator_fops structure.
-
-3. Edit xlator.c and add to fill_defaults.
-
-4. Edit protocol.h and add struct necessary for the new FOP.
-
-5. Edit defaults.[ch] and provide default implementation.
-
-6. Edit call-stub.[ch] and provide stub implementation.
-
-7. Edit common-utils.c and add to gf_global_variable_init().
-
-8. Edit client-protocol and add your FOP.
-
-9. Edit server-protocol and add your FOP.
-
-10. Implement your FOP in any translator for which the default implementation
-    is not sufficient.
-
-==========================================
-Last updated: Mon Oct 27 21:35:49 IST 2008
-
-Author: Vikas Gorur <vikas@gluster.com>
-==========================================
diff --git a/doc/hacker-guide/bdb.txt b/doc/hacker-guide/bdb.txt
deleted file mode 100644
index fd0bd365206..00000000000
--- a/doc/hacker-guide/bdb.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-
-* How does file translates to key/value pair?
----------------------------------------------
-
-  in bdb a file is identified by key (obtained by taking basename() of the path of
-the file) and file contents are stored as value corresponding to the key in database
-file (defaults to glusterfs_storage.db under dirname() directory).
-
-* symlinks, directories
------------------------
-
-  symlinks and directories are stored as is.
-
-* db (database) files
----------------------
-
-  every directory, including root directory, contains a database file called
-glusterfs_storage.db. all the regular files contained in the directory are stored
-as key/value pair inside the glusterfs_storage.db.
-
-* internal data cache
----------------------
-
-  db does not provide a way to find out the size of the value corresponding to a key. 
-so, bdb makes DB->get() call for key and takes the length of the value returned. 
-since DB->get() also returns file contents for key, bdb maintains an internal cache and
-stores the file contents in the cache.
-  every directory maintains a seperate cache.
-  
-* inode number transformation
------------------------------
-
-  bdb allocates a inode number to each file and directory on its own. bdb maintains a 
-global counter and increments it after allocating inode number for each file
-(regular, symlink or directory). NOTE: bdb does not guarantee persistent inode numbers.
-
-* checkpoint thread
--------------------
-
-  bdb creates a checkpoint thread at the time of init(). checkpoint thread does a 
-periodic checkpoint on the DB_ENV. checkpoint is the mechanism, provided by db, to 
-forcefully commit the logged transactions to the storage.
-
-NOTES ABOUT FOPS:
------------------
-
-lookup() -
- 1> do lstat() on the path, if lstat fails, we assume that the file being looked up
-    is either a regular file or doesn't exist.
- 2> lookup in the DB of parent directory for key corresponding to path. if key exists,
-    return key, with. 
-    NOTE: 'struct stat' stat()ed from DB file is used as a container for 'struct stat'
-           of the regular file. st_ino, st_size, st_blocks are updated with file's values.
-
-readv() -
- 1> do a lookup in bctx cache. if successful, return the requested data from cache.
- 2> if cache missed, do a DB->get() the entire file content and insert to cache.
-
-writev():
- 1> flush any cached content of this file.
- 2> do a DB->put(), with DB_DBT_PARTIAL flag. 
-    NOTE: DB_DBT_PARTIAL is used to do partial update of a value in DB.
-
-readdir():
- 1> regular readdir() in a loop, and vomit all DB_ENV log files and DB files that
-    we encounter.
- 2> if the readdir() buffer still has space, open a DB cursor and do a sequential
-    DBC->get() to fill the reaadir buffer.
-
-
diff --git a/doc/hacker-guide/call-stub.txt b/doc/hacker-guide/call-stub.txt
deleted file mode 100644
index bca1579b22d..00000000000
--- a/doc/hacker-guide/call-stub.txt
+++ /dev/null
@@ -1,1033 +0,0 @@
-creating a call stub and pausing a call
----------------------------------------
-libglusterfs provides seperate API to pause each of the fop. parameters to each API is
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-       NOTE: @fn should exactly take the same type and number of parameters that 
-             the corresponding regular fop takes.
-rest will be the regular parameters to corresponding fop.
-
-NOTE: @frame can never be NULL. fop_<operation>_stub() fails with errno
-      set to EINVAL, if @frame is NULL. also wherever @loc is applicable,
-      @loc cannot be NULL.
-
-refer to individual stub creation API to know about call-stub creation's behaviour with
-specific parameters.
-
-here is the list of stub creation APIs for xlator fops.
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@loc       - pointer to location structure.
-             NOTE: @loc will be copied to a different location, with inode_ref() to
-	           @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-		   copied to a different location.
-@need_xattr - flag to specify if xattr should be returned or not.
-call_stub_t *
-fop_lookup_stub (call_frame_t *frame,
-		 fop_lookup_t fn,
-		 loc_t *loc,
-		 int32_t need_xattr);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-call_stub_t *
-fop_stat_stub (call_frame_t *frame,
-	       fop_stat_t fn,
-	       loc_t *loc);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-call_stub_t *
-fop_fstat_stub (call_frame_t *frame,
-		fop_fstat_t fn,
-		fd_t *fd);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
-	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
-@mode  - mode parameter to chmod.
-call_stub_t *
-fop_chmod_stub (call_frame_t *frame,
-		fop_chmod_t fn,
-		loc_t *loc,
-		mode_t mode);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-@mode  - mode parameter for fchmod fop.
-call_stub_t *
-fop_fchmod_stub (call_frame_t *frame,
-		 fop_fchmod_t fn,
-		 fd_t *fd,
-		 mode_t mode);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
-	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
-@uid   - uid parameter to chown.
-@gid   - gid parameter to chown.
-call_stub_t *
-fop_chown_stub (call_frame_t *frame,
-		fop_chown_t fn,
-		loc_t *loc,
-		uid_t uid,
-		gid_t gid);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-@uid   - uid parameter to fchown.
-@gid   - gid parameter to fchown.
-call_stub_t *
-fop_fchown_stub (call_frame_t *frame,
-		 fop_fchown_t fn,
-		 fd_t *fd,
-		 uid_t uid,
-		 gid_t gid);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location, if not NULL.
-@off   - offset parameter to truncate fop.
-call_stub_t *
-fop_truncate_stub (call_frame_t *frame,
-		   fop_truncate_t fn,
-		   loc_t *loc,
-		   off_t off);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-@off   - offset parameter to ftruncate fop.
-call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame,
-		    fop_ftruncate_t fn,
-		    fd_t *fd,
-		    off_t off);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@tv    - tv parameter to utimens fop.
-call_stub_t *
-fop_utimens_stub (call_frame_t *frame,
-		  fop_utimens_t fn,
-		  loc_t *loc,
-		  struct timespec tv[2]);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@mask  - mask parameter for access fop.
-call_stub_t *
-fop_access_stub (call_frame_t *frame,
-		 fop_access_t fn,
-		 loc_t *loc,
-		 int32_t mask);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@size  - size parameter to readlink fop.
-call_stub_t *
-fop_readlink_stub (call_frame_t *frame,
-		   fop_readlink_t fn,
-		   loc_t *loc,
-		   size_t size);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@mode  - mode parameter to mknod fop.
-@rdev  - rdev parameter to mknod fop.
-call_stub_t *
-fop_mknod_stub (call_frame_t *frame,
-		fop_mknod_t fn,
-		loc_t *loc,
-		mode_t mode,
-		dev_t rdev);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@mode  - mode parameter to mkdir fop.
-call_stub_t *
-fop_mkdir_stub (call_frame_t *frame,
-		fop_mkdir_t fn,
-		loc_t *loc,
-		mode_t mode);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-call_stub_t *
-fop_unlink_stub (call_frame_t *frame,
-		 fop_unlink_t fn,
-		 loc_t *loc);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-call_stub_t *
-fop_rmdir_stub (call_frame_t *frame,
-		fop_rmdir_t fn,
-		loc_t *loc);
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@linkname - linkname parameter to symlink fop.
-@loc      - pointer to location structure.
-            NOTE: @loc will be copied to a different location, with inode_ref() to
-	          @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-		  copied to a different location.
-call_stub_t *
-fop_symlink_stub (call_frame_t *frame,
-		  fop_symlink_t fn,
-		  const char *linkname,
-		  loc_t *loc);
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@oldloc   - pointer to location structure.
-            NOTE: @oldloc will be copied to a different location, with inode_ref() to 
-	          @oldloc->inode and @oldloc->parent, if not NULL. also @oldloc->path will 
-		  be copied to a different location, if not NULL.
-@newloc   - pointer to location structure.
-            NOTE: @newloc will be copied to a different location, with inode_ref() to
-	          @newloc->inode and @newloc->parent, if not NULL. also @newloc->path will
-		  be copied to a different location, if not NULL.
-call_stub_t *
-fop_rename_stub (call_frame_t *frame,
-		 fop_rename_t fn,
-		 loc_t *oldloc,
-		 loc_t *newloc);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc     - pointer to location structure.
-           NOTE: @loc will be copied to a different location, with inode_ref() to
-	         @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-		 copied to a different location.
-@newpath - newpath parameter to link fop.
-call_stub_t *
-fop_link_stub (call_frame_t *frame,
-	       fop_link_t fn,
-	       loc_t *oldloc,
-	       const char *newpath);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@flags - flags parameter to create fop.
-@mode  - mode parameter to create fop.
-@fd    - file descriptor parameter to create fop.
-         NOTE: @fd is stored with a fd_ref().
-call_stub_t *
-fop_create_stub (call_frame_t *frame,
-		 fop_create_t fn,
-		 loc_t *loc,
-		 int32_t flags,
-		 mode_t mode, fd_t *fd);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@flags - flags parameter to open fop.
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-call_stub_t *
-fop_open_stub (call_frame_t *frame,
-	       fop_open_t fn,
-	       loc_t *loc,
-	       int32_t flags,
-	       fd_t *fd);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-@size  - size parameter to readv fop.
-@off   - offset parameter to readv fop.
-call_stub_t *
-fop_readv_stub (call_frame_t *frame,
-		fop_readv_t fn,
-		fd_t *fd,
-		size_t size,
-		off_t off);
-
-@frame  - call frame which has to be used to resume the call at call_resume().
-@fn     - procedure to call during call_resume(). 
-@fd     - file descriptor parameter to lk fop.
-          NOTE: @fd is stored with a fd_ref().
-@vector - vector parameter to writev fop. 
-	  NOTE: @vector is iov_dup()ed while creating stub. and frame->root->req_refs
-                dictionary is dict_ref()ed.
-@count  - count parameter to writev fop.
-@off    - off parameter to writev fop.
-call_stub_t *
-fop_writev_stub (call_frame_t *frame,
-		 fop_writev_t fn,
-		 fd_t *fd,
-		 struct iovec *vector,
-		 int32_t count,
-		 off_t off);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to flush fop.
-         NOTE: @fd is stored with a fd_ref().
-call_stub_t *
-fop_flush_stub (call_frame_t *frame,
-		fop_flush_t fn,
-		fd_t *fd);
-
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@fd       - file descriptor parameter to lk fop.
-            NOTE: @fd is stored with a fd_ref().
-@datasync - datasync parameter to fsync fop.
-call_stub_t *
-fop_fsync_stub (call_frame_t *frame,
-		fop_fsync_t fn,
-		fd_t *fd,
-		int32_t datasync);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
-	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
-@fd    - file descriptor parameter to opendir fop.
-         NOTE: @fd is stored with a fd_ref().
-call_stub_t *
-fop_opendir_stub (call_frame_t *frame,
-		  fop_opendir_t fn,
-		  loc_t *loc, 
-		  fd_t *fd);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to getdents fop.
-         NOTE: @fd is stored with a fd_ref().
-@size  - size parameter to getdents fop.
-@off   - off parameter to getdents fop.
-@flags - flags parameter to getdents fop.
-call_stub_t *
-fop_getdents_stub (call_frame_t *frame,
-		   fop_getdents_t fn,
-		   fd_t *fd,
-		   size_t size,
-		   off_t off,
-		   int32_t flag);
-
-@frame   - call frame which has to be used to resume the call at call_resume().
-@fn      - procedure to call during call_resume(). 
-@fd      - file descriptor parameter to setdents fop.
-           NOTE: @fd is stored with a fd_ref().
-@flags   - flags parameter to setdents fop.
-@entries - entries parameter to setdents fop.
-call_stub_t *
-fop_setdents_stub (call_frame_t *frame,
-		   fop_setdents_t fn,
-		   fd_t *fd,
-		   int32_t flags,
-		   dir_entry_t *entries,
-		   int32_t count);
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@fd       - file descriptor parameter to setdents fop.
-            NOTE: @fd is stored with a fd_ref().
-@datasync - datasync parameter to fsyncdir fop.
-call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame,
-		   fop_fsyncdir_t fn,
-		   fd_t *fd,
-		   int32_t datasync);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-call_stub_t *
-fop_statfs_stub (call_frame_t *frame,
-		 fop_statfs_t fn,
-		 loc_t *loc);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to 
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@dict  - dict parameter to setxattr fop.
-         NOTE: stub creation procedure stores @dict pointer with dict_ref() to it.
-call_stub_t *
-fop_setxattr_stub (call_frame_t *frame,
-		   fop_setxattr_t fn,
-		   loc_t *loc,
-		   dict_t *dict,
-		   int32_t flags);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@name  - name parameter to getxattr fop.
-call_stub_t *
-fop_getxattr_stub (call_frame_t *frame,
-		   fop_getxattr_t fn,
-		   loc_t *loc,
-		   const char *name);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@name  - name parameter to removexattr fop.
-         NOTE: name string will be copied to a different location while creating stub.
-call_stub_t *
-fop_removexattr_stub (call_frame_t *frame,
-		      fop_removexattr_t fn,
-		      loc_t *loc,
-		      const char *name);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to lk fop.
-         NOTE: @fd is stored with a fd_ref().
-@cmd   - command parameter to lk fop.
-@lock  - lock parameter to lk fop.
-         NOTE: lock will be copied to a different location while creating stub.
-call_stub_t *
-fop_lk_stub (call_frame_t *frame,
-	     fop_lk_t fn,
-	     fd_t *fd,
-	     int32_t cmd,
-	     struct flock *lock);
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@fd       - fd parameter to gf_lk fop.
-	    NOTE: @fd is fd_ref()ed while creating stub, if not NULL.
-@cmd      - cmd parameter to gf_lk fop.
-@lock     - lock paramater to gf_lk fop.
-	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
-call_stub_t *
-fop_gf_lk_stub (call_frame_t *frame,
-		fop_gf_lk_t fn,
-		fd_t *fd,
-		int32_t cmd,
-		struct flock *lock);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@fd    - file descriptor parameter to readdir fop.
-         NOTE: @fd is stored with a fd_ref().
-@size  - size parameter to readdir fop.
-@off   - offset parameter to readdir fop.
-call_stub_t *
-fop_readdir_stub (call_frame_t *frame,
-		  fop_readdir_t fn,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off);
-
-@frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-@loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to
-	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
-	       copied to a different location.
-@flags - flags parameter to checksum fop.
-call_stub_t *
-fop_checksum_stub (call_frame_t *frame,
-		   fop_checksum_t fn,
-		   loc_t *loc,
-		   int32_t flags);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-@dict      - dict parameter to @fn.
-	     NOTE: @dict pointer is stored with dict_ref().
-call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame,
-		     fop_lookup_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct stat *buf,
-		     dict_t *dict);
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame,
-		   fop_stat_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct stat *buf);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame,
-		    fop_fstat_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_chmod_cbk_stub (call_frame_t *frame,
-		    fop_chmod_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_fchmod_cbk_stub (call_frame_t *frame,
-		     fop_fchmod_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_chown_cbk_stub (call_frame_t *frame,
-		    fop_chown_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_fchown_cbk_stub (call_frame_t *frame,
-		     fop_fchown_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame,
-		       fop_truncate_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame,
-			fop_ftruncate_cbk_t fn,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_utimens_cbk_stub (call_frame_t *frame,
-		      fop_utimens_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame,
-		     fop_access_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@path      - path parameter to @fn.
-	     NOTE: @path is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame,
-		       fop_readlink_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       const char *path);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame,
-		    fop_mknod_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame,
-		    fop_mkdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame,
-		     fop_unlink_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame,
-		    fop_rmdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame,
-		      fop_symlink_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-		      struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame,
-		     fop_rename_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame,
-		   fop_link_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct stat *buf);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@fd        - fd parameter to @fn.
-	     NOTE: @fd pointer is stored with a fd_ref().
-@inode     - inode parameter to @fn.
-	     NOTE: @inode pointer is stored with a inode_ref().
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame,
-		     fop_create_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct stat *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@fd        - fd parameter to @fn.
-	     NOTE: @fd pointer is stored with a fd_ref().
-call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame,
-		   fop_open_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@vector    - vector parameter to @fn.	
-	     NOTE: @vector is copied to a different memory location, if not NULL. also
-	           frame->root->rsp_refs is dict_ref()ed.
-@stbuf     - stbuf parameter to @fn.
-	     NOTE: @stbuf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame,
-		    fop_readv_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iovec *vector,
-		    int32_t count,
-		    struct stat *stbuf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@stbuf     - stbuf parameter to @fn.
-	     NOTE: @stbuf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame,
-		     fop_writev_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *stbuf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame,
-		    fop_flush_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame,
-		    fop_fsync_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@fd        - fd parameter to @fn.
-	     NOTE: @fd pointer is stored with a fd_ref().
-call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame,
-		      fop_opendir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@entries   - entries parameter to @fn.
-@count     - count parameter to @fn.
-call_stub_t *
-fop_getdents_cbk_stub (call_frame_t *frame,
-		      fop_getdents_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dir_entry_t *entries,
-		      int32_t count);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_setdents_cbk_stub (call_frame_t *frame,
-		       fop_setdents_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno);
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame,
-		       fop_fsyncdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@buf       - buf parameter to @fn.
-	     NOTE: @buf is copied to a different memory location, if not NULL.
-call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame,
-		     fop_statfs_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct statvfs *buf);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_setxattr_cbk_stub (call_frame_t *frame,
-		       fop_setxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-@value     - value dictionary parameter to @fn.
-	     NOTE: @value pointer is stored with a dict_ref().
-call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame,
-		       fop_getxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *value);
-
-
-@frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
-@op_ret    - op_ret parameter to @fn.
-@op_errno  - op_errno parameter to @fn.
-call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame,
-			  fop_removexattr_cbk_t fn,
-			  int32_t op_ret,
-			  int32_t op_errno);
-
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@op_ret   - op_ret parameter to @fn.
-@op_errno - op_errno parameter to @fn.
-@lock     - lock parameter to @fn.
-	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
-call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame,
-		 fop_lk_cbk_t fn,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct flock *lock);
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@op_ret   - op_ret parameter to @fn.
-@op_errno - op_errno parameter to @fn.
-@lock     - lock parameter to @fn.
-	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
-call_stub_t *
-fop_gf_lk_cbk_stub (call_frame_t *frame,
-		    fop_gf_lk_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct flock *lock);
-
-
-@frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
-@op_ret   - op_ret parameter to @fn.
-@op_errno - op_errno parameter to @fn.
-@entries  - entries parameter to @fn.
-call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame,
-		      fop_readdir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      gf_dirent_t *entries);
-
-
-@frame         - call frame which has to be used to resume the call at call_resume().
-@fn            - procedure to call during call_resume(). 
-@op_ret        - op_ret parameter to @fn.
-@op_errno      - op_errno parameter to @fn.
-@file_checksum - file_checksum parameter to @fn.
-                 NOTE: file_checksum will be copied to a different memory location 
-		       while creating stub.
-@dir_checksum  - dir_checksum parameter to @fn.
-                 NOTE: file_checksum will be copied to a different memory location 
-		       while creating stub.
-call_stub_t *
-fop_checksum_cbk_stub (call_frame_t *frame,
-		       fop_checksum_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum);
-
-resuming a call:
----------------
-  call can be resumed using call stub through call_resume API.
-
-  void call_resume (call_stub_t *stub);
-
-  stub - call stub created during pausing a call.
-
-  NOTE: call_resume() will decrease reference count of any fd_t, dict_t and inode_t that it finds
-        in  stub->args.<operation>.<fd_t-or-inode_t-or-dict_t>. so, if any fd_t, dict_t or
-	inode_t pointers are assigned at stub->args.<operation>.<fd_t-or-inode_t-or-dict_t> after
-	fop_<operation>_stub() call, they must be <fd_t-or-inode_t-or-dict_t>_ref()ed.
-	
-	call_resume does not STACK_DESTROY() for any fop.
-	
-  if stub->fn is NULL, call_resume does STACK_WIND() or STACK_UNWIND() using the stub->frame.
-
-  return - call resume fails only if stub is NULL. call resume fails with errno set to EINVAL.
diff --git a/doc/hacker-guide/hacker-guide.tex b/doc/hacker-guide/hacker-guide.tex
deleted file mode 100644
index c2d7255d7df..00000000000
--- a/doc/hacker-guide/hacker-guide.tex
+++ /dev/null
@@ -1,309 +0,0 @@
-\documentclass{book}[12pt]
-\usepackage{graphicx}
-% \usepackage{fancyhdr}
-
-% \pagestyle{fancy}
-\begin{document}
-
-% \headheight 117pt
-% \rhead{\includegraphics{zr-logo.eps}}
-
-\author{Gluster}
-\title{GlusterFS 1.3 Hacker's Guide}
-\date{June 1, 2007}
-
-\maketitle
-\frontmatter
-\tableofcontents
-
-\mainmatter
-\chapter{Introduction}
-
-\section{Coding guidelines}
-GlusterFS uses Git for version control. To get the latest source do:
-\begin{verbatim}
-  $ git clone git://git.gluster.com/glusterfs.git glusterfs
-\end{verbatim}
-\noindent
-GlusterFS follows the GNU coding
-standards\footnote{http://www.gnu.org/prep/standards\_toc.html} for the
-most part.
-
-\chapter{Major components}
-\section{libglusterfs}
-\texttt{libglusterfs} contains supporting code used by all the other components. 
-The important files here are:
-
-\texttt{dict.c}: This is an implementation of a serializable dictionary type. It is
-used by the protocol code to send requests and replies. It is also used to pass options
-to translators.
-
-\texttt{logging.c}: This is a thread-safe logging library. The log messages go to a
-file (default \texttt{/usr/local/var/log/glusterfs/*}).
-
-\texttt{protocol.c}: This file implements the GlusterFS on-the-wire
-protocol. The protocol itself is a simple ASCII protocol, designed to
-be easy to parse and be human readable.
-
-A sample GlusterFS protocol block looks like this:
-\begin{verbatim}
-  Block Start                            header
-  0000000000000023                       callid
-  00000001                               type
-  00000016                               op
-  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx       human-readable name
-  00000000000000000000000000000ac3       block size
-  <...>                                  block
-  Block End
-\end{verbatim}
-
-\texttt{stack.h}: This file defines the \texttt{STACK\_WIND} and
-\texttt{STACK\_UNWIND} macros which are used to implement the parallel
-stack that is maintained for inter-xlator calls. See the \textsl{Taking control
-of the stack} section below for more details.
-
-\texttt{spec.y}: This contains the Yacc grammar for the GlusterFS
-specification file, and the parsing code.
-
-
-Draw diagrams of trees
-Two rules:
-(1) directory structure is same
-(2) file can exist only on one node
-
-\section{glusterfs-fuse}
-\section{glusterfsd}
-\section{transport}
-\section{scheduler}
-\section{xlator}
-
-\chapter{xlators}
-\section{Taking control of the stack}
-One can think of STACK\_WIND/UNWIND as a very specific RPC mechanism.
-
-% \includegraphics{stack.eps}
-
-\section{Overview of xlators}
-
-\flushleft{\LARGE\texttt{cluster/}}
-\vskip 2ex
-\flushleft{\Large\texttt{afr}}
-\vskip 2ex
-\flushleft{\Large\texttt{stripe}}
-\vskip 2ex
-\flushleft{\Large\texttt{unify}}
-
-\vskip 4ex
-\flushleft{\LARGE\texttt{debug/}}
-\vskip 2ex
-\flushleft{\Large\texttt{trace}}
-\vskip 2ex
-The trace xlator simply logs all fops and mops, and passes them through to its child.
-
-\vskip 4ex
-\flushleft{\LARGE\texttt{features/}}
-\flushleft{\Large\texttt{posix-locks}}
-\vskip 2ex
-This xlator implements \textsc{posix} record locking semantics over
-any kind of storage.
-
-\vskip 4ex
-\flushleft{\LARGE\texttt{performance/}}
-
-\flushleft{\Large\texttt{io-threads}}
-\vskip 2ex
-\flushleft{\Large\texttt{read-ahead}}
-\vskip 2ex
-\flushleft{\Large\texttt{stat-prefetch}}
-\vskip 2ex
-\flushleft{\Large\texttt{write-behind}}
-\vskip 2ex
-
-\vskip 4ex
-\flushleft{\LARGE\texttt{protocol/}}
-\vskip 2ex
-
-\flushleft{\Large\texttt{client}}
-\vskip 2ex
-
-\flushleft{\Large\texttt{server}}
-\vskip 2ex
-
-\vskip 4ex
-\flushleft{\LARGE\texttt{storage/}}
-\flushleft{\Large\texttt{posix}}
-\vskip 2ex
-The \texttt{posix} xlator is the one which actually makes calls to the
-on-disk filesystem. Currently this is the only storage xlator available. However,
-plans to develop other storage xlators, such as one for Amazon's S3 service, are
-on the roadmap.
-
-\chapter{Writing a simple xlator}
-\noindent
-In this section we're going to write a rot13 xlator. ``Rot13'' is a
-simple substitution cipher which obscures a text by replacing each
-letter with the letter thirteen places down the alphabet. So `a' (0)
-would become `n' (12), `b' would be 'm', and so on.  Rot13 applied to
-a piece of ciphertext yields the plaintext again, because rot13 is its
-own inverse, since:
-
-\[
-x_c = x + 13\; (mod\; 26)
-\]
-\[
-x_c + 13\; (mod\; 26) = x + 13 + 13\; (mod\; 26) = x
-\]
-
-First we include the requisite headers.
-
-\begin{verbatim}
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-/*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
- * This xlator is meant as an example, not for production
- *  use ;) (hence no error-checking)
- */
-
-\end{verbatim}
-
-Then we write the rot13 function itself. For simplicity, we only transform lower case
-letters. Any other byte is passed through as it is.
-
-\begin{verbatim}
-/* We only handle lower case letters for simplicity */
-static void 
-rot13 (char *buf, int len)
-{
-  int i;
-  for (i = 0; i < len; i++) {
-    if (isalpha (buf[i]))
-      buf[i] = (buf[i] - 'a' + 13) % 26;
-    else if (buf[i] <= 26)
-      buf[i] = (buf[i] + 13) % 26 + 'a';
-  }
-}
-\end{verbatim}
-
-Next comes a utility function whose purpose will be clear after looking at the code
-below.
-
-\begin{verbatim}
-static void
-rot13_iovec (struct iovec *vector, int count)
-{
-  int i;
-  for (i = 0; i < count; i++) {
-    rot13 (vector[i].iov_base, vector[i].iov_len);
-  }
-}
-\end{verbatim}
-
-\begin{verbatim}
-static int32_t
-rot13_readv_cbk (call_frame_t *frame,
-                 call_frame_t *prev_frame,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iovec *vector,
-                 int32_t count)
-{
-  rot13_iovec (vector, count);
-
-  STACK_UNWIND (frame, op_ret, op_errno, vector, count);
-  return 0;
-}
-
-static int32_t
-rot13_readv (call_frame_t *frame,
-             xlator_t *this,
-             dict_t *ctx,
-             size_t size,
-             off_t offset)
-{
-  STACK_WIND (frame,
-              rot13_readv_cbk,
-              FIRST_CHILD (this),
-              FIRST_CHILD (this)->fops->readv,
-              ctx, size, offset);
-  return 0;
-}
-
-static int32_t
-rot13_writev_cbk (call_frame_t *frame,
-                  call_frame_t *prev_frame,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno)
-{
-  STACK_UNWIND (frame, op_ret, op_errno);
-  return 0;
-}
-
-static int32_t
-rot13_writev (call_frame_t *frame,
-              xlator_t *this,
-              dict_t *ctx,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  rot13_iovec (vector, count);
-
-  STACK_WIND (frame, 
-              rot13_writev_cbk,
-              FIRST_CHILD (this),
-              FIRST_CHILD (this)->fops->writev,
-              ctx, vector, count, offset);
-  return 0;
-}
-
-\end{verbatim}
-
-Every xlator must define two functions and two external symbols. The functions are 
-\texttt{init} and \texttt{fini}, and the symbols are \texttt{fops} and \texttt{mops}.
-The \texttt{init} function is called when the xlator is loaded by GlusterFS, and 
-contains code for the xlator to initialize itself. Note that if an xlator is present
-multiple times in the spec tree, the \texttt{init} function will be called each time
-the xlator is loaded.
-
-\begin{verbatim}
-int32_t
-init (xlator_t *this)
-{
-  if (!this->children) {
-    gf_log ("rot13", GF_LOG_ERROR, 
-            "FATAL: rot13 should have exactly one child");
-    return -1;
-  }
-
-  gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
-  return 0;
-}
-\end{verbatim}
-
-\begin{verbatim}
-
-void 
-fini (xlator_t *this)
-{
-  return;
-}
-
-struct xlator_fops fops = {
-  .readv        = rot13_readv,
-  .writev       = rot13_writev
-};
-
-
-\end{verbatim}
-
-\end{document}
-
diff --git a/doc/hacker-guide/lock-ahead.txt b/doc/hacker-guide/lock-ahead.txt
deleted file mode 100644
index 63392b7fa2c..00000000000
--- a/doc/hacker-guide/lock-ahead.txt
+++ /dev/null
@@ -1,80 +0,0 @@
-			Lock-ahead translator
-                        ---------------------
-
-The objective of the lock-ahead translator is to speculatively
-hold locks (inodelk and entrylk) on the universal set (0 - infinity
-in case of inodelk and all basenames in case of entrylk) even
-when a lock is requested only on a subset, in anticipation that
-further locks will be requested within the same universal set.
-
-So, for example, when cluster/replicate locks a region before
-writing to it, lock-ahead would instead lock the entire file.
-On further writes, lock-ahead can immediately return success for 
-the lock requests, since the entire file has been previously locked.
-
-To avoid starvation of other clients/mountpoints, we employ a
-notify mechanism, described below.
-
-typedef struct {
-        struct list_head subset_locks;
-} la_universal_lock_t;
-
-Universal lock structure is stored in the inode context.
-
-typedef struct {
-	enum {LOCK_AHEAD_ENTRYLK, LOCK_AHEAD_FENTRYLK,
-              LOCK_AHEAD_INODELK, LOCK_AHEAD_FINODELK};
-
-        union {
-	      fd_t *fd;
-              loc_t loc;
-	};
-
-	off_t l_start;
-        off_t l_len;
-
-	const char *basename;
-
-	struct list_head universal_lock;
-} la_subset_lock_t;
-
-
-fops implemented:
-
-* inodelk/finodelk/entrylk/fentrylk:
-
-lock:
-  if universal lock held:
-    add subset to it (save loc_t or fd) and return success
-  else:
-    send lock-notify fop
-    hold universal lock and return
-      (set inode context, add subset to it, save loc_t or fd)
-    
-    if this fails:
-      forward the lock request
-
-unlock:
-  if subset exists in universal lock:
-    delete subset lock from list
-  else:
-    forward it
-
-* release:
-    hold subset locks (each subset lock using the saved loc_t or fd)
-    and release universal lock
-
-* lock-notify (on unwind) (new fop)
-    hold subset locks and release universal lock
-
-
-lock-notify in locks translator:
-
-if a subset lock in entrylk/inodelk cannot be satisfied
-because of a universal lock held by someone else:
-    unwind the lock-notify fop
-
-==============================================
-$ Last updated: Tue Feb 17 11:31:18 IST 2009 $
-$ Author: Vikas Gorur <vikas@gluster.com>  $
-==============================================
diff --git a/doc/hacker-guide/posix.txt b/doc/hacker-guide/posix.txt
deleted file mode 100644
index d0132abfedc..00000000000
--- a/doc/hacker-guide/posix.txt
+++ /dev/null
@@ -1,59 +0,0 @@
----------------
-* storage/posix
----------------
-
-- SET_FS_ID
-
-  This is so that all filesystem checks are done with the user's 
-  uid/gid and not GlusterFS's uid/gid.
-
-- MAKE_REAL_PATH
- 
-  This macro concatenates the base directory of the posix volume
-  ('option directory') with the given path.
-
-- need_xattr in lookup
-
-  If this flag is passed, lookup returns a xattr dictionary that contains
-  the file's create time, the file's contents, and the version number
-  of the file.
-
-  This is a hack to increase small file performance. If an application 
-  wants to read a small file, it can finish its job with just a lookup 
-  call instead of a lookup followed by read.
-
-- getdents/setdents
-
-  These are used by unify to set and get directory entries.
-
-- ALIGN_BUF
-  
-  Macro to align an address to a page boundary (4K).
-
-- priv->export_statfs
-
-  In some cases, two exported volumes may reside on the same
-  partition on the server. Sending statvfs info for both
-  the volumes will lead to erroneous df output at the client,
-  since free space on the partition will be counted twice.
-
-  In such cases, user can disable exporting statvfs info
-  on one of the volumes by setting this option.
-
-- xattrop
-
-  This fop is used by replicate to set version numbers on files. 
-
-- getxattr/setxattr hack to read/write files
-
-  A key, GLUSTERFS_FILE_CONTENT_STRING, is handled in a special way by
-  getxattr/setxattr. A getxattr with the key will return the entire
-  content of the file as the value. A setxattr with the key will write
-  the value as the entire content of the file.
-
-- posix_checksum
-  
-  This calculates a simple XOR checksum on all entry names in a
-  directory that is used by unify to compare directory contents.
-
-
diff --git a/doc/hacker-guide/replicate.txt b/doc/hacker-guide/replicate.txt
deleted file mode 100644
index fd1ef2747f0..00000000000
--- a/doc/hacker-guide/replicate.txt
+++ /dev/null
@@ -1,206 +0,0 @@
----------------
-* cluster/replicate
----------------
-
-Before understanding replicate, one must understand two internal FOPs:
-
-GF_FILE_LK:
-  This is exactly like fcntl(2) locking, except the locks are in a 
-  separate domain from locks held by applications.
-
-GF_DIR_LK (loc_t *loc, char *basename):
-  This allows one to lock a name under a directory. For example,
-  to lock /mnt/glusterfs/foo, one would use the call:
-
-  GF_DIR_LK ({loc_t for "/mnt/glusterfs"}, "foo")
-
-  If one wishes to lock *all* the names under a particular directory,
-  supply the basename argument as NULL.
-
-  The locks can either be read locks or write locks; consult the 
-  function prototype for more details.
-
-Both these operations are implemented by the features/locks (earlier
-known as posix-locks) translator.
-
---------------
-* Basic design
---------------
-
-All FOPs can be classified into four major groups:
-
- - inode-read
-   Operations that read an inode's data (file contents) or metadata (perms, etc.).
-
-   access, getxattr, fstat, readlink, readv, stat.
-
- - inode-write
-   Operations that modify an inode's data or metadata.
-
-   chmod, chown, truncate, writev, utimens.
-
- - dir-read
-   Operations that read a directory's contents or metadata.
-
-   readdir, getdents, checksum.
-
- - dir-write
-   Operations that modify a directory's contents or metadata.
-
-   create, link, mkdir, mknod, rename, rmdir, symlink, unlink.
-
-   Some of these make a subgroup in that they modify *two* different entries:
-        link, rename, symlink.
-
- - Others
-   Other operations.
-
-   flush, lookup, open, opendir, statfs.
-
-------------
-* Algorithms
-------------
-
-Each of the four major groups has its own algorithm:
-
- ----------------------
- - inode-read, dir-read
- ----------------------
-
- = Send a request to the first child that is up:
-   - if it fails:
-       try the next available child
-   - if we have exhausted all children:
-       return failure
-
- -------------
- - inode-write
- -------------
-
- All operations are done in parallel unless specified otherwise.
-
- (1) Send a GF_FILE_LK request on all children for a write lock on 
-     the appropriate region
-            (for metadata operations: entire file (0, 0)
-             for writev: (offset, offset+size of buffer))
-
-     - If a lock request fails on a child:
-         unlock all children
-         try to acquire a blocking lock (F_SETLKW) on each child, serially.
-	 
-	 If this fails (due to ENOTCONN or EINVAL):
-           Consider this child as dead for rest of transaction.
-
- (2) Mark all children as "pending" on all (alive) children 
-     (see below for meaning of "pending").
-
-     - If it fails on any child:
-         mark it as dead (in transaction local state).
-
- (3) Perform operation on all (alive) children.
-
-     - If it fails on any child:
-         mark it as dead (in transaction local state).
-
- (4) Unmark all successful children as not "pending" on all nodes.
-
- (5) Unlock region on all (alive) children.
-     
- -----------
- - dir-write
- -----------
-
- The algorithm for dir-write is same as above except instead of holding
- GF_FILE_LK locks we hold a GF_DIR_LK lock on the name being operated upon.
- In case of link-type calls, we hold locks on both the operand names.
-
------------
-* "pending"
------------
-
- The "pending" number is like a journal entry. A pending entry is an
- array of 32-bit integers stored in network byte-order as the extended
- attribute of an inode (which can be a directory as well).
- 
- There are three keys corresponding to three types of pending operations:
-
- - AFR_METADATA_PENDING
-     There are some metadata operations pending on this inode (perms, ctime/mtime, 
-     xattr, etc.).
-
- - AFR_DATA_PENDING
-     There is some data pending on this inode (writev).
-
- - AFR_ENTRY_PENDING
-     There are some directory operations pending on this directory
-     (create, unlink, etc.).
-    
------------
-* Self heal
------------
-
- - On lookup, gather extended attribute data:
-   - If entry is a regular file:
-     - If an entry is present on one child and not on others:
-       - create entry on others.
-     - If entries exist but have different metadata (perms, etc.):
-       - consider the entry with the highest AFR_METADATA_PENDING number as
-         definitive and replicate its attributes on children.
-
-   - If entry is a directory:
-     - Consider the entry with the higest AFR_ENTRY_PENDING number as
-       definitive and replicate its contents on all children.
-
-   - If any two entries have non-matching types (i.e., one is file and
-     other is directory):
-     - Announce to the user via log that a split-brain situation has been
-       detected, and do nothing.
- 
- - On open, gather extended attribute data:
-   - Consider the file with the highest AFR_DATA_PENDING number as
-     the definitive one and replicate its contents on all other
-     children.
-
- During all self heal operations, appropriate locks must be held on all
- regions/entries being affected.
-
----------------
-* Inode scaling
----------------
-
-Inode scaling is necessary because if a situation arises where:
-  - An inode number is returned for a directory (by lookup) which was
-    previously the inode number of a file (as per FUSE's table), then
-    FUSE gets horribly confused (consult a FUSE expert for more details).
-
-To avoid such a situation, we distribute the 64-bit inode space equally
-among all children of replicate.
-
-To illustrate:
-
-If c1, c2, c3 are children of replicate, they each get 1/3 of the available
-inode space:
-
-Child:        c1   c2   c3   c1   c2   c3   c1   c2   c3   c1   c2 ...
-Inode number: 1    2    3    4    5    6    7    8    9    10   11 ...
-
-Thus, if lookup on c1 returns an inode number "2", it is scaled to "4"
-(which is the second inode number in c1's space).
-
-This way we ensure that there is never a collision of inode numbers from
-two different children.
-
-This reduction of inode space doesn't really reduce the usability of 
-replicate since even if we assume replicate has 1024 children (which would be a
-highly unusual scenario), each child still has a 54-bit inode space.
-
-2^54 ~ 1.8 * 10^16
-
-which is much larger than any real world requirement.
-
-
-==============================================
-$ Last updated: Sun Oct 12 23:17:01 IST 2008 $
-$ Author: Vikas Gorur <vikas@gluster.com>  $
-==============================================
-
diff --git a/doc/hacker-guide/write-behind.txt b/doc/hacker-guide/write-behind.txt
deleted file mode 100644
index 498e95480ae..00000000000
--- a/doc/hacker-guide/write-behind.txt
+++ /dev/null
@@ -1,45 +0,0 @@
-basic working
---------------
-
-  write behind is basically a translator to lie to the application that the write-requests are finished, even before it is actually finished.
-
-  on a regular translator tree without write-behind, control flow is like this:
-  
-  1. application makes a write() system call.
-  2. VFS ==> FUSE ==> /dev/fuse.
-  3. fuse-bridge initiates a glusterfs writev() call.
-  4. writev() is STACK_WIND()ed upto client-protocol or storage translator.
-  5. client-protocol, on recieving reply from server, starts STACK_UNWIND() towards the fuse-bridge.
-
-  on a translator tree with write-behind, control flow is like this:
-  
-  1. application makes a write() system call.
-  2. VFS ==> FUSE ==> /dev/fuse.
-  3. fuse-bridge initiates a glusterfs writev() call.
-  4. writev() is STACK_WIND()ed upto write-behind translator.
-  5. write-behind adds the write buffer to its internal queue and does a STACK_UNWIND() towards the fuse-bridge.
-  
-  write call is completed in application's percepective. after STACK_UNWIND()ing towards the fuse-bridge, write-behind initiates a fresh writev() call to its child translator, whose replies will be consumed by write-behind itself. write-behind _doesn't_ cache the write buffer, unless 'option flush-behind on' is specified in volume specification file.
-
-windowing
----------
-
-  write respect to write-behind, each write-buffer has three flags: 'stack_wound', 'write_behind' and 'got_reply'.
-
-  stack_wound: if set, indicates that write-behind has initiated STACK_WIND() towards child translator. 
-
-  write_behind: if set, indicates that write-behind has done STACK_UNWIND() towards fuse-bridge.
-
-  got_reply: if set, indicates that write-behind has recieved reply from child translator for a writev() STACK_WIND(). a request will be destroyed by write-behind only if this flag is set.
-
-  currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0.
-  
-  window size limits the aggregate size of currently pending write requests. once the pending requests' size has reached the window size, write-behind blocks  writev() calls from fuse-bridge. 
-  blocking is only from application's perspective. write-behind does STACK_WIND() to child translator straight-away, but hold behind the STACK_UNWIND() towards fuse-bridge. STACK_UNWIND() is done only once write-behind gets enough replies to accomodate for currently blocked request.
-  
-flush behind
-------------
-
-  if 'option flush-behind on' is specified in volume specification file, then write-behind sends aggregate write requests to child translator, instead of regular per request STACK_WIND()s.
-  
-
diff --git a/doc/handling-options.txt b/doc/handling-options.txt
deleted file mode 100644
index cac1fe93947..00000000000
--- a/doc/handling-options.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-
-How to add a new option to a given volume ?
-===========================================
-
-* Add a entry in 'struct volume_options options[]' with your key, what is 
-  the type of the 'key', etc.
-
-* The 'key' and corresponding 'value' given for the same by user are validated 
-  before calling init() of the translator/transport/scheduler/auth-module.
-
-* Once the complete init() is successful, user will get a warning if he has 
-  given a 'key' which is not defined in these modules.
-
diff --git a/doc/mac-related-xattrs.txt b/doc/mac-related-xattrs.txt
deleted file mode 100644
index 805658334e6..00000000000
--- a/doc/mac-related-xattrs.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-
-This document is intended to briefly explain how the Extended Attributes on 
-Darwin 10.5.x releases works
-----
-
-On Darwin other than all the normal filesystem operations, 'Finder' (like 
-Explorer in Windows but a little more) keeps its information in two extended 
-attributes named 'com.apple.FinderInfo' and 'com.apple.ResourceFork'. If these 
-xattrs are not implemented the filesystem won't be shown on Finder, and if they 
-are not implemented properly there may be issues when some of the file operations 
-are done through GUI of Finder. But when a filesystem is used over mountpoint in a 
-terminal, everything is fine and these xattrs are not required. 
-
-Currently the way these xattrs are implemented is simple. All the xattr calls 
-(getxattr, setxattr, listxattr, removexattr) are passed down to underlaying filesystem,
-most of the cases when exported FS is on MacOS X itself, these keys are supported, hence
-the fops succeed. But in the case of using exports of different OS on Darwin the issue is 
-extended attribute prefix like 'com.apple.' may not be supported, hence the problem with 
-Finder. To solve this issue, GlusterFS returns virtual default values to these keys, which
-works fine on most of the cases.
-
diff --git a/doc/mount.glusterfs.8 b/doc/mount.glusterfs.8
index aced363f74f..ce16e9e40b7 100644
--- a/doc/mount.glusterfs.8
+++ b/doc/mount.glusterfs.8
@@ -1,37 +1,31 @@
-.\"  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU Affero General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  Affero General Public License for more details.
 .\"
-.\"  You should have received a copy of the GNU Affero General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
 .\"
-.\"
-.\"
-.TH GlusterFS 8 "Cluster Filesystem" "18 March 2010" "Gluster Inc."
+.TH GlusterFS 8 "Cluster Filesystem" "14 September 2013" "Red Hat, Inc."
 .SH NAME
-mount.glusterfs - script to mount native GlusterFS volume
+.B mount.glusterfs - script to mount native GlusterFS volume
 .SH SYNOPSIS
-.B mount -t glusterfs
-.I [-o <options>] <volumeserver>:<volumeid> <mountpoint>
+.B mount -t glusterfs [-o <options>] <volumeserver>:/<volume>[/<subdir>]
+.B         <mountpoint>
+.TP
+.B mount -t glusterfs [-o <options>] <server1>,<server2>,
+.B        <server3>,..<serverN>:/<volname>[/<subdir>] <mount_point>
 .TP
-.B mount -t glusterfs
-.I [-o <options>] <path/to/volumefile> <mountpoint>
+.TP
+.B mount -t glusterfs [-o <options>] <path/to/volumefile> <mountpoint>
 .PP
 .SH DESCRIPTION
 This tool is part of \fBglusterfs\fR(8) package, which is used to mount using
 GlusterFS native binary.
 
-\fBmount.glusterfs\fR  is meant to be used by the mount(8) command for mounting
+\fBmount.glusterfs\fR is meant to be used by the mount(8) command for mounting
 native GlusterFS client. This subcommand, however, can also be used as a
 standalone command with limited functionality.
 
@@ -45,43 +39,139 @@ File to use for logging [default:/var/log/glusterfs/glusterfs.log]
 .TP
 \fBlog\-level=\fRLOG-LEVEL
 Logging severity.  Valid options are TRACE, DEBUG, WARNING, ERROR, CRITICAL
-NORMAL and NONE [default: NORMAL]
+INFO and NONE [default: INFO]
+.TP
+\fBacl
+Mount the filesystem with POSIX ACL support
+.TP
+\fBfopen\-keep\-cache[=BOOL]
+Do not purge the cache on file open (default: false)
+.TP
+\fBworm
+Mount the filesystem in 'worm' mode
+.TP
+\fBaux\-gfid\-mount
+Enable access to filesystem through gfid directly
 .TP
 \fBro\fR
 Mount the filesystem read-only
+.TP
+\fBenable\-ino32=\fRBOOL
+Use 32-bit inodes when mounting to workaround broken applications that don't
+support 64-bit inodes
+.TP
+\fBmem\-accounting
+Enable internal memory accounting
+.TP
+\fBcapability
+Enable file capability setting and retrival
+.TP
+\fBthin-client
+Enables thin mount and connects via gfproxyd daemon
+
 .PP
 .SS "Advanced options"
 .PP
 .TP
-\fBvolfile\-id=\fRKEY
-Volume key or name of the volume file to be fetched from server
+\fBattribute\-timeout=\fRSECONDS
+Set attribute timeout to SECONDS for inodes in fuse kernel module [default: 1]
+.TP
+\fBentry\-timeout=\fRSECONDS
+Set entry timeout to SECONDS in fuse kernel module [default: 1]
+.TP
+\fBbackground\-qlen=\fRN
+Set fuse module's background queue length to N [default: 64]
 .TP
-\fBtransport=\fRTRANSPORT-TYPE
-Transport type to get volume file from server [default: socket]
+\fBgid\-timeout=\fRSECONDS
+Set auxiliary group list timeout to SECONDS for fuse translator [default: 0]
+.TP
+\fBnegative\-timeout=\fRSECONDS
+Set negative timeout to SECONDS in fuse kernel module [default: 0]
 .TP
 \fBvolume\-name=\fRVOLUME-NAME
 Volume name to be used for MOUNT-POINT [default: top most volume in
 VOLUME-FILE]
 .TP
-\fBdirect\-io\-mode=\fRdisable
-Disable direct I/O mode in fuse kernel module
+\fBdirect\-io\-mode=\fRBOOL|auto
+Specify fuse direct I/O strategy [default: auto]
+.TP
+\fBcongestion\-threshold=\fRN
+Set fuse module's congestion threshold to N [default: 48]
+.TP
+\fsubdir\-mount=\fRN
+Set the subdirectory mount option [default: NULL, ie, no subdirectory mount]
+.TP
+.TP
+\fBbackup\-volfile\-servers=\fRSERVERLIST
+Provide list of backup volfile servers in the following format [default: None]
+
+\fB$ mount \-t glusterfs \-obackup\-volfile\-servers=<server2>:\fR
+\fB       <server3>:...:<serverN> <server1>:/<volname> <mount_point>\fR
+
+.TP
+.TP
+\fBbackupvolfile\-server=\fRSERVER
+Provide list of backup volfile servers in the following format [default: None]
+
+\fB $ mount \-t glusterfs \-obackupvolfile\-server=<server2>
+\fB         <server1>:/<volname> <mount_point>
+
+.TP
+.TP
+\fBfetch-attempts=\fRN
+\fBDeprecated\fR option - placed here for backward compatibility [default: 1]
+.TP
+.TP
+\fBlru-limit=\fRN
+Set fuse module's limit for number of inodes kept in LRU list to N [default: 65536]
+.TP
+.TP
+\fBinvalidate-limit=\fRN
+Suspend fuse invalidations implied by 'lru-limit' if  number of outstanding
+invalidations reaches N
+.TP
+.TP
+\fBbackground-qlen=\fRN
+Set fuse module's background queue length to N [default: 64]
+.TP
+\fBno\-root\-squash=\fRBOOL
+disable root squashing for the trusted client [default: off]
+.TP
+\fBroot\-squash=\fRBOOL
+enable root squashing for the trusted client [default: on]
+.TP
+\fBuse\-readdirp=\fRBOOL
+Use readdirp() mode in fuse kernel module [default: on]
+.TP
+\fBdump\-fuse=\fRPATH
+Dump fuse traffic to PATH
+.TP
+\fBkernel\-writeback\-cache=\fRBOOL
+Enable fuse in-kernel writeback cache [default: off]
+.TP
+\fBattr\-times\-granularity=\fRNS
+Declare supported granularity of file attribute [default: 0]
 .TP
+\fBauto\-invalidation=\fRBOOL
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
 .PP
 .SH FILES
 .TP
 .I /etc/fstab
 A typical GlusterFS entry in /etc/fstab looks like below
 
-server1.gluster.com:mirror  /mnt/mirror  glusterfs  log-file=/var/log/mirror.vol,ro,defaults   0  0
+\fBserver1:/mirror  /mnt/mirror  glusterfs log-file=/var/log/mirror.log,acl   0  0\fR
 
 .TP
-.I /etc/mtab
-An example entry of a GlusterFS mountpoint in /etc/mtab looks like below
+.I /proc/mounts
+An example entry of a GlusterFS mountpoint in /proc/mounts looks like below
 
-mirror.vol /mnt/glusterfs fuse.glusterfs rw,allow_other,default_permissions,max_read=131072 0 0
+\fBserver1:/mirror /mnt/glusterfs fuse.glusterfs rw,allow_other,default_permissions,max_read=131072 0 0\fR
 
 .SH SEE ALSO
-\fBglusterfs\fR(8), \fBmount\fR(8)
+\fBglusterfs\fR(8), \fBmount\fR(8), \fBgluster\fR(8)
 
 .SH COPYRIGHT
-Copyright(c) 2006-2010   Gluster, Inc.   <http://www.gluster.com>
+Copyright(c) 2006-2013   Red Hat, Inc.   <http://www.redhat.com>
diff --git a/doc/porting_guide.txt b/doc/porting_guide.txt
deleted file mode 100644
index 905bb422858..00000000000
--- a/doc/porting_guide.txt
+++ /dev/null
@@ -1,45 +0,0 @@
-		       GlusterFS Porting Guide
-                       -----------------------
-
-* General setup
-
-The configure script will detect the target platform for the build. 
-All platform-specific CFLAGS, macro definitions should be done
-in configure.ac
-
-Platform-specific code can be written like this:
-
-#ifdef GF_DARWIN_HOST_OS
-  /* some code specific to Darwin */
-#endif
-
-* Coding guidelines
-
-In general, avoid glibc extensions. For example, nested functions don't work 
-on Mac OS X. It is best to stick to C99.
-
-When using library calls and system calls, pay attention to the
-portability notes. As far as possible stick to POSIX-specified behavior. 
-Do not use anything expressly permitted by the specification. For example,
-some fields in structures may be present only on certain platforms. Avoid
-use of such things.
-
-Do not pass values of constants such as F_*, O_*, errno values, etc. across
-platforms.
-
-Please refer compat-errno.h for more details about errno handling inside 
-glusterfs for cross platform. 
-
-* Specific issues
-
-- The argp library is available only on Linux through glibc, but for other
-  platforms glusterfs has already included argp-standalone library which will
-  statically linked during the glusterfs build. 
-
-- Extended attribute calls (setxattr, listxattr, etc.) have differing prototypes
-  on different platforms. See compat.h for macro definitions to resolve this, also
-  read out the specific extended attribute documentation for your platforms.
-
-------------------------------------------
-Last revised: Thu Feb 28 13:58:07 IST 2008
-------------------------------------------
diff --git a/doc/qa/qa-client.vol b/doc/qa/qa-client.vol
deleted file mode 100644
index 176dda5892a..00000000000
--- a/doc/qa/qa-client.vol
+++ /dev/null
@@ -1,170 +0,0 @@
-# This spec file should be used for testing before any release
-# 
-
-# 1st client
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-# option transport.ib-verbs.work-request-send-size  131072
-# option transport.ib-verbs.work-request-send-count 64
-# option transport.ib-verbs.work-request-recv-size  131072
-# option transport.ib-verbs.work-request-recv-count 64
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra1
-end-volume
-
-# 2nd client
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra2
-end-volume
-
-# 3rd client
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra3
-end-volume
-
-# 4th client
-volume client4
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra4
-end-volume
-
-# 5th client
-volume client5
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra5
-end-volume
-
-# 6th client
-volume client6
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra6
-end-volume
-
-# 7th client
-volume client7
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra7
-end-volume
-
-# 8th client 
-volume client8
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra8
-end-volume
-
-# 1st Stripe (client1 client2)
-volume stripe1
-  type cluster/stripe
-  subvolumes client1 client2
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 2st Stripe (client3 client4)
-volume stripe2
-  type cluster/stripe
-  subvolumes client3 client4
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 3st Stripe (client5 client6)
-volume stripe3
-  type cluster/stripe
-  subvolumes client5 client6
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 4st Stripe (client7 client8)
-volume stripe4
-  type cluster/stripe
-  subvolumes client7 client8
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-
-# 1st replicate
-volume replicate1
-  type cluster/replicate
-  subvolumes stripe1 stripe2
-end-volume
-
-# 2nd replicate
-volume replicate2
-  type cluster/replicate
-  subvolumes stripe3 stripe4
-end-volume
-
-volume ns
-  type protocol/client
-  option transport-type tcp
-  option remote-host 127.0.0.1
-  option remote-subvolume brick-ns
-end-volume
-
-# Unify
-volume unify0
-  type cluster/unify
-  subvolumes replicate1 replicate2
-#  subvolumes stripe1 stripe3
-  option namespace ns
-  option scheduler rr # random # alu # nufa
-  option rr.limits.min-free-disk 1GB
-# option alu.order x
-# option alu.x.entry-threshold
-# option alu.x.exit-threshold
-end-volume
-
-
-# ==== Performance Translators ====
-# The default options for performance translators should be the best for 90+% of the cases
-volume iot
-  type performance/io-threads
-  subvolumes unify0
-end-volume
-
-volume wb
-  type performance/write-behind
-  subvolumes iot
-end-volume
-
-volume ioc
- type performance/io-cache
- subvolumes wb
-end-volume
-
-volume ra
-  type performance/read-ahead
-  subvolumes ioc
-end-volume
diff --git a/doc/qa/qa-high-avail-client.vol b/doc/qa/qa-high-avail-client.vol
deleted file mode 100644
index 69cb8dd30f1..00000000000
--- a/doc/qa/qa-high-avail-client.vol
+++ /dev/null
@@ -1,17 +0,0 @@
-volume client
- type protocol/client
- option transport-type tcp
- option remote-host localhost
- option transport.socket.remote-port 7001
- option remote-subvolume server1-iot
-end-volume
-
-volume ra
- type performance/read-ahead
- subvolumes client
-end-volume
-
-volume wb
- type performance/write-behind
- subvolumes ra
-end-volume
diff --git a/doc/qa/qa-high-avail-server.vol b/doc/qa/qa-high-avail-server.vol
deleted file mode 100644
index 3556b9dae99..00000000000
--- a/doc/qa/qa-high-avail-server.vol
+++ /dev/null
@@ -1,344 +0,0 @@
-
-# -- server 1 --
-volume server1-posix1
- type storage/posix
- option directory /tmp/ha-export1/
-end-volume
-
-volume server1-ns1
- type storage/posix
- option directory /tmp/ha-export-ns1/
-end-volume
-
-volume server1-client2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-posix2
-end-volume
-
-volume server1-ns2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-ns2
-end-volume
-
-volume server1-client3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-posix3
-end-volume
-
-volume server1-ns3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-ns3
-end-volume
-
-volume server1-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server1-posix1
-end-volume
-
-
-volume server1-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server1-client2
-end-volume
-
-volume server1-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server1-client3
-end-volume
-
-volume server1-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns1
-end-volume
-
-volume server1-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns2
-end-volume
-
-volume server1-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns3
-end-volume
-
-volume server1-ns-replicate
- type cluster/replicate
- subvolumes server1-ns-io1 server1-ns-io2 server1-ns-io3
-end-volume
-
-volume server1-storage-replicate
- type cluster/replicate
- subvolumes server1-io1 server1-io2 server1-io3
-end-volume
-
-volume server1-unify
- type cluster/unify
- #option self-heal off
- subvolumes server1-storage-replicate
- option namespace server1-ns-replicate
- option scheduler rr
-end-volume
-
-volume server1-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server1-unify
-end-volume
-
-volume server1
- type protocol/server
- option transport-type tcp
- subvolumes server1-iot
- option transport.socket.listen-port 7001
- option auth.addr.server1-posix1.allow *
- option auth.addr.server1-ns1.allow *
- option auth.addr.server1-iot.allow * 
-end-volume
-
-
-# == Server2 ==
-volume server2-client1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-posix1
-end-volume
-
-volume server2-ns1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-ns1
-end-volume
-
-volume server2-posix2
- type storage/posix
- option directory /tmp/ha-export2/
-end-volume
-
-volume server2-ns2
- type storage/posix
- option directory /tmp/ha-export-ns2/
-end-volume
-
-volume server2-client3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-posix3
-end-volume
-
-volume server2-ns3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-ns3
-end-volume
-
-volume server2-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server2-client1
-end-volume
-
-
-volume server2-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server2-posix2
-end-volume
-
-volume server2-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server2-client3
-end-volume
-
-volume server2-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns1
-end-volume
-
-volume server2-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns2
-end-volume
-
-volume server2-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns3
-end-volume
-
-volume server2-ns-replicate
- type cluster/replicate
- subvolumes server2-ns-io1 server2-ns-io2 server2-ns-io3
-end-volume
-
-volume server2-storage-replicate
- type cluster/replicate
- subvolumes server2-io2 server2-io3 server2-io1
-end-volume
-
-volume server2-unify
- type cluster/unify
- option self-heal off
- subvolumes server2-storage-replicate
- option namespace server2-ns-replicate
- option scheduler rr
-end-volume
-
-volume server2-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server2-unify
-end-volume
-
-volume server2
- type protocol/server
- option transport-type tcp
- subvolumes server2-iot
- option transport.socket.listen-port 7002
- option auth.addr.server2-posix2.allow *
- option auth.addr.server2-ns2.allow *
- option auth.addr.server2-iot.allow * 
-end-volume
-
-# == server 3 ==
-volume server3-client1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-posix1
-end-volume
-
-volume server3-ns1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-ns1
-end-volume
-
-volume server3-client2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-posix2
-end-volume
-
-volume server3-ns2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-ns2
-end-volume
-
-volume server3-posix3
- type storage/posix
- option directory /tmp/ha-export3/
-end-volume
-
-volume server3-ns3
- type storage/posix
- option directory /tmp/ha-export-ns3/
-end-volume
-
-volume server3-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server3-client1
-end-volume
-
-
-volume server3-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server3-client2
-end-volume
-
-volume server3-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server3-posix3
-end-volume
-
-volume server3-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns1
-end-volume
-
-volume server3-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns2
-end-volume
-
-volume server3-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns3
-end-volume
-
-volume server3-ns-replicate
- type cluster/replicate
- subvolumes server3-ns-io1 server3-ns-io2 server3-ns-io3
-end-volume
-
-volume server3-storage-replicate
- type cluster/replicate
- subvolumes server3-io3 server3-io2 server3-io1
-end-volume
-
-volume server3-unify
- type cluster/unify
- option self-heal off
- subvolumes server3-storage-replicate
- option namespace server3-ns-replicate
- option scheduler rr
-end-volume
-
-volume server3-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server3-unify
-end-volume
-
-volume server3
- type protocol/server
- option transport-type tcp
- subvolumes server3-iot
- option transport.socket.listen-port 7003
- option auth.addr.server3-posix3.allow *
- option auth.addr.server3-ns3.allow *
- option auth.addr.server3-iot.allow * 
-end-volume
-
diff --git a/doc/qa/qa-server.vol b/doc/qa/qa-server.vol
deleted file mode 100644
index 1c245c324ad..00000000000
--- a/doc/qa/qa-server.vol
+++ /dev/null
@@ -1,284 +0,0 @@
-# This spec file should be used for testing before any release
-# 
-
-# Namespace posix
-volume brick-ns
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export-ns        # Export this directory
-end-volume
-
-# 1st server
-
-volume brick1
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export1        # Export this directory
-end-volume
-
-# == Posix-Locks ==
- volume plocks1
-   type features/posix-locks
-#  option mandatory on
-   subvolumes brick1
- end-volume
-
-volume iot1
-  type performance/io-threads
-  subvolumes plocks1 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb1
-  type performance/write-behind
-  subvolumes iot1
-# option <key> <value>
-end-volume
-
-volume ra1
-  type performance/read-ahead
-  subvolumes wb1
-# option <key> <value>
-end-volume
-
-volume brick2
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export2        # Export this directory
-end-volume
-
-# == TrashCan Translator ==
-# volume trash2
-#   type features/trash
-#   option trash-dir /.trashcan
-#   subvolumes brick2
-# end-volume
-
-# == Posix-Locks ==
-volume plocks2
-  type features/posix-locks
-#  option <something> <something>
-  subvolumes brick2
-end-volume
-
-volume iot2
-  type performance/io-threads
-  subvolumes plocks2 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb2
-  type performance/write-behind
-  subvolumes iot2
-# option <key> <value>
-end-volume
-
-volume ra2
-  type performance/read-ahead
-  subvolumes wb2
-# option <key> <value>
-end-volume
-
-volume brick3
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export3        # Export this directory
-end-volume
-
-# == TrashCan Translator ==
-# volume trash3
-#   type features/trash
-#   option trash-dir /.trashcan
-#   subvolumes brick3
-# end-volume
-
-# == Posix-Locks ==
-volume plocks3
-  type features/posix-locks
-#  option <something> <something>
-  subvolumes brick3
-end-volume
-
-volume iot3
-  type performance/io-threads
-  subvolumes plocks3 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb3
-  type performance/write-behind
-  subvolumes iot3
-# option <key> <value>
-end-volume
-
-volume ra3
-  type performance/read-ahead
-  subvolumes wb3
-# option <key> <value>
-end-volume
-
-volume brick4
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export4        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks4
-  type features/posix-locks
-# option <something> <something>
-  subvolumes brick4
-end-volume
-
-volume iot4
-  type performance/io-threads
-  subvolumes plocks4 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb4
-  type performance/write-behind
-  subvolumes iot4
-# option <key> <value>
-end-volume
-
-volume ra4
-  type performance/read-ahead
-  subvolumes wb4
-# option <key> <value>
-end-volume
-
-volume brick5
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export5        # Export this directory
-end-volume
-
-
-# == Posix-Locks ==
-volume plocks5
-  type features/posix-locks
-# option <something> <something>
-  subvolumes brick5
-end-volume
-
-volume iot5
-  type performance/io-threads
-  subvolumes plocks5 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb5
-  type performance/write-behind
-  subvolumes iot5
-# option <key> <value>
-end-volume
-
-volume ra5
-  type performance/read-ahead
-  subvolumes wb5
-# option <key> <value>
-end-volume
-
-volume brick6
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export6        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks6
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick6
-end-volume
-
-volume iot6
-  type performance/io-threads
-  subvolumes plocks6 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb6
-  type performance/write-behind
-  subvolumes iot6
-# option <key> <value>
-end-volume
-
-volume ra6
-  type performance/read-ahead
-  subvolumes wb6
-# option <key> <value>
-end-volume
-
-volume brick7
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export7        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks7
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick7
-end-volume
-
-volume iot7
-  type performance/io-threads
-  subvolumes plocks7 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb7
-  type performance/write-behind
-  subvolumes iot7
-# option <key> <value>
-end-volume
-
-volume ra7
-  type performance/read-ahead
-  subvolumes wb7
-# option <key> <value>
-end-volume
-
-volume brick8
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export8        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks8
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick8
-end-volume
-
-volume iot8
-  type performance/io-threads
-  subvolumes plocks8 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb8
-  type performance/write-behind
-  subvolumes iot8
-# option <key> <value>
-end-volume
-
-volume ra8
-  type performance/read-ahead
-  subvolumes wb8
-# option <key> <value>
-end-volume
-
-volume server8
-  type protocol/server
-  subvolumes ra8 ra1 ra2 ra3 ra4 ra5 ra6 ra7 brick-ns
-  option transport-type tcp     # For TCP/IP transport
-# option transport-type ib-sdp  # For Infiniband transport
-# option transport-type ib-verbs # For ib-verbs transport
-  option client-volume-filename /examples/qa-client.vol
-  option auth.addr.ra1.allow * # Allow access to "stat8" volume
-  option auth.addr.ra2.allow * # Allow access to "stat8" volume
-  option auth.addr.ra3.allow * # Allow access to "stat8" volume
-  option auth.addr.ra4.allow * # Allow access to "stat8" volume
-  option auth.addr.ra5.allow * # Allow access to "stat8" volume
-  option auth.addr.ra6.allow * # Allow access to "stat8" volume
-  option auth.addr.ra7.allow * # Allow access to "stat8" volume
-  option auth.addr.ra8.allow * # Allow access to "stat8" volume
-  option auth.addr.brick-ns.allow * # Allow access to "stat8" volume
-end-volume
-
diff --git a/doc/replicate.lyx b/doc/replicate.lyx
deleted file mode 100644
index d11a92beedd..00000000000
--- a/doc/replicate.lyx
+++ /dev/null
@@ -1,797 +0,0 @@
-#LyX 1.4.2 created this file. For more info see http://www.lyx.org/
-\lyxformat 245
-\begin_document
-\begin_header
-\textclass article
-\language english
-\inputencoding auto
-\fontscheme default
-\graphics default
-\paperfontsize default
-\spacing single
-\papersize default
-\use_geometry false
-\use_amsmath 1
-\cite_engine basic
-\use_bibtopic false
-\paperorientation portrait
-\secnumdepth 3
-\tocdepth 3
-\paragraph_separation skip
-\defskip medskip
-\quotes_language english
-\papercolumns 1
-\papersides 1
-\paperpagestyle default
-\tracking_changes false
-\output_changes false
-\end_header
-
-\begin_body
-
-\begin_layout Title
-
-\size larger
-Automatic File Replication (replicate) in GlusterFS
-\end_layout
-
-\begin_layout Author
-Vikas Gorur 
-\family typewriter
-\size larger
-<vikas@gluster.com>
-\end_layout
-
-\begin_layout Standard
-\begin_inset ERT
-status open
-
-\begin_layout Standard
-
-
-\backslash
-hrule
-\end_layout
-
-\end_inset
-
-
-\end_layout
-
-\begin_layout Section*
-Overview
-\end_layout
-
-\begin_layout Standard
-This document describes the design and usage of the replicate translator in GlusterFS.
- This document is valid for the 1.4.x releases, and not earlier ones.
-\end_layout
-
-\begin_layout Standard
-The replicate translator of GlusterFS aims to keep identical copies of a file
- on all its subvolumes, as far as possible.
- It tries to do this by performing all filesystem mutation operations (writing
- data, creating files, changing ownership, etc.) on all its subvolumes in
- such a way that if an operation succeeds on atleast one subvolume, all
- other subvolumes can later be brought up to date.
-\end_layout
-
-\begin_layout Standard
-In the rest of the document the terms 
-\begin_inset Quotes eld
-\end_inset
-
-subvolume
-\begin_inset Quotes erd
-\end_inset
-
- and 
-\begin_inset Quotes eld
-\end_inset
-
-server
-\begin_inset Quotes erd
-\end_inset
-
- are used interchangeably, trusting that it will cause no confusion to the
- reader.
-\end_layout
-
-\begin_layout Section*
-Usage
-\end_layout
-
-\begin_layout Standard
-A sample volume declaration for replicate looks like this:
-\end_layout
-
-\begin_layout Standard
-\begin_inset ERT
-status open
-
-\begin_layout Standard
-
-
-\backslash
-begin{verbatim}
-\end_layout
-
-\begin_layout Standard
-
-volume replicate
-\end_layout
-
-\begin_layout Standard
-
- type cluster/replicate
-\end_layout
-
-\begin_layout Standard
-
- # options, see below for description
-\end_layout
-
-\begin_layout Standard
-
- subvolumes brick1 brick2
-\end_layout
-
-\begin_layout Standard
-
-end-volume
-\end_layout
-
-\begin_layout Standard
-
-
-\backslash
-end{verbatim}
-\end_layout
-
-\begin_layout Standard
-
-\end_layout
-
-\begin_layout Standard
-
-\end_layout
-
-\begin_layout Standard
-
-\end_layout
-
-\end_inset
-
-
-\end_layout
-
-\begin_layout Standard
-This defines an replicate volume with two subvolumes, brick1, and brick2.
- For replicate to work properly, it is essential that its subvolumes support 
-\series bold
-extended attributes
-\series default
-.
- This means that you should choose a backend filesystem that supports extended
- attributes, like XFS, ReiserFS, or Ext3.
-\end_layout
-
-\begin_layout Standard
-The storage volumes used as backend for replicate 
-\emph on
-must
-\emph default
- have a posix-locks volume loaded above them.
-\end_layout
-
-\begin_layout Standard
-\begin_inset ERT
-status open
-
-\begin_layout Standard
-
-
-\backslash
-begin{verbatim}
-\end_layout
-
-\begin_layout Standard
-
-volume brick1
-\end_layout
-
-\begin_layout Standard
-
- type features/posix-locks
-\end_layout
-
-\begin_layout Standard
-
- subvolumes brick1-ds
-\end_layout
-
-\begin_layout Standard
-
-end-volume
-\end_layout
-
-\begin_layout Standard
-
-
-\backslash
-end{verbatim}
-\end_layout
-
-\end_inset
-
-
-\end_layout
-
-\begin_layout Section*
-Design
-\end_layout
-
-\begin_layout Subsection*
-Read algorithm
-\end_layout
-
-\begin_layout Standard
-All operations that do not modify the file or directory are sent to all
- the subvolumes and the first successful reply is returned to the application.
-\end_layout
-
-\begin_layout Standard
-The read() system call (reading data from a file) is an exception.
- For read() calls, replicate tries to do load balancing by sending all reads from
- a particular file to a particular server.
-\end_layout
-
-\begin_layout Standard
-The read algorithm is also affected by the option read-subvolume; see below
- for details.
-\end_layout
-
-\begin_layout Subsection*
-Classes of file operations
-\end_layout
-
-\begin_layout Standard
-replicate divides all filesystem write operations into three classes:
-\end_layout
-
-\begin_layout Itemize
-
-\series bold
-data: 
-\series default
-Operations that modify the contents of a file (write, truncate).
-\end_layout
-
-\begin_layout Itemize
-
-\series bold
-metadata: 
-\series default
-Operations that modify attributes of a file or directory (permissions, ownership
-, etc.).
-\end_layout
-
-\begin_layout Itemize
-
-\series bold
-entry: 
-\series default
-Operations that create or delete directory entries (mkdir, create, rename,
- rmdir, unlink, etc.).
-\end_layout
-
-\begin_layout Subsection*
-Locking and Change Log
-\end_layout
-
-\begin_layout Standard
-To ensure consistency across subvolumes, replicate holds a lock whenever a modificatio
-n is being made to a file or directory.
- By default, replicate considers the first subvolume as the sole lock server.
- However, the number of lock servers can be increased upto the total number
- of subvolumes.
-\end_layout
-
-\begin_layout Standard
-The change log is a set of extended attributes associated with files and
- directories that replicate maintains.
- The change log keeps track of the changes made to files and directories
- (data, metadata, entry) so that the self-heal algorithm knows which copy
- of a file or directory is the most recent one.
-\end_layout
-
-\begin_layout Subsection*
-Write algorithm
-\end_layout
-
-\begin_layout Standard
-The algorithm for all write operations (data, metadata, entry) is:
-\end_layout
-
-\begin_layout Enumerate
-Lock the file (or directory) on all of the lock servers (see options below).
-\end_layout
-
-\begin_layout Enumerate
-Write change log entries on all servers.
-\end_layout
-
-\begin_layout Enumerate
-Perform the operation.
-\end_layout
-
-\begin_layout Enumerate
-Erase change log entries.
-\end_layout
-
-\begin_layout Enumerate
-Unlock the file (or directory) on all of the lock servers.
-\end_layout
-
-\begin_layout Standard
-The above algorithm is a simplified version intended for general users.
- Please refer to the source code for the full details.
-\end_layout
-
-\begin_layout Subsection*
-Self-Heal
-\end_layout
-
-\begin_layout Standard
-replicate automatically tries to fix any inconsistencies it detects among different
- copies of a file.
- It uses information in the change log to determine which copy is the 
-\begin_inset Quotes eld
-\end_inset
-
-correct
-\begin_inset Quotes erd
-\end_inset
-
- version.
-\end_layout
-
-\begin_layout Standard
-Self-heal is triggered when a file or directory is first 
-\begin_inset Quotes eld
-\end_inset
-
-accessed
-\begin_inset Quotes erd
-\end_inset
-
-, that is, the first time any operation is attempted on it.
- The self-heal algorithm does the following things:
-\end_layout
-
-\begin_layout Standard
-If the entry being accessed is a directory:
-\end_layout
-
-\begin_layout Itemize
-The contents of the 
-\begin_inset Quotes eld
-\end_inset
-
-correct
-\begin_inset Quotes erd
-\end_inset
-
- version is replicated on all subvolumes, by deleting entries and creating
- entries as necessary.
-\end_layout
-
-\begin_layout Standard
-If the entry being accessed is a file:
-\end_layout
-
-\begin_layout Itemize
-If the file does not exist on some subvolumes, it is created.
-\end_layout
-
-\begin_layout Itemize
-If there is a mismatch in the size of the file, or ownership, or permission,
- it is fixed.
-\end_layout
-
-\begin_layout Itemize
-If the change log indicates that some copies need updating, they are updated.
-\end_layout
-
-\begin_layout Subsection*
-Split-brain
-\end_layout
-
-\begin_layout Standard
-It may happen that one replicate client can access only some of the servers in
- a cluster and another replicate client can access the remaining servers.
- Or it may happen that in a cluster of two servers, one server goes down
- and comes back up, but the other goes down immediately.
- Both these scenarios result in a 
-\begin_inset Quotes eld
-\end_inset
-
-split-brain
-\begin_inset Quotes erd
-\end_inset
-
-.
-\end_layout
-
-\begin_layout Standard
-In a split-brain situation, there will be two or more copies of a file,
- all of which are 
-\begin_inset Quotes eld
-\end_inset
-
-correct
-\begin_inset Quotes erd
-\end_inset
-
- in some sense.
- replicate without manual intervention has no way of knowing what to do, since
- it cannot consider any single copy as definitive, nor does it know of any
- meaningful way to merge the copies.
-\end_layout
-
-\begin_layout Standard
-If replicate detects that a split-brain has happened on a file, it disallows opening
- of that file.
- You will have to manually resolve the conflict by deleting all but one
- copy of the file.
- Alternatively you can set an automatic split-brain resolution policy by
- using the `favorite-child' option (see below).
-\end_layout
-
-\begin_layout Section*
-Translator Options
-\end_layout
-
-\begin_layout Standard
-replicate accepts the following options:
-\end_layout
-
-\begin_layout Subsection*
-read-subvolume (default: none)
-\end_layout
-
-\begin_layout Standard
-The value of this option must be the name of a subvolume.
- If given, all read operations are sent to only the specified subvolume,
- instead of being balanced across all subvolumes.
-\end_layout
-
-\begin_layout Subsection*
-favorite-child (default: none)
-\end_layout
-
-\begin_layout Standard
-The value of this option must be the name of a subvolume.
- If given, the specified subvolume will be preferentially used in resolving
- conflicts (
-\begin_inset Quotes eld
-\end_inset
-
-split-brain
-\begin_inset Quotes erd
-\end_inset
-
-).
- This means if a discrepancy is noticed in the attributes or content of
- a file, the copy on the `favorite-child' will be considered the definitive
- version and its contents will 
-\emph on
-overwrite 
-\emph default
-the contents of all other copies.
- Use this option with caution! It is possible to 
-\emph on
-lose data
-\emph default
- with this option.
- If you are in doubt, do not specify this option.
-\end_layout
-
-\begin_layout Subsection*
-Self-heal options
-\end_layout
-
-\begin_layout Standard
-Setting any of these options to 
-\begin_inset Quotes eld
-\end_inset
-
-off
-\begin_inset Quotes erd
-\end_inset
-
- prevents that kind of self-heal from being done on a file or directory.
- For example, if metadata self-heal is turned off, permissions and ownership
- are no longer fixed automatically.
-\end_layout
-
-\begin_layout Subsubsection*
-data-self-heal (default: on)
-\end_layout
-
-\begin_layout Standard
-Enable/disable self-healing of file contents.
-\end_layout
-
-\begin_layout Subsubsection*
-metadata-self-heal (default: off)
-\end_layout
-
-\begin_layout Standard
-Enable/disable self-healing of metadata (permissions, ownership, modification
- times).
-\end_layout
-
-\begin_layout Subsubsection*
-entry-self-heal (default: on)
-\end_layout
-
-\begin_layout Standard
-Enable/disable self-healing of directory entries.
-\end_layout
-
-\begin_layout Subsection*
-Change Log options
-\end_layout
-
-\begin_layout Standard
-If any of these options is turned off, it disables writing of change log
- entries for that class of file operations.
- That is, steps 2 and 4 of the write algorithm (see above) are not done.
- Note that if the change log is not written, the self-heal algorithm cannot
- determine the 
-\begin_inset Quotes eld
-\end_inset
-
-correct
-\begin_inset Quotes erd
-\end_inset
-
- version of a file and hence self-heal will only be able to fix 
-\begin_inset Quotes eld
-\end_inset
-
-obviously
-\begin_inset Quotes erd
-\end_inset
-
- wrong things (such as a file existing on only one node).
-\end_layout
-
-\begin_layout Subsubsection*
-data-change-log (default: on)
-\end_layout
-
-\begin_layout Standard
-Enable/disable writing of change log for data operations.
-\end_layout
-
-\begin_layout Subsubsection*
-metadata-change-log (default: on)
-\end_layout
-
-\begin_layout Standard
-Enable/disable writing of change log for metadata operations.
-\end_layout
-
-\begin_layout Subsubsection*
-entry-change-log (default: on)
-\end_layout
-
-\begin_layout Standard
-Enable/disable writing of change log for entry operations.
-\end_layout
-
-\begin_layout Subsection*
-Locking options
-\end_layout
-
-\begin_layout Standard
-These options let you specify the number of lock servers to use for each
- class of file operations.
- The default values are satisfactory in most cases.
- If you are extra paranoid, you may want to increase the values.
- However, be very cautious if you set the data- or entry- lock server counts
- to zero, since this can result in 
-\emph on
-lost data.
-
-\emph default
- For example, if you set the data-lock-server-count to zero, and two application
-s write to the same region of a file, there is a possibility that none of
- your servers will have all the data.
- In other words, the copies will be 
-\emph on
-inconsistent
-\emph default
-, and 
-\emph on
-incomplete
-\emph default
-.
- Do not set data- and entry- lock server counts to zero unless you absolutely
- know what you are doing and agree to not hold GlusterFS responsible for
- any lost data.
-\end_layout
-
-\begin_layout Subsubsection*
-data-lock-server-count (default: 1)
-\end_layout
-
-\begin_layout Standard
-Number of lock servers to use for data operations.
-\end_layout
-
-\begin_layout Subsubsection*
-metadata-lock-server-count (default: 0)
-\end_layout
-
-\begin_layout Standard
-Number of lock servers to use for metadata operations.
-\end_layout
-
-\begin_layout Subsubsection*
-entry-lock-server-count (default: 1)
-\end_layout
-
-\begin_layout Standard
-Number of lock servers to use for entry operations.
-\end_layout
-
-\begin_layout Section*
-Known Issues
-\end_layout
-
-\begin_layout Subsection*
-Self-heal of file with more than one link (hard links):
-\end_layout
-
-\begin_layout Standard
-Consider two servers, A and B.
- Assume A is down, and the user creates a file `new' as a hard link to a
- file `old'.
- When A comes back up, replicate will see that the file `new' does not exist on
- A, and self-heal will create the file and copy the contents from B.
- However, now on server A the file `new' is not a link to the file `old'
- but an entirely different file.
-\end_layout
-
-\begin_layout Standard
-We know of no easy way to fix this problem, but we will try to fix it in
- forthcoming releases.
-\end_layout
-
-\begin_layout Subsection*
-File re-opening after a server comes back up:
-\end_layout
-
-\begin_layout Standard
-If a server A goes down and comes back up, any files which were opened while
- A was down and are still open will not have their writes replicated on
- A.
- In other words, data replication only happens on those servers which were
- alive when the file was opened.
-\end_layout
-
-\begin_layout Standard
-This is a rather tricky issue but we hope to fix it very soon.
-\end_layout
-
-\begin_layout Section*
-Frequently Asked Questions
-\end_layout
-
-\begin_layout Subsection*
-1.
- How can I force self-heal to happen?
-\end_layout
-
-\begin_layout Standard
-You can force self-heal to happen on your cluster by running a script or
- a command that accesses every file.
- A simple way to do it would be:
-\end_layout
-
-\begin_layout Standard
-\begin_inset ERT
-status open
-
-\begin_layout Standard
-
-\end_layout
-
-\begin_layout Standard
-
-
-\backslash
-begin{verbatim}
-\end_layout
-
-\begin_layout Standard
-
-$ ls -lR
-\end_layout
-
-\begin_layout Standard
-
-
-\backslash
-end{verbatim}
-\end_layout
-
-\begin_layout Standard
-
-\end_layout
-
-\end_inset
-
-
-\end_layout
-
-\begin_layout Standard
-Run the command in all directories which you want to forcibly self-heal.
-\end_layout
-
-\begin_layout Subsection*
-2.
- Which backend filesystem should I use for replicate?
-\end_layout
-
-\begin_layout Standard
-You can use any backend filesystem that supports extended attributes.
- We know of users successfully using XFR, ReiserFS, and Ext3.
-\end_layout
-
-\begin_layout Subsection*
-3.
- What can I do to improve replicate performance?
-\end_layout
-
-\begin_layout Standard
-Try loading performance translators such as io-threads, write-behind, io-cache,
- and read-ahead depending on your workload.
- If you are willing to sacrifice correctness in corner cases, you can experiment
- with the lock-server-count and the change-log options (see above).
- As warned earlier, be very careful!
-\end_layout
-
-\begin_layout Subsection*
-4.
- How can I selectively replicate files?
-\end_layout
-
-\begin_layout Standard
-There is no support for selective replication in replicate itself.
- You can achieve selective replication by loading the unify translator over
- replicate, and using the switch scheduler.
- Configure unify with two subvolumes, one of them being replicate.
- Using the switch scheduler, schedule all files for which you need replication
- to the replicate subvolume.
- Consult unify and switch documentation for more details.
-\end_layout
-
-\begin_layout Section*
-Contact
-\end_layout
-
-\begin_layout Standard
-If you need more assistance on replicate, contact us on the mailing list <gluster-user
-s@gluster.org> (visit gluster.org for details on how to subscribe).
-\end_layout
-
-\begin_layout Standard
-Send you comments and suggestions about this document to <vikas@gluster.com>.
-\end_layout
-
-\end_body
-\end_document
diff --git a/doc/replicate.pdf b/doc/replicate.pdf
deleted file mode 100644
index b7212af2b4e..00000000000
--- a/doc/replicate.pdf
+++ /dev/null
diff --git a/doc/solaris-related-xattrs.txt b/doc/solaris-related-xattrs.txt
deleted file mode 100644
index e26efa5d189..00000000000
--- a/doc/solaris-related-xattrs.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-                   Solaris Extended Attributes
-
-In solaris extended attributes are logically supported as files
-within the filesystem.  The file system is therefore augmented 
-with an orthogonal namespace of file attributes. Attribute values
-are accessed by file descriptors obtained through a special attribute
-interface. This type of logical view of "attributes as files" allows 
-the leveraging of existing file system interface functionality to 
-support the construction, deletion and manipulation of attributes. 
-
-But as we have tested through this functionality provided by Solaris
-we have come accross two major issues as written below.
-
-1. Symlink XATTR_NOFOLLOW not present for creating extended attributes 
-   directly on the symlinks like other platforms Linux,MAC-OSX,BSD etc.
-   An implementation is present for O_NOFOLLOW for "openat()" call sets 
-   up errno ELOOP whenever encountered with a symlink and also another 
-   implementation AT_SYMLINK_NOFOLLOW which is not present for calls like
-   "attropen(), openat()"
-
-   a snippet of test code which helped us understand this behaviour
-   --------------------------------------
-    attrfd = attropen (path, key, 
-    flags|AT_SYMLINK_NOFOLLOW|O_CREAT|O_WRONLY|O_NOFOLLOW, 0777);
-    if (attrfd >= 0) {
-      ftruncate (attrfd, 0);
-      ret = write (attrfd, value, size);
-      close (attrfd);
-    } else {
-       fprintf (stderr, "Couldn't set extended attribute for %s (%d)\n", 
-       	        path, errno);
-    }		
-    --------------------------------------
-
-2. Extended attribute support for special files like device files, fifo files
-   is not supported under solaris. 
-
-Apart from these glitches almost everything regarding porting functionality
-for extended attribute calls has been properly implemented in compat.c 
-with writing wrapper around functions over
-"attropen()", "openat()", "unlinkat()"
-
-
-
diff --git a/doc/stat-prefetch-design.txt b/doc/stat-prefetch-design.txt
deleted file mode 100644
index 06d0ad37e7d..00000000000
--- a/doc/stat-prefetch-design.txt
+++ /dev/null
@@ -1,154 +0,0 @@
-what is stat-prefetch?
-======================
-It is a translator which caches the dentries read in readdir. This dentry
-list is stored in the context of fd. Later when lookup happens on 
-[parent-inode, basename (path)] combination, this list is searched for the
-basename. The dentry thus searched is used to fill up the stat corresponding
-to path being looked upon, thereby short-cutting lookup calls. This cache is
-preserved till closedir is called on the fd. The purpose of this translator 
-is to optimize operations like 'ls -l', where a readdir is followed by 
-lookup (stat) calls on each directory entry.
-
-1. stat-prefetch harnesses the efficiency of short lookup calls 
-   (saves network roundtrip time for lookup calls from being accounted to 
-   the stat call).
-2. To maintain the correctness, it does lookup-behind - lookup is winded to 
-   underlying translators after it is unwound to upper translators. 
-   lookup-behind is necessary as inode gets populated in server inode table
-   only in lookup-cbk and also because various translators store their 
-   contexts in inode contexts during lookup calls.
-
-fops to be implemented:
-=======================
-* lookup
-  1. check the dentry cache stored in context of fds opened by the same process 
-     on parent inode for basename. If found unwind with cached stat, else wind
-     the lookup call to underlying translators. 
-  2. stat is stored in the context of inode if the path being looked upon 
-     happens to be directory. This stat will be used to fill postparent stat
-     when lookup happens on any of the directory contents.
-
-* readdir
-  1. cache the direntries returned in readdir_cbk in the context of fd.
-  2. if the readdir is happening on non-expected offsets (means a seekdir/rewinddir 
-     has happened), cache has to be flushed.
-  3. delete the entry corresponding to basename of path on which fd is opened 
-     from cache stored in parent.
-
-* chmod/fchmod
-  delete the entry corresponding to basename from cache stored in context of
-  fds opened on parent inode, since these calls change st_mode and st_ctime of 
-  stat.
- 
-* chown/fchown
-  delete the entry corresponding to basename from cache stored in context of 
-  fds opened on parent inode, since these calls change st_uid/st_gid and 
-  st_ctime of stat.
-
-* truncate/ftruncate
-  delete the entry corresponding to basename from cache stored in context of 
-  fds opened on parent inode, since these calls change st_size/st_mtime of stat.
-
-* utimens
-  delete the entry corresponding to basename from cache stored in context of 
-  fds opened on parent inode, since this call changes st_atime/st_mtime of stat.
-
-* readlink
-  delete the entry corresponding to basename from cache stored in context of fds
-  opened on parent inode, since this call changes st_atime of stat.
- 
-* unlink
-  1. delete the entry corresponding to basename from cache stored in context of 
-     fds, opened on parent directory containing the file being unlinked.
-  2. delete the entry corresponding to basename of parent directory from cache
-     of grand-parent.
-
-* rmdir
-  1. delete the entry corresponding to basename from cache stored in context of
-     fds opened on parent inode.
-  2. remove the entire cache from all fds opened on inode corresponding to 
-     directory being removed.
-  3. delete the entry correspondig to basename of parent from cache stored in
-     grand-parent.
-
-* readv
-  delete the entry corresponding to basename from cache stored in context of fds
-  opened on parent inode, since readv changes st_atime of file. 
-
-* writev
-  delete the entry corresponding to basename from cache stored in context of fds
-  opened on parent inode, since writev can possibly change st_size and definitely
-  changes st_mtime of file.
-
-* fsync
-  there is a confusion here as to whether fsync updates mtime/ctimes. Disk based
-  filesystems (atleast ext2) just writes the times stored in inode to disk 
-  during fsync and not the time at which fsync is being done. But in glusterfs, 
-  a translator like write-behind actually sends writes during fsync which will 
-  change mtime/ctime. Hence stat-prefetch implements fsync to delete the entry 
-  corresponding to basename from cache stored in context of fds opened on parent
-  inode.
- 
-* rename
-  1. remove entry corresponding to oldname from cache stored in fd contexts of 
-     oldparent.
-  2. remove entry corresponding to newname from cache stored in fd contexts of
-     newparent. 
-  3. remove entry corresponding to oldparent from cache stored in 
-     old-grand-parent, since removing oldname changes st_mtime and st_ctime
-     of oldparent stat.
-  4. remove entry corresponding to newparent from cache stored in 
-     new-grand-parent, since adding newname changes st_mtime and st_ctime
-     of newparent stat.
-  5. if oldname happens to be a directory, remove entire cache from all fds 
-     opened on it.
-
-* create/mknod/mkdir/symlink/link
-  delete entry corresponding to basename of parent directory in which these 
-  operations are happening, from cache stored in context of fds opened on
-  grand-parent, since adding a new entry to a directory changes st_mtime
-  and st_ctime of parent directory.
-
-* setxattr/removexattr
-  delete the entry corresponding to basename from cache stored in context of
-  fds opened on parent inode, since setxattr changes st_ctime of file.
-
-* setdents
-  1. remove entry corresponding to basename of path on which fd is opened from
-     cache stored in context of fds opened on parent.
-  2. for each of the entry in the direntry list, delete from cache stored in 
-     context of fd, the entry corresponding to basename of path being passed.
-
-* getdents
-  1. remove entry corresponding to basename of path on which fd is opened from
-     cache stored in parent, since getdents changes st_atime. 
-  2. remove entries corresponding to symbolic links from cache, since readlink 
-     would've changed st_atime.
-
-* checksum
-  delete the entry corresponding to basename from cache stored in context of
-  fds opened on parent inode, since st_atime is changed during this call.
-
-* xattrop/fxattrop
-  delete the entry corresponding to basename from cache stored in context of fds
-  opened on parent inode, since these calls modify st_ctime of file.
-
-callbacks to be implemented:
-============================
-* releasedir
-  free the context stored in fd.
-
-* forget
-  dree the stat if the inode corresponds to a directory.
-
-limitations:
-============
-* since a readdir does not return extended attributes of file, if need_xattr is
-  set, short-cutting of lookup does not happen and lookup is passed to 
-  underlying translators.
-
-* posix_readdir does not check whether the dentries are spanning across multiple
-  mount points. Hence it is not transforming inode numbers in stat buffers if 
-  posix is configured to allow export directory spanning on multiple mountpoints.
-  This is a bug which needs to be fixed. posix_readdir should treat dentries the 
-  same way as if lookup is happening on dentries.
diff --git a/doc/translator-options.txt b/doc/translator-options.txt
deleted file mode 100644
index 278ef5b0083..00000000000
--- a/doc/translator-options.txt
+++ /dev/null
@@ -1,224 +0,0 @@
-mount/fuse:
-	* direct-io-mode            GF_OPTION_TYPE_BOOL   on|off|yes|no
-	* mount-point (mountpoint)  GF_OPTION_TYPE_PATH   <any-posix-valid-path>
-	* attribute-timeout         GF_OPTION_TYPE_DOUBLE   0.0 
-	* entry-timeout             GF_OPTION_TYPE_DOUBLE   0.0
-
-protocol/server:
- 	* transport-type            GF_OPTION_TYPE_STR    tcp|socket|ib-verbs|unix|ib-sdp|
-	  			    			  tcp/client|ib-verbs/client
-        * volume-filename.*         GF_OPTION_TYPE_PATH
-	* inode-lru-limit           GF_OPTION_TYPE_INT    0-(1 * GF_UNIT_MB)
-	* client-volume-filename    GF_OPTION_TYPE_PATH
-
-protocol/client:
-	* username                  GF_OPTION_TYPE_ANY
-	* password                  GF_OPTION_TYPE_ANY 
-	* transport-type            GF_OPTION_TYPE_STR   tcp|socket|ib-verbs|unix|ib-sdp|
-	  			    			 tcp/client|ib-verbs/client
-	* remote-host               GF_OPTION_TYPE_ANY 
-	* remote-subvolume          GF_OPTION_TYPE_ANY 
-	* transport-timeout         GF_OPTION_TYPE_TIME  5-1013 
-
-cluster/replicate:
-	* read-subvolume	    GF_OPTION_TYPE_XLATOR
-	* favorite-child 	    GF_OPTION_TYPE_XLATOR
-	* data-self-heal 	    GF_OPTION_TYPE_BOOL 
-	* metadata-self-heal 	    GF_OPTION_TYPE_BOOL
-	* entry-self-heal 	    GF_OPTION_TYPE_BOOL 
-	* data-change-log 	    GF_OPTION_TYPE_BOOL 
-	* metadata-change-log 	    GF_OPTION_TYPE_BOOL
-	* entry-change-log 	    GF_OPTION_TYPE_BOOL
-	* data-lock-server-count    GF_OPTION_TYPE_INT    0
-	* metadata-lock-server-count GF_OPTION_TYPE_INT   0
-	* entry-lock-server-count    GF_OPTION_TYPE_INT   0
-
-cluster/distribute:
-	* lookup-unhashed           GF_OPTION_TYPE_BOOL 
-
-cluster/unify:
-	* namespace		    GF_OPTION_TYPE_XLATOR 
-	* scheduler 		    GF_OPTION_TYPE_STR    alu|rr|random|nufa|switch 
-	* self-heal 		    GF_OPTION_TYPE_STR    foreground|background|off
-	* optimist  		    GF_OPTION_TYPE_BOOL 
-
-cluster/nufa:
-	local-volume-name           GF_OPTION_TYPE_XLATOR 
-
-cluster/stripe:
-	* block-size		    GF_OPTION_TYPE_ANY 
-	* use-xattr  		    GF_OPTION_TYPE_BOOL
-
-debug/trace:
-	* include-ops (include)     GF_OPTION_TYPE_STR
-	* exclude-ops (exclude)     GF_OPTION_TYPE_STR 
-
-encryption/rot-13:
-	* encrypt-write             GF_OPTION_TYPE_BOOL
-	* decrypt-read  	    GF_OPTION_TYPE_BOOL 
-
-features/path-convertor:
-	* start-offset              GF_OPTION_TYPE_INT     0-4095 
-	* end-offset 		    GF_OPTION_TYPE_INT	   1-4096 
-	* replace-with		    GF_OPTION_TYPE_ANY
-
-features/trash:
-	* trash-dir	            GF_OPTION_TYPE_PATH 
-
-features/locks:
-	* mandatory-locks (mandatory) GF_OPTION_TYPE_BOOL 
-
-features/filter:
-	* root-squashing	    GF_OPTION_TYPE_BOOL 
-	* read-only                 GF_OPTION_TYPE_BOOL
-	* fixed-uid		    GF_OPTION_TYPE_INT
-	* fixed-gid		    GF_OPTION_TYPE_INT
-	* translate-uid             GF_OPTION_TYPE_ANY 
-	* translate-gid		    GF_OPTION_TYPE_ANY
-	* filter-uid		    GF_OPTION_TYPE_ANY 
-	* filter-gid		    GF_OPTION_TYPE_ANY 
-
-features/quota:
-	* min-free-disk-limit	    GF_OPTION_TYPE_PERCENT
-	* refresh-interval	    GF_OPTION_TYPE_TIME
-	* disk-usage-limit	    GF_OPTION_TYPE_SIZET 
-
-storage/posix:
-	* o-direct		    GF_OPTION_TYPE_BOOL
-	* directory		    GF_OPTION_TYPE_PATH
-	* export-statfs-size	    GF_OPTION_TYPE_BOOL
-	* mandate-attribute	    GF_OPTION_TYPE_BOOL
-
-storage/bdb:
-	* directory                 GF_OPTION_TYPE_PATH
-	* logdir		    GF_OPTION_TYPE_PATH
-	* errfile		    GF_OPTION_TYPE_PATH
-	* dir-mode		    GF_OPTION_TYPE_ANY
-	* file-mode		    GF_OPTION_TYPE_ANY
-	* page-size		    GF_OPTION_TYPE_SIZET
-	* lru-limit		    GF_OPTION_TYPE_INT
-	* lock-timeout		    GF_OPTION_TYPE_TIME
-	* checkpoint-timeout	    GF_OPTION_TYPE_TIME
-	* transaction-timeout	    GF_OPTION_TYPE_TIME
-	* mode			    GF_OPTION_TYPE_BOOL
-	* access-mode		    GF_OPTION_TYPE_STR
-
-performance/read-ahead:
-	* force-atime-update        GF_OPTION_TYPE_BOOL 
-	* page-size		    GF_OPTION_TYPE_SIZET (64 * GF_UNIT_KB)-(2 * GF_UNIT_MB)
-	* page-count		    GF_OPTION_TYPE_INT   1-16 
-
-performance/write-behind:
-	* flush-behind		    GF_OPTION_TYPE_BOOL
-	* aggregate-size	    GF_OPTION_TYPE_SIZET  (128 * GF_UNIT_KB)-(4 * GF_UNIT_MB) 
-	* window-size		    GF_OPTION_TYPE_SIZET  (512 * GF_UNIT_KB)-(1 * GF_UNIT_GB) 
-	* enable-O_SYNC		    GF_OPTION_TYPE_BOOL  
-	* disable-for-first-nbytes  GF_OPTION_TYPE_SIZET  1 - (1 * GF_UNIT_MB) 
-
-performance/symlink-cache:
-
-performance/io-threads:
-	* thread-count	            GF_OPTION_TYPE_INT    1-32
-
-performance/io-cache:
-	* priority	            GF_OPTION_TYPE_ANY 
-	* cache-timeout (force-revalidate-timeout) GF_OPTION_TYPE_INT 0-60 
-	* page-size	            GF_OPTION_TYPE_SIZET  (16 * GF_UNIT_KB)-(4 * GF_UNIT_MB) 
-        * cache-size                GF_OPTION_TYPE_SIZET  (4 * GF_UNIT_MB)-(6 * GF_UNIT_GB)
-
-performance/quick-read:
-        * cache-timeout             GF_OPTION_TYPE_INT    1-60
-        * max-file-size             GF_OPTION_TYPE_SIZET  0-(1000 * GF_UNIT_KB)
-
-auth:
-- addr:
-	* auth.addr.*.allow	    GF_OPTION_TYPE_ANY 
-	* auth.addr.*.reject	    GF_OPTION_TYPE_ANY 
-
-- login:
-	* auth.login.*.allow	    GF_OPTION_TYPE_ANY 
-	* auth.login.*.password	    GF_OPTION_TYPE_ANY
-
-scheduler/alu:
-	* scheduler.alu.order (alu.order) 
-	                            GF_OPTION_TYPE_ANY 
-	* scheduler.alu.disk-usage.entry-threshold (alu.disk-usage.entry-threshold)
-	  			    GF_OPTION_TYPE_SIZET
-        * scheduler.alu.disk-usage.exit-threshold (alu.disk-usage.exit-threshold)
-                                    GF_OPTION_TYPE_SIZET
-        * scheduler.alu.write-usage.entry-threshold (alu.write-usage.entry-threshold)
-                                    GF_OPTION_TYPE_SIZET
-        * scheduler.alu.write-usage.exit-threshold (alu.write-usage.exit-threshold)
-                                    GF_OPTION_TYPE_SIZET 
-        * scheduler.alu.read-usage.entry-threshold (alu.read-usage.entry-threshold)
-                                    GF_OPTION_TYPE_SIZET
-        * scheduler.alu.read-usage.exit-threshold (alu.read-usage.exit-threshold)
-                                    GF_OPTION_TYPE_SIZET 
-        * scheduler.alu.open-files-usage.entry-threshold (alu.open-files-usage.entry-threshold)
-                                    GF_OPTION_TYPE_INT
-        * scheduler.alu.open-files-usage.exit-threshold (alu.open-files-usage.exit-threshold)
-                                    GF_OPTION_TYPE_INT 
-        * scheduler.read-only-subvolumes (alu.read-only-subvolumes)
-                                    GF_OPTION_TYPE_ANY 
-        * scheduler.refresh-interval (alu.refresh-interval)
-                                    GF_OPTION_TYPE_TIME
-        * scheduler.limits.min-free-disk (alu.limits.min-free-disk)
-                                    GF_OPTION_TYPE_PERCENT
-        * scheduler.alu.stat-refresh.num-file-create (alu.stat-refresh.num-file-create)
-                                    GF_OPTION_TYPE_INT
-
-scheduler/nufa:
-	* scheduler.refresh-interval (nufa.refresh-interval) 
-	                            GF_OPTION_TYPE_TIME
-	* scheduler.limits.min-free-disk (nufa.limits.min-free-disk) 
-	                            GF_OPTION_TYPE_PERCENT
-	* scheduler.local-volume-name (nufa.local-volume-name) 
-	                            GF_OPTION_TYPE_XLATOR
-
-scheduler/random:
-	* scheduler.refresh-interval (random.refresh-interval) GF_OPTION_TYPE_TIME
-	* scheduler.limits.min-free-disk (random.limits.min-free-disk) GF_OPTION_TYPE_PERCENT
-
-scheduler/rr:
-	* scheduler.refresh-interval (rr.refresh-interval)  GF_OPTION_TYPE_TIME
-	* scheduler.limits.min-free-disk (rr.limits.min-free-disk) GF_OPTION_TYPE_PERCENT
-	* scheduler.read-only-subvolumes (rr.read-only-subvolumes) GF_OPTION_TYPE_ANY
-
-scheduler/switch:
-	* scheduler.read-only-subvolumes (switch.read-only-subvolumes) GF_OPTION_TYPE_ANY
-	* scheduler.local-volume-name (switch.nufa.local-volume-name) GF_OPTION_TYPE_XLATOR
-	* scheduler.switch.case (switch.case) GF_OPTION_TYPE_ANY
-
-transport/ib-verbs:
-	* transport.ib-verbs.port (ib-verbs-port) GF_OPTION_TYPE_INT 1-4
-	                         check the option by 'ibv_devinfo'
-        * transport.ib-verbs.mtu (ib-verbs-mtu) GF_OPTION_TYPE_INT
-	* transport.ib-verbs.device-name (ib-verbs-device-name) GF_OPTION_TYPE_ANY,
-                                 check by 'ibv_devinfo'
-        * transport.ib-verbs.work-request-send-size (ib-verbs-work-request-send-size)
-	                         GF_OPTION_TYPE_INT,
-        * transport.ib-verbs.work-request-recv-size (ib-verbs-work-request-recv-size)
-	                         GF_OPTION_TYPE_INT
-        * transport.ib-verbs.work-request-send-count (ib-verbs-work-request-send-count)
-                                 GF_OPTION_TYPE_INT
-        * transport.ib-verbs.work-request-recv-count (ib-verbs-work-request-recv-count)
-                                 GF_OPTION_TYPE_INT
-        * remote-port (transport.remote-port,transport.ib-verbs.remote-port)
-	                         GF_OPTION_TYPE_INT 
-        * transport.ib-verbs.listen-port  GF_OPTION_TYPE_INT 
-        * transport.ib-verbs.connect-path (connect-path) GF_OPTION_TYPE_ANY 
-        * transport.ib-verbs.bind-path (bind-path) GF_OPTION_TYPE_ANY 
-        * transport.ib-verbs.listen-path (listen-path) GF_OPTION_TYPE_ANY 
-        * transport.address-family (address-family) GF_OPTION_TYPE_STR inet|inet6|inet/inet6|
-                                                                       inet6/inet|unix|inet-sdp
-
-transport/socket:
-	* transport.remote-port (remote-port,transport.socket.remote-port) GF_OPTION_TYPE_INT 
-	* transport.socket.listen-port (listen-port) GF_OPTION_TYPE_INT 
-	* transport.socket.bind-address (bind-address) GF_OPTION_TYPE_ANY 
-	* transport.socket.connect-path (connect-path) GF_OPTION_TYPE_ANY 
-	* transport.socket.bind-path (bind-path)       GF_OPTION_TYPE_ANY 
-	* transport.socket.listen-path (listen-path)   GF_OPTION_TYPE_ANY
-	* transport.address-family (address-family)    GF_OPTION_TYPE_STR  inet|inet6|
-	  			   		       			   inet/inet6|inet6/inet|
-									   unix|inet-sdp
diff --git a/doc/user-guide/Makefile.am b/doc/user-guide/Makefile.am
deleted file mode 100644
index 800e7321d79..00000000000
--- a/doc/user-guide/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-info_TEXINFOS = user-guide.texi
-CLEANFILES = *~ 
-DISTCLEANFILES = .deps/*.P *.info *vti
diff --git a/doc/user-guide/advanced-stripe.odg b/doc/user-guide/advanced-stripe.odg
deleted file mode 100644
index 7686d7091b2..00000000000
--- a/doc/user-guide/advanced-stripe.odg
+++ /dev/null
diff --git a/doc/user-guide/advanced-stripe.pdf b/doc/user-guide/advanced-stripe.pdf
deleted file mode 100644
index ec8b03dcfbb..00000000000
--- a/doc/user-guide/advanced-stripe.pdf
+++ /dev/null
diff --git a/doc/user-guide/colonO-icon.jpg b/doc/user-guide/colonO-icon.jpg
deleted file mode 100644
index 3e66f7a2775..00000000000
--- a/doc/user-guide/colonO-icon.jpg
+++ /dev/null
diff --git a/doc/user-guide/fdl.texi b/doc/user-guide/fdl.texi
deleted file mode 100644
index 92cfa81a79f..00000000000
--- a/doc/user-guide/fdl.texi
+++ /dev/null
@@ -1,454 +0,0 @@
-
-@c @node GNU Free Documentation License
-@c @appendixsec GNU Free Documentation License
-
-@cindex FDL, GNU Free Documentation License
-@center Version 1.2, November 2002
-
-@display
-Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc.
-59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
-
-Everyone is permitted to copy and distribute verbatim copies
-of this license document, but changing it is not allowed.
-@end display
-
-@enumerate 0
-@item
-PREAMBLE
-
-The purpose of this License is to make a manual, textbook, or other
-functional and useful document @dfn{free} in the sense of freedom: to
-assure everyone the effective freedom to copy and redistribute it,
-with or without modifying it, either commercially or noncommercially.
-Secondarily, this License preserves for the author and publisher a way
-to get credit for their work, while not being considered responsible
-for modifications made by others.
-
-This License is a kind of ``copyleft'', which means that derivative
-works of the document must themselves be free in the same sense.  It
-complements the GNU Affero General Public License, which is a copyleft
-license designed for free software.
-
-We have designed this License in order to use it for manuals for free
-software, because free software needs free documentation: a free
-program should come with manuals providing the same freedoms that the
-software does.  But this License is not limited to software manuals;
-it can be used for any textual work, regardless of subject matter or
-whether it is published as a printed book.  We recommend this License
-principally for works whose purpose is instruction or reference.
-
-@item
-APPLICABILITY AND DEFINITIONS
-
-This License applies to any manual or other work, in any medium, that
-contains a notice placed by the copyright holder saying it can be
-distributed under the terms of this License.  Such a notice grants a
-world-wide, royalty-free license, unlimited in duration, to use that
-work under the conditions stated herein.  The ``Document'', below,
-refers to any such manual or work.  Any member of the public is a
-licensee, and is addressed as ``you''.  You accept the license if you
-copy, modify or distribute the work in a way requiring permission
-under copyright law.
-
-A ``Modified Version'' of the Document means any work containing the
-Document or a portion of it, either copied verbatim, or with
-modifications and/or translated into another language.
-
-A ``Secondary Section'' is a named appendix or a front-matter section
-of the Document that deals exclusively with the relationship of the
-publishers or authors of the Document to the Document's overall
-subject (or to related matters) and contains nothing that could fall
-directly within that overall subject.  (Thus, if the Document is in
-part a textbook of mathematics, a Secondary Section may not explain
-any mathematics.)  The relationship could be a matter of historical
-connection with the subject or with related matters, or of legal,
-commercial, philosophical, ethical or political position regarding
-them.
-
-The ``Invariant Sections'' are certain Secondary Sections whose titles
-are designated, as being those of Invariant Sections, in the notice
-that says that the Document is released under this License.  If a
-section does not fit the above definition of Secondary then it is not
-allowed to be designated as Invariant.  The Document may contain zero
-Invariant Sections.  If the Document does not identify any Invariant
-Sections then there are none.
-
-The ``Cover Texts'' are certain short passages of text that are listed,
-as Front-Cover Texts or Back-Cover Texts, in the notice that says that
-the Document is released under this License.  A Front-Cover Text may
-be at most 5 words, and a Back-Cover Text may be at most 25 words.
-
-A ``Transparent'' copy of the Document means a machine-readable copy,
-represented in a format whose specification is available to the
-general public, that is suitable for revising the document
-straightforwardly with generic text editors or (for images composed of
-pixels) generic paint programs or (for drawings) some widely available
-drawing editor, and that is suitable for input to text formatters or
-for automatic translation to a variety of formats suitable for input
-to text formatters.  A copy made in an otherwise Transparent file
-format whose markup, or absence of markup, has been arranged to thwart
-or discourage subsequent modification by readers is not Transparent.
-An image format is not Transparent if used for any substantial amount
-of text.  A copy that is not ``Transparent'' is called ``Opaque''.
-
-Examples of suitable formats for Transparent copies include plain
-@sc{ascii} without markup, Texinfo input format, La@TeX{} input
-format, @acronym{SGML} or @acronym{XML} using a publicly available
-@acronym{DTD}, and standard-conforming simple @acronym{HTML},
-PostScript or @acronym{PDF} designed for human modification.  Examples
-of transparent image formats include @acronym{PNG}, @acronym{XCF} and
-@acronym{JPG}.  Opaque formats include proprietary formats that can be
-read and edited only by proprietary word processors, @acronym{SGML} or
-@acronym{XML} for which the @acronym{DTD} and/or processing tools are
-not generally available, and the machine-generated @acronym{HTML},
-PostScript or @acronym{PDF} produced by some word processors for
-output purposes only.
-
-The ``Title Page'' means, for a printed book, the title page itself,
-plus such following pages as are needed to hold, legibly, the material
-this License requires to appear in the title page.  For works in
-formats which do not have any title page as such, ``Title Page'' means
-the text near the most prominent appearance of the work's title,
-preceding the beginning of the body of the text.
-
-A section ``Entitled XYZ'' means a named subunit of the Document whose
-title either is precisely XYZ or contains XYZ in parentheses following
-text that translates XYZ in another language.  (Here XYZ stands for a
-specific section name mentioned below, such as ``Acknowledgements'',
-``Dedications'', ``Endorsements'', or ``History''.)  To ``Preserve the Title''
-of such a section when you modify the Document means that it remains a
-section ``Entitled XYZ'' according to this definition.
-
-The Document may include Warranty Disclaimers next to the notice which
-states that this License applies to the Document.  These Warranty
-Disclaimers are considered to be included by reference in this
-License, but only as regards disclaiming warranties: any other
-implication that these Warranty Disclaimers may have is void and has
-no effect on the meaning of this License.
-
-@item
-VERBATIM COPYING
-
-You may copy and distribute the Document in any medium, either
-commercially or noncommercially, provided that this License, the
-copyright notices, and the license notice saying this License applies
-to the Document are reproduced in all copies, and that you add no other
-conditions whatsoever to those of this License.  You may not use
-technical measures to obstruct or control the reading or further
-copying of the copies you make or distribute.  However, you may accept
-compensation in exchange for copies.  If you distribute a large enough
-number of copies you must also follow the conditions in section 3.
-
-You may also lend copies, under the same conditions stated above, and
-you may publicly display copies.
-
-@item
-COPYING IN QUANTITY
-
-If you publish printed copies (or copies in media that commonly have
-printed covers) of the Document, numbering more than 100, and the
-Document's license notice requires Cover Texts, you must enclose the
-copies in covers that carry, clearly and legibly, all these Cover
-Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on
-the back cover.  Both covers must also clearly and legibly identify
-you as the publisher of these copies.  The front cover must present
-the full title with all words of the title equally prominent and
-visible.  You may add other material on the covers in addition.
-Copying with changes limited to the covers, as long as they preserve
-the title of the Document and satisfy these conditions, can be treated
-as verbatim copying in other respects.
-
-If the required texts for either cover are too voluminous to fit
-legibly, you should put the first ones listed (as many as fit
-reasonably) on the actual cover, and continue the rest onto adjacent
-pages.
-
-If you publish or distribute Opaque copies of the Document numbering
-more than 100, you must either include a machine-readable Transparent
-copy along with each Opaque copy, or state in or with each Opaque copy
-a computer-network location from which the general network-using
-public has access to download using public-standard network protocols
-a complete Transparent copy of the Document, free of added material.
-If you use the latter option, you must take reasonably prudent steps,
-when you begin distribution of Opaque copies in quantity, to ensure
-that this Transparent copy will remain thus accessible at the stated
-location until at least one year after the last time you distribute an
-Opaque copy (directly or through your agents or retailers) of that
-edition to the public.
-
-It is requested, but not required, that you contact the authors of the
-Document well before redistributing any large number of copies, to give
-them a chance to provide you with an updated version of the Document.
-
-@item
-MODIFICATIONS
-
-You may copy and distribute a Modified Version of the Document under
-the conditions of sections 2 and 3 above, provided that you release
-the Modified Version under precisely this License, with the Modified
-Version filling the role of the Document, thus licensing distribution
-and modification of the Modified Version to whoever possesses a copy
-of it.  In addition, you must do these things in the Modified Version:
-
-@enumerate A
-@item
-Use in the Title Page (and on the covers, if any) a title distinct
-from that of the Document, and from those of previous versions
-(which should, if there were any, be listed in the History section
-of the Document).  You may use the same title as a previous version
-if the original publisher of that version gives permission.
-
-@item
-List on the Title Page, as authors, one or more persons or entities
-responsible for authorship of the modifications in the Modified
-Version, together with at least five of the principal authors of the
-Document (all of its principal authors, if it has fewer than five),
-unless they release you from this requirement.
-
-@item
-State on the Title page the name of the publisher of the
-Modified Version, as the publisher.
-
-@item
-Preserve all the copyright notices of the Document.
-
-@item
-Add an appropriate copyright notice for your modifications
-adjacent to the other copyright notices.
-
-@item
-Include, immediately after the copyright notices, a license notice
-giving the public permission to use the Modified Version under the
-terms of this License, in the form shown in the Addendum below.
-
-@item
-Preserve in that license notice the full lists of Invariant Sections
-and required Cover Texts given in the Document's license notice.
-
-@item
-Include an unaltered copy of this License.
-
-@item
-Preserve the section Entitled ``History'', Preserve its Title, and add
-to it an item stating at least the title, year, new authors, and
-publisher of the Modified Version as given on the Title Page.  If
-there is no section Entitled ``History'' in the Document, create one
-stating the title, year, authors, and publisher of the Document as
-given on its Title Page, then add an item describing the Modified
-Version as stated in the previous sentence.
-
-@item
-Preserve the network location, if any, given in the Document for
-public access to a Transparent copy of the Document, and likewise
-the network locations given in the Document for previous versions
-it was based on.  These may be placed in the ``History'' section.
-You may omit a network location for a work that was published at
-least four years before the Document itself, or if the original
-publisher of the version it refers to gives permission.
-
-@item
-For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve
-the Title of the section, and preserve in the section all the
-substance and tone of each of the contributor acknowledgements and/or
-dedications given therein.
-
-@item
-Preserve all the Invariant Sections of the Document,
-unaltered in their text and in their titles.  Section numbers
-or the equivalent are not considered part of the section titles.
-
-@item
-Delete any section Entitled ``Endorsements''.  Such a section
-may not be included in the Modified Version.
-
-@item
-Do not retitle any existing section to be Entitled ``Endorsements'' or
-to conflict in title with any Invariant Section.
-
-@item
-Preserve any Warranty Disclaimers.
-@end enumerate
-
-If the Modified Version includes new front-matter sections or
-appendices that qualify as Secondary Sections and contain no material
-copied from the Document, you may at your option designate some or all
-of these sections as invariant.  To do this, add their titles to the
-list of Invariant Sections in the Modified Version's license notice.
-These titles must be distinct from any other section titles.
-
-You may add a section Entitled ``Endorsements'', provided it contains
-nothing but endorsements of your Modified Version by various
-parties---for example, statements of peer review or that the text has
-been approved by an organization as the authoritative definition of a
-standard.
-
-You may add a passage of up to five words as a Front-Cover Text, and a
-passage of up to 25 words as a Back-Cover Text, to the end of the list
-of Cover Texts in the Modified Version.  Only one passage of
-Front-Cover Text and one of Back-Cover Text may be added by (or
-through arrangements made by) any one entity.  If the Document already
-includes a cover text for the same cover, previously added by you or
-by arrangement made by the same entity you are acting on behalf of,
-you may not add another; but you may replace the old one, on explicit
-permission from the previous publisher that added the old one.
-
-The author(s) and publisher(s) of the Document do not by this License
-give permission to use their names for publicity for or to assert or
-imply endorsement of any Modified Version.
-
-@item
-COMBINING DOCUMENTS
-
-You may combine the Document with other documents released under this
-License, under the terms defined in section 4 above for modified
-versions, provided that you include in the combination all of the
-Invariant Sections of all of the original documents, unmodified, and
-list them all as Invariant Sections of your combined work in its
-license notice, and that you preserve all their Warranty Disclaimers.
-
-The combined work need only contain one copy of this License, and
-multiple identical Invariant Sections may be replaced with a single
-copy.  If there are multiple Invariant Sections with the same name but
-different contents, make the title of each such section unique by
-adding at the end of it, in parentheses, the name of the original
-author or publisher of that section if known, or else a unique number.
-Make the same adjustment to the section titles in the list of
-Invariant Sections in the license notice of the combined work.
-
-In the combination, you must combine any sections Entitled ``History''
-in the various original documents, forming one section Entitled
-``History''; likewise combine any sections Entitled ``Acknowledgements'',
-and any sections Entitled ``Dedications''.  You must delete all
-sections Entitled ``Endorsements.''
-
-@item
-COLLECTIONS OF DOCUMENTS
-
-You may make a collection consisting of the Document and other documents
-released under this License, and replace the individual copies of this
-License in the various documents with a single copy that is included in
-the collection, provided that you follow the rules of this License for
-verbatim copying of each of the documents in all other respects.
-
-You may extract a single document from such a collection, and distribute
-it individually under this License, provided you insert a copy of this
-License into the extracted document, and follow this License in all
-other respects regarding verbatim copying of that document.
-
-@item
-AGGREGATION WITH INDEPENDENT WORKS
-
-A compilation of the Document or its derivatives with other separate
-and independent documents or works, in or on a volume of a storage or
-distribution medium, is called an ``aggregate'' if the copyright
-resulting from the compilation is not used to limit the legal rights
-of the compilation's users beyond what the individual works permit.
-When the Document is included in an aggregate, this License does not
-apply to the other works in the aggregate which are not themselves
-derivative works of the Document.
-
-If the Cover Text requirement of section 3 is applicable to these
-copies of the Document, then if the Document is less than one half of
-the entire aggregate, the Document's Cover Texts may be placed on
-covers that bracket the Document within the aggregate, or the
-electronic equivalent of covers if the Document is in electronic form.
-Otherwise they must appear on printed covers that bracket the whole
-aggregate.
-
-@item
-TRANSLATION
-
-Translation is considered a kind of modification, so you may
-distribute translations of the Document under the terms of section 4.
-Replacing Invariant Sections with translations requires special
-permission from their copyright holders, but you may include
-translations of some or all Invariant Sections in addition to the
-original versions of these Invariant Sections.  You may include a
-translation of this License, and all the license notices in the
-Document, and any Warranty Disclaimers, provided that you also include
-the original English version of this License and the original versions
-of those notices and disclaimers.  In case of a disagreement between
-the translation and the original version of this License or a notice
-or disclaimer, the original version will prevail.
-
-If a section in the Document is Entitled ``Acknowledgements'',
-``Dedications'', or ``History'', the requirement (section 4) to Preserve
-its Title (section 1) will typically require changing the actual
-title.
-
-@item
-TERMINATION
-
-You may not copy, modify, sublicense, or distribute the Document except
-as expressly provided for under this License.  Any other attempt to
-copy, modify, sublicense or distribute the Document is void, and will
-automatically terminate your rights under this License.  However,
-parties who have received copies, or rights, from you under this
-License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-@item
-FUTURE REVISIONS OF THIS LICENSE
-
-The Free Software Foundation may publish new, revised versions
-of the GNU Free Documentation License from time to time.  Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.  See
-@uref{http://www.gnu.org/copyleft/}.
-
-Each version of the License is given a distinguishing version number.
-If the Document specifies that a particular numbered version of this
-License ``or any later version'' applies to it, you have the option of
-following the terms and conditions either of that specified version or
-of any later version that has been published (not as a draft) by the
-Free Software Foundation.  If the Document does not specify a version
-number of this License, you may choose any version ever published (not
-as a draft) by the Free Software Foundation.
-@end enumerate
-
-@page
-@c @appendixsubsec ADDENDUM: How to use this License for your
-@c documents
-@subsection ADDENDUM: How to use this License for your documents
-
-To use this License in a document you have written, include a copy of
-the License in the document and put the following copyright and
-license notices just after the title page:
-
-@smallexample
-@group
-  Copyright (C)  @var{year}  @var{your name}.
-  Permission is granted to copy, distribute and/or modify this document
-  under the terms of the GNU Free Documentation License, Version 1.2
-  or any later version published by the Free Software Foundation;
-  with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
-  Texts.  A copy of the license is included in the section entitled ``GNU
-  Free Documentation License''.
-@end group
-@end smallexample
-
-If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts,
-replace the ``with...Texts.'' line with this:
-
-@smallexample
-@group
-    with the Invariant Sections being @var{list their titles}, with
-    the Front-Cover Texts being @var{list}, and with the Back-Cover Texts
-    being @var{list}.
-@end group
-@end smallexample
-
-If you have Invariant Sections without Cover Texts, or some other
-combination of the three, merge those two alternatives to suit the
-situation.
-
-If your document contains nontrivial examples of program code, we
-recommend releasing these examples in parallel under your choice of
-free software license, such as the GNU Affero General Public License,
-to permit their use in free software.
-
-@c Local Variables:
-@c ispell-local-pdict: "ispell-dict"
-@c End:
-
diff --git a/doc/user-guide/fuse.odg b/doc/user-guide/fuse.odg
deleted file mode 100644
index 61bd103c78b..00000000000
--- a/doc/user-guide/fuse.odg
+++ /dev/null
diff --git a/doc/user-guide/fuse.pdf b/doc/user-guide/fuse.pdf
deleted file mode 100644
index a7d13faff56..00000000000
--- a/doc/user-guide/fuse.pdf
+++ /dev/null
diff --git a/doc/user-guide/ha.odg b/doc/user-guide/ha.odg
deleted file mode 100644
index e4b8b72d08b..00000000000
--- a/doc/user-guide/ha.odg
+++ /dev/null
diff --git a/doc/user-guide/ha.pdf b/doc/user-guide/ha.pdf
deleted file mode 100644
index e372c0ab03e..00000000000
--- a/doc/user-guide/ha.pdf
+++ /dev/null
diff --git a/doc/user-guide/stripe.odg b/doc/user-guide/stripe.odg
deleted file mode 100644
index 79441bf1452..00000000000
--- a/doc/user-guide/stripe.odg
+++ /dev/null
diff --git a/doc/user-guide/stripe.pdf b/doc/user-guide/stripe.pdf
deleted file mode 100644
index b94446feb56..00000000000
--- a/doc/user-guide/stripe.pdf
+++ /dev/null
diff --git a/doc/user-guide/unify.odg b/doc/user-guide/unify.odg
deleted file mode 100644
index ccaa9bf16f9..00000000000
--- a/doc/user-guide/unify.odg
+++ /dev/null
diff --git a/doc/user-guide/unify.pdf b/doc/user-guide/unify.pdf
deleted file mode 100644
index c22027f66e7..00000000000
--- a/doc/user-guide/unify.pdf
+++ /dev/null
diff --git a/doc/user-guide/user-guide.info b/doc/user-guide/user-guide.info
deleted file mode 100644
index 494c0e94ee4..00000000000
--- a/doc/user-guide/user-guide.info
+++ /dev/null
@@ -1,2697 +0,0 @@
-This is ../../../doc/user-guide/user-guide.info, produced by makeinfo version 4.13 from ../../../doc/user-guide/user-guide.texi.
-
-START-INFO-DIR-ENTRY
-* GlusterFS: (user-guide). GlusterFS distributed filesystem user guide
-END-INFO-DIR-ENTRY
-
-   This is the user manual for GlusterFS 2.0.
-
-   Copyright (C) 2007-2010 Gluster, Inc. Permission is granted to
-copy, distribute and/or modify this document under the terms of the GNU
-Free Documentation License, Version 1.2 or any later version published
-by the Free Software Foundation; with no Invariant Sections, no
-Front-Cover Texts, and no Back-Cover Texts. A copy of the license is
-included in the chapter entitled "GNU Free Documentation License".
-
-
-File: user-guide.info,  Node: Top,  Next: Acknowledgements,  Up: (dir)
-
-GlusterFS 2.0 User Guide
-************************
-
-This is the user manual for GlusterFS 2.0.
-
-   Copyright (C) 2007-2010 Gluster, Inc. Permission is granted to
-copy, distribute and/or modify this document under the terms of the GNU
-Free Documentation License, Version 1.2 or any later version published
-by the Free Software Foundation; with no Invariant Sections, no
-Front-Cover Texts, and no Back-Cover Texts. A copy of the license is
-included in the chapter entitled "GNU Free Documentation License".
-
-* Menu:
-
-* Acknowledgements::
-* Introduction::
-* Installation and Invocation::
-* Concepts::
-* Translators::
-* Usage Scenarios::
-* Troubleshooting::
-* GNU Free Documentation Licence::
-* Index::
-
- --- The Detailed Node Listing ---
-
-Installation and Invocation
-
-* Pre requisites::
-* Getting GlusterFS::
-* Building::
-* Running GlusterFS::
-* A Tutorial Introduction::
-
-Running GlusterFS
-
-* Server::
-* Client::
-
-Concepts
-
-* Filesystems in Userspace::
-* Translator::
-* Volume specification file::
-
-Translators
-
-* Storage Translators::
-* Client and Server Translators::
-* Clustering Translators::
-* Performance Translators::
-* Features Translators::
-
-Storage Translators
-
-* POSIX::
-
-Client and Server Translators
-
-* Transport modules::
-* Client protocol::
-* Server protocol::
-
-Clustering Translators
-
-* Unify::
-* Replicate::
-* Stripe::
-
-Performance Translators
-
-* Read Ahead::
-* Write Behind::
-* IO Threads::
-* IO Cache::
-
-Features Translators
-
-* POSIX Locks::
-* Fixed ID::
-
-Miscellaneous Translators
-
-* ROT-13::
-* Trace::
-
-
-File: user-guide.info,  Node: Acknowledgements,  Next: Introduction,  Prev: Top,  Up: Top
-
-Acknowledgements
-****************
-
-GlusterFS continues to be a wonderful and enriching experience for all
-of us involved.
-
-   GlusterFS development would not have been possible at this pace if
-not for our enthusiastic users. People from around the world have
-helped us with bug reports, performance numbers, and feature
-suggestions.  A huge thanks to them all.
-
-   Matthew Paine - for RPMs & general enthu
-
-   Leonardo Rodrigues de Mello - for DEBs
-
-   Julian Perez & Adam D'Auria - for multi-server tutorial
-
-   Paul England - for HA spec
-
-   Brent Nelson - for many bug reports
-
-   Jacques Mattheij - for Europe mirror.
-
-   Patrick Negri - for TCP non-blocking connect.
-        http://gluster.org/core-team.php (<list-hacking@gluster.com>)
-                                                          Gluster 
-
-
-File: user-guide.info,  Node: Introduction,  Next: Installation and Invocation,  Prev: Acknowledgements,  Up: Top
-
-1 Introduction
-**************
-
-GlusterFS is a distributed filesystem. It works at the file level, not
-block level.
-
-   A network filesystem is one which allows us to access remote files. A
-distributed filesystem is one that stores data on multiple machines and
-makes them all appear to be a part of the same filesystem.
-
-   Need for distributed filesystems
-
-   * Scalability: A distributed filesystem allows us to store more data
-     than what can be stored on a single machine.
-
-   * Redundancy: We might want to replicate crucial data on to several
-     machines.
-
-   * Uniform access: One can mount a remote volume (for example your
-     home directory) from any machine and access the same data.
-
-1.1 Contacting us
-=================
-
-You can reach us through the mailing list *gluster-devel*
-(<gluster-devel@nongnu.org>).  
-
-   You can also find many of the developers on IRC, on the `#gluster'
-channel on Freenode (<irc.freenode.net>).  
-
-   The GlusterFS documentation wiki is also useful:
-<http://gluster.org/docs/index.php/GlusterFS>
-
-   For commercial support, you can contact Gluster at: 
-
-     3194 Winding Vista Common
-     Fremont, CA 94539
-     USA.
-
-     Phone: +1 (510) 354 6801
-     Toll free: +1 (888) 813 6309
-     Fax: +1 (510) 372 0604
-
-   You can also email us at <support@gluster.com>.
-
-
-File: user-guide.info,  Node: Installation and Invocation,  Next: Concepts,  Prev: Introduction,  Up: Top
-
-2 Installation and Invocation
-*****************************
-
-* Menu:
-
-* Pre requisites::
-* Getting GlusterFS::
-* Building::
-* Running GlusterFS::
-* A Tutorial Introduction::
-
-
-File: user-guide.info,  Node: Pre requisites,  Next: Getting GlusterFS,  Up: Installation and Invocation
-
-2.1 Pre requisites
-==================
-
-Before installing GlusterFS make sure you have the following components
-installed.
-
-2.1.1 FUSE
-----------
-
-You'll need FUSE version 2.6.0 or higher to use GlusterFS. You can omit
-installing FUSE if you want to build _only_ the server. Note that you
-won't be able to mount a GlusterFS filesystem on a machine that does
-not have FUSE installed.
-
-   FUSE can be downloaded from: <http://fuse.sourceforge.net/>
-
-   To get the best performance from GlusterFS, however, it is
-recommended that you use our patched version of FUSE. See Patched FUSE
-for details.
-
-2.1.2 Patched FUSE
-------------------
-
-The GlusterFS project maintains a patched version of FUSE meant to be
-used with GlusterFS. The patches increase GlusterFS performance. It is
-recommended that all users use the patched FUSE.
-
-   The patched FUSE tarball can be downloaded from:
-
-   <ftp://ftp.gluster.com/pub/gluster/glusterfs/fuse/>
-
-   The specific changes made to FUSE are:
-
-   * The communication channel size between FUSE kernel module and
-     GlusterFS has been increased to 1MB, permitting large reads and
-     writes to be sent in bigger chunks.
-
-   * The kernel's read-ahead boundry has been extended upto 1MB.
-
-   * Block size returned in the `stat()'/`fstat()' calls tuned to 1MB,
-     to make cp and similar commands perform I/O using that block size.
-
-   * `flock()' locking support has been added (although some rework in
-     GlusterFS is needed for perfect compliance).
-
-2.1.3 libibverbs (optional)
----------------------------
-
-This is only needed if you want GlusterFS to use InfiniBand as the
-interconnect mechanism between server and client. You can get it from:
-
-   <http://www.openfabrics.org/downloads.htm>.
-
-2.1.4 Bison and Flex
---------------------
-
-These should be already installed on most Linux systems. If not, use
-your distribution's normal software installation procedures to install
-them. Make sure you install the relevant developer packages also.
-
-
-File: user-guide.info,  Node: Getting GlusterFS,  Next: Building,  Prev: Pre requisites,  Up: Installation and Invocation
-
-2.2 Getting GlusterFS
-=====================
-
-There are many ways to get hold of GlusterFS. For a production
-deployment, the recommended method is to download the latest release
-tarball.  Release tarballs are available at:
-<http://gluster.org/download.php>.
-
-   If you want the bleeding edge development source, you can get them
-from the GNU Arch(1) repository. First you must install GNU Arch
-itself. Then register the GlusterFS archive by doing:
-
-     $ tla register-archive http://arch.sv.gnu.org/archives/gluster
-
-   Now you can check out the source itself:
-
-     $ tla get -A gluster@sv.gnu.org glusterfs--mainline--3.0
-
-   ---------- Footnotes ----------
-
-   (1) <http://www.gnu.org/software/gnu-arch/>
-
-
-File: user-guide.info,  Node: Building,  Next: Running GlusterFS,  Prev: Getting GlusterFS,  Up: Installation and Invocation
-
-2.3 Building
-============
-
-You can skip this section if you're installing from RPMs or DEBs.
-
-   GlusterFS uses the Autotools mechanism to build. As such, the
-procedure is straight-forward. First, change into the GlusterFS source
-directory.
-
-     $ cd glusterfs-<version>
-
-   If you checked out the source from the Arch repository, you'll need
-to run `./autogen.sh' first. Note that you'll need to have Autoconf and
-Automake installed for this.
-
-   Run `configure'.
-
-     $ ./configure
-
-   The configure script accepts the following options:
-
-`--disable-ibverbs'
-     Disable the InfiniBand transport mechanism.
-
-`--disable-fuse-client'
-     Disable the FUSE client.
-
-`--disable-server'
-     Disable building of the GlusterFS server.
-
-`--disable-bdb'
-     Disable building of Berkeley DB based storage translator.
-
-`--disable-mod_glusterfs'
-     Disable building of Apache/lighttpd glusterfs plugins.
-
-`--disable-epoll'
-     Use poll instead of epoll.
-
-`--disable-libglusterfsclient'
-     Disable building of libglusterfsclient
-
-
-   Build and install GlusterFS.
-
-     # make install
-
-   The binaries (`glusterfsd' and `glusterfs') will be by default
-installed in `/usr/local/sbin/'. Translator, scheduler, and transport
-shared libraries will be installed in
-`/usr/local/lib/glusterfs/<version>/'. Sample volume specification
-files will be in `/usr/local/etc/glusterfs/'.  This document itself can
-be found in `/usr/local/share/doc/glusterfs/'. If you passed the
-`--prefix' argument to the configure script, then replace `/usr/local'
-in the preceding paths with the prefix.
-
-
-File: user-guide.info,  Node: Running GlusterFS,  Next: A Tutorial Introduction,  Prev: Building,  Up: Installation and Invocation
-
-2.4 Running GlusterFS
-=====================
-
-* Menu:
-
-* Server::
-* Client::
-
-
-File: user-guide.info,  Node: Server,  Next: Client,  Up: Running GlusterFS
-
-2.4.1 Server
-------------
-
-The GlusterFS server is necessary to export storage volumes to remote
-clients (See *note Server protocol:: for more info). This section
-documents the invocation of the GlusterFS server program and all the
-command-line options accepted by it.
-
-     Basic Options
-
-`-f, --volfile=<path>'
-     Use the volume file as the volume specification.
-
-`-s, --volfile-server=<hostname>'
-     Server to get volume file from. This option overrides -volfile
-     option.
-
-`-l, --log-file=<path>'
-     Specify the path for the log file.
-
-`-L, --log-level=<level>'
-     Set the log level for the server. Log level should be one of DEBUG,
-     WARNING, ERROR, CRITICAL, or NONE.
-
-     Advanced Options
-
-`--debug'
-     Run in debug mode. This option sets -no-daemon, -log-level to
-     DEBUG and       -log-file to console.
-
-`-N, --no-daemon'
-     Run glusterfsd as a foreground process.
-
-`-p, --pid-file=<path>'
-     Path for the PID file.
-
-`--volfile-id=<key>'
-     'key' of the volfile to be fetched from server.
-
-`--volfile-server-port=<port-number>'
-     Listening port number of volfile server.
-
-`--volfile-server-transport=[socket|ib-verbs]'
-     Transport type to get volfile from server. [default: `socket']
-
-`--xlator-options=<volume-name.option=value>'
-     Add/override a translator option for a volume with specified value.
-
-     Miscellaneous Options
-
-`-?, --help'
-     Show this help text.
-
-`--usage'
-     Display a short usage message.
-
-`-V, --version'
-     Show version information.
-
-
-File: user-guide.info,  Node: Client,  Prev: Server,  Up: Running GlusterFS
-
-2.4.2 Client
-------------
-
-The GlusterFS client process is necessary to access remote storage
-volumes and mount them locally using FUSE. This section documents the
-invocation of the client process and all its command-line arguments.
-
-       # glusterfs [options] <mountpoint>
-
-   The `mountpoint' is the directory where you want the GlusterFS
-filesystem to appear. Example:
-
-       # glusterfs -f /usr/local/etc/glusterfs-client.vol /mnt
-
-   The command-line options are detailed below.
-
-     Basic Options
-
-`-f, --volfile=<path>'
-     Use the volume file as the volume specification.
-
-`-s, --volfile-server=<hostname>'
-     Server to get volume file from. This option overrides -volfile
-     option.
-
-`-l, --log-file=<path>'
-     Specify the path for the log file.
-
-`-L, --log-level=<level>'
-     Set the log level for the server. Log level should be one of DEBUG,
-     WARNING, ERROR, CRITICAL, or NONE.
-
-     Advanced Options
-
-`--debug'
-     Run in debug mode. This option sets -no-daemon, -log-level to
-     DEBUG and       -log-file to console.
-
-`-N, --no-daemon'
-     Run `glusterfs' as a foreground process.
-
-`-p, --pid-file=<path>'
-     Path for the PID file.
-
-`--volfile-id=<key>'
-     'key' of the volfile to be fetched from server.
-
-`--volfile-server-port=<port-number>'
-     Listening port number of volfile server.
-
-`--volfile-server-transport=[socket|ib-verbs]'
-     Transport type to get volfile from server. [default: `socket']
-
-`--xlator-options=<volume-name.option=value>'
-     Add/override a translator option for a volume with specified value.
-
-`--volume-name=<volume name>'
-     Volume name in client spec to use. Defaults to the root volume.
-
-     FUSE Options
-
-`--attribute-timeout=<n>'
-     Attribute timeout for inodes in the kernel, in seconds. Defaults
-     to 1 second.
-
-`--disable-direct-io-mode'
-     Disable direct I/O mode in FUSE kernel module.
-
-`-e, --entry-timeout=<n>'
-     Entry timeout for directory entries in the kernel, in seconds.
-        Defaults to 1 second.
-
-     Missellaneous Options
-
-`-?, --help'
-     Show this help information.
-
-`-V, --version'
-     Show version information.
-
-
-File: user-guide.info,  Node: A Tutorial Introduction,  Prev: Running GlusterFS,  Up: Installation and Invocation
-
-2.5 A Tutorial Introduction
-===========================
-
-This section will show you how to quickly get GlusterFS up and running.
-We'll configure GlusterFS as a simple network filesystem, with one
-server and one client.  In this mode of usage, GlusterFS can serve as a
-replacement for NFS.
-
-   We'll make use of two machines; call them _server_ and _client_ (If
-you don't want to setup two machines, just run everything that follows
-on the same machine).  In the examples that follow, the shell prompts
-will use these names to clarify the machine on which the command is
-being run. For example, a command that should be run on the server will
-be shown with the prompt:
-
-     [root@server]#
-
-   Our goal is to make a directory on the _server_ (say, `/export')
-accessible to the _client_.
-
-   First of all, get GlusterFS installed on both the machines, as
-described in the previous sections. Make sure you have the FUSE kernel
-module loaded. You can ensure this by running:
-
-     [root@server]# modprobe fuse
-
-   Before we can run the GlusterFS client or server programs, we need
-to write two files called _volume specifications_ (equivalently refered
-to as _volfiles_).  The volfile describes the _translator tree_ on a
-node. The next chapter will explain the concepts of `translator' and
-`volume specification' in detail. For now, just assume that the volfile
-is like an NFS `/etc/export' file.
-
-   On the server, create a text file somewhere (we'll assume the path
-`/tmp/glusterfsd.vol') with the following contents.
-
-     volume colon-o
-       type storage/posix
-       option directory /export
-     end-volume
-
-     volume server
-       type protocol/server
-       subvolumes colon-o
-       option transport-type tcp
-       option auth.addr.colon-o.allow *
-     end-volume
-
-   A brief explanation of the file's contents. The first section
-defines a storage volume, named "colon-o" (the volume names are
-arbitrary), which exports the `/export' directory. The second section
-defines options for the translator which will make the storage volume
-accessible remotely. It specifies `colon-o' as a subvolume. This
-defines the _translator tree_, about which more will be said in the
-next chapter. The two options specify that the TCP protocol is to be
-used (as opposed to InfiniBand, for example), and that access to the
-storage volume is to be provided to clients with any IP address at all.
-If you wanted to restrict access to this server to only your subnet for
-example, you'd specify something like `192.168.1.*' in the second
-option line.
-
-   On the client machine, create the following text file (again, we'll
-assume the path to be `/tmp/glusterfs-client.vol'). Replace
-_server-ip-address_ with the IP address of your server machine. If you
-are doing all this on a single machine, use `127.0.0.1'.
-
-     volume client
-       type protocol/client
-       option transport-type tcp
-       option remote-host _server-ip-address_
-       option remote-subvolume colon-o
-     end-volume
-
-   Now we need to start both the server and client programs. To start
-the server:
-
-     [root@server]# glusterfsd -f /tmp/glusterfs-server.vol
-
-   To start the client:
-
-     [root@client]# glusterfs -f /tmp/glusterfs-client.vol /mnt/glusterfs
-
-   You should now be able to see the files under the server's `/export'
-directory in the `/mnt/glusterfs' directory on the client. That's it;
-GlusterFS is now working as a network file system.
-
-
-File: user-guide.info,  Node: Concepts,  Next: Translators,  Prev: Installation and Invocation,  Up: Top
-
-3 Concepts
-**********
-
-* Menu:
-
-* Filesystems in Userspace::
-* Translator::
-* Volume specification file::
-
-
-File: user-guide.info,  Node: Filesystems in Userspace,  Next: Translator,  Up: Concepts
-
-3.1 Filesystems in Userspace
-============================
-
-A filesystem is usually implemented in kernel space. Kernel space
-development is much harder than userspace development. FUSE is a kernel
-module/library that allows us to write a filesystem completely in
-userspace.
-
-   FUSE consists of a kernel module which interacts with the userspace
-implementation using a device file `/dev/fuse'. When a process makes a
-syscall on a FUSE filesystem, VFS hands the request to the FUSE module,
-which writes the request to `/dev/fuse'. The userspace implementation
-polls `/dev/fuse', and when a request arrives, processes it and writes
-the result back to `/dev/fuse'. The kernel then reads from the device
-file and returns the result to the user process.
-
-   In case of GlusterFS, the userspace program is the GlusterFS client.
-The control flow is shown in the diagram below. The GlusterFS client
-services the request by sending it to the server, which in turn hands
-it to the local POSIX filesystem.
-
-
-                   Fig 1. Control flow in GlusterFS
-
-
-File: user-guide.info,  Node: Translator,  Next: Volume specification file,  Prev: Filesystems in Userspace,  Up: Concepts
-
-3.2 Translator
-==============
-
-The _translator_ is the most important concept in GlusterFS. In fact,
-GlusterFS is nothing but a collection of translators working together,
-forming a translator _tree_.
-
-   The idea of a translator is perhaps best understood using an
-analogy. Consider the VFS in the Linux kernel. The VFS abstracts the
-various filesystem implementations (such as EXT3, ReiserFS, XFS, etc.)
-supported by the kernel. When an application calls the kernel to
-perform an operation on a file, the kernel passes the request on to the
-appropriate filesystem implementation.
-
-   For example, let's say there are two partitions on a Linux machine:
-`/', which is an EXT3 partition, and `/usr', which is a ReiserFS
-partition. Now if an application wants to open a file called, say,
-`/etc/fstab', then the kernel will internally pass the request to the
-EXT3 implementation.  If on the other hand, an application wants to
-read a file called `/usr/src/linux/CREDITS', then the kernel will call
-upon the ReiserFS implementation to do the job.
-
-   The "filesystem implementation" objects are analogous to GlusterFS
-translators. A GlusterFS translator implements all the filesystem
-operations.  Whereas in VFS there is a two-level tree (with the kernel
-at the root and all the filesystem implementation as its children), in
-GlusterFS there exists a more elaborate tree structure.
-
-   We can now define translators more precisely. A GlusterFS translator
-is a shared object (`.so') that implements every filesystem call.
-GlusterFS translators can be arranged in an arbitrary tree structure
-(subject to constraints imposed by the translators). When GlusterFS
-receives a filesystem call, it passes it on to the translator at the
-root of the translator tree. The root translator may in turn pass it on
-to any or all of its children, and so on, until the leaf nodes are
-reached. The result of a filesystem call is communicated in the reverse
-fashion, from the leaf nodes up to the root node, and then on to the
-application.
-
-   So what might a translator tree look like?
-
-
-                    Fig 2. A sample translator tree
-
-   The diagram depicts three servers and one GlusterFS client. It is
-important to note that conceptually, the translator tree spans machine
-boundaries.  Thus, the client machine in the diagram, `10.0.0.1', can
-access the aggregated storage of the filesystems on the server machines
-`10.0.0.2', `10.0.0.3', and `10.0.0.4'. The translator diagram will
-make more sense once you've read the next chapter and understood the
-functions of the various translators.
-
-
-File: user-guide.info,  Node: Volume specification file,  Prev: Translator,  Up: Concepts
-
-3.3 Volume specification file
-=============================
-
-The volume specification file describes the translator tree for both the
-server and client programs.
-
-   A volume specification file is a sequence of volume definitions.
-The syntax of a volume definition is explained below:
-
-     *volume* _volume-name_
-       *type* _translator-name_
-       *option* _option-name_ _option-value_
-       ...
-       *subvolumes* _subvolume1_ _subvolume2_ ...
-     *end-volume*
-
-   ...
-
-_volume-name_
-     An identifier for the volume. This is just a human-readable name,
-     and can contain any alphanumeric character. For instance,
-     "storage-1", "colon-o", or "forty-two".
-
-_translator-name_
-     Name of one of the available translators. Example:
-     `protocol/client', `cluster/unify'.
-
-_option-name_
-     Name of a valid option for the translator.
-
-_option-value_
-     Value for the option. Everything following the "option" keyword to
-     the end of the line is considered the value; it is up to the
-     translator to parse it.
-
-_subvolume1_, _subvolume2_, ...
-     Volume names of sub-volumes. The sub-volumes must already have
-     been defined earlier in the file.
-
-   There are a few rules you must follow when writing a volume
-specification file:
-
-   * Everything following a ``#'' is considered a comment and is
-     ignored. Blank lines are also ignored.
-
-   * All names and keywords are case-sensitive.
-
-   * The order of options inside a volume definition does not matter.
-
-   * An option value may not span multiple lines.
-
-   * If an option is not specified, it will assume its default value.
-
-   * A sub-volume must have already been defined before it can be
-     referenced. This means you have to write the specification file
-     "bottom-up", starting from the leaf nodes of the translator tree
-     and moving up to the root.
-
-   A simple example volume specification file is shown below:
-
-     # This is a comment line
-     volume client
-      type protocol/client
-      option transport-type tcp
-      option remote-host localhost      # Also a comment
-      option remote-subvolume brick
-     # The subvolumes line may be absent
-     end-volume
-
-     volume iot
-      type performance/io-threads
-      option thread-count 4
-      subvolumes client
-     end-volume
-
-     volume wb
-      type performance/write-behind
-      subvolumes iot
-     end-volume
-
-
-File: user-guide.info,  Node: Translators,  Next: Usage Scenarios,  Prev: Concepts,  Up: Top
-
-4 Translators
-*************
-
-* Menu:
-
-* Storage Translators::
-* Client and Server Translators::
-* Clustering Translators::
-* Performance Translators::
-* Features Translators::
-* Miscellaneous Translators::
-
-   This chapter documents all the available GlusterFS translators in
-detail.  Each translator section will show its name (for example,
-`cluster/unify'), briefly describe its purpose and workings, and list
-every option accepted by that translator and their meaning.
-
-
-File: user-guide.info,  Node: Storage Translators,  Next: Client and Server Translators,  Up: Translators
-
-4.1 Storage Translators
-=======================
-
-The storage translators form the "backend" for GlusterFS. Currently,
-the only available storage translator is the POSIX translator, which
-stores files on a normal POSIX filesystem. A pleasant consequence of
-this is that your data will still be accessible if GlusterFS crashes or
-cannot be started.
-
-   Other storage backends are planned for the future. One of the
-possibilities is an Amazon S3 translator. Amazon S3 is an unlimited
-online storage service accessible through a web services API. The S3
-translator will allow you to access the storage as a normal POSIX
-filesystem.  (1)
-
-* Menu:
-
-* POSIX::
-* BDB::
-
-   ---------- Footnotes ----------
-
-   (1) Some more discussion about this can be found at:
-
-http://developer.amazonwebservices.com/connect/message.jspa?messageID=52873
-
-
-File: user-guide.info,  Node: POSIX,  Next: BDB,  Up: Storage Translators
-
-4.1.1 POSIX
------------
-
-     type storage/posix
-
-   The `posix' translator uses a normal POSIX filesystem as its
-"backend" to actually store files and directories. This can be any
-filesystem that supports extended attributes (EXT3, ReiserFS, XFS,
-...). Extended attributes are used by some translators to store
-metadata, for example, by the replicate and stripe translators. See
-*note Replicate:: and *note Stripe::, respectively for details.
-
-`directory <path>'
-     The directory on the local filesystem which is to be used for
-     storage.
-
-
-File: user-guide.info,  Node: BDB,  Prev: POSIX,  Up: Storage Translators
-
-4.1.2 BDB
----------
-
-     type storage/bdb
-
-   The `BDB' translator uses a Berkeley DB database as its "backend" to
-actually store files as key-value pair in the database and directories
-as regular POSIX directories. Note that BDB does not provide extended
-attribute support for regular files. Do not use BDB as storage
-translator while using any translator that demands extended attributes
-on "backend".
-
-`directory <path>'
-     The directory on the local filesystem which is to be used for
-     storage.
-
-`mode [cache|persistent] (cache)'
-     When BDB is run in `cache' mode, recovery of back-end is not
-     completely guaranteed. `persistent' guarantees that BDB can
-     recover back-end from Berkeley DB even if GlusterFS crashes.
-
-`errfile <path>'
-     The path of the file to be used as `errfile' for Berkeley DB to
-     report detailed error messages, if any. Note that all the contents
-     of this file will be written by Berkeley DB, not GlusterFS.
-
-`logdir <path>'
-
-
-File: user-guide.info,  Node: Client and Server Translators,  Next: Clustering Translators,  Prev: Storage Translators,  Up: Translators
-
-4.2 Client and Server Translators
-=================================
-
-The client and server translator enable GlusterFS to export a
-translator tree over the network or access a remote GlusterFS server.
-These two translators implement GlusterFS's network protocol.
-
-* Menu:
-
-* Transport modules::
-* Client protocol::
-* Server protocol::
-
-
-File: user-guide.info,  Node: Transport modules,  Next: Client protocol,  Up: Client and Server Translators
-
-4.2.1 Transport modules
------------------------
-
-The client and server translators are capable of using any of the
-pluggable transport modules. Currently available transport modules are
-`tcp', which uses a TCP connection between client and server to
-communicate; `ib-sdp', which uses a TCP connection over InfiniBand, and
-`ibverbs', which uses high-speed InfiniBand connections.
-
-   Each transport module comes in two different versions, one to be
-used on the server side and the other on the client side.
-
-4.2.1.1 TCP
-...........
-
-The TCP transport module uses a TCP/IP connection between the server
-and the client.
-
-       option transport-type tcp
-
-   The TCP client module accepts the following options:
-
-`non-blocking-connect [no|off|on|yes] (on)'
-     Whether to make the connection attempt asynchronous.
-
-`remote-port <n> (24007)'
-     Server port to connect to.  
-
-`remote-host <hostname> *'
-     Hostname or IP address of the server. If the host name resolves to
-     multiple IP addresses, all of them will be tried in a round-robin
-     fashion. This feature can be used to implement fail-over.
-
-   The TCP server module accepts the following options:
-
-`bind-address <address> (0.0.0.0)'
-     The local interface on which the server should listen to requests.
-     Default is to listen on all interfaces.
-
-`listen-port <n> (24007)'
-     The local port to listen on.
-
-4.2.1.2 IB-SDP
-..............
-
-       option transport-type ib-sdp
-
-   kernel implements socket interface for ib hardware. SDP is over
-ib-verbs.  This module accepts the same options as `tcp'
-
-4.2.1.3 ibverbs
-...............
-
-       option transport-type tcp
-
-   InfiniBand is a scalable switched fabric interconnect mechanism
-primarily used in high-performance computing. InfiniBand can deliver
-data throughput of the order of 10 Gbit/s, with latencies of 4-5 ms.
-
-   The `ib-verbs' transport accesses the InfiniBand hardware through
-the "verbs" API, which is the lowest level of software access possible
-and which gives the highest performance. On InfiniBand hardware, it is
-always best to use `ib-verbs'. Use `ib-sdp' only if you cannot get
-`ib-verbs' working for some reason.
-
-   The `ib-verbs' client module accepts the following options:
-
-`non-blocking-connect [no|off|on|yes] (on)'
-     Whether to make the connection attempt asynchronous.
-
-`remote-port <n> (24007)'
-     Server port to connect to.  
-
-`remote-host <hostname> *'
-     Hostname or IP address of the server. If the host name resolves to
-     multiple IP addresses, all of them will be tried in a round-robin
-     fashion. This feature can be used to implement fail-over.
-
-   The `ib-verbs' server module accepts the following options:
-
-`bind-address <address> (0.0.0.0)'
-     The local interface on which the server should listen to requests.
-     Default is to listen on all interfaces.
-
-`listen-port <n> (24007)'
-     The local port to listen on.
-
-   The following options are common to both the client and server
-modules:
-
-   If you are familiar with InfiniBand jargon, the mode is used by
-GlusterFS is "reliable connection-oriented channel transfer".
-
-`ib-verbs-work-request-send-count <n> (64)'
-     Length of the send queue in datagrams. [Reason to
-     increase/decrease?]
-
-`ib-verbs-work-request-recv-count <n> (64)'
-     Length of the receive queue in datagrams. [Reason to
-     increase/decrease?]
-
-`ib-verbs-work-request-send-size <size> (128KB)'
-     Size of each datagram that is sent. [Reason to increase/decrease?]
-
-`ib-verbs-work-request-recv-size <size> (128KB)'
-     Size of each datagram that is received. [Reason to
-     increase/decrease?]
-
-`ib-verbs-port <n> (1)'
-     Port number for ib-verbs.
-
-`ib-verbs-mtu [256|512|1024|2048|4096] (2048)'
-     The Maximum Transmission Unit [Reason to increase/decrease?]
-
-`ib-verbs-device-name <device-name> (first device in the list)'
-     InfiniBand device to be used.
-
-   For maximum performance, you should ensure that the send/receive
-counts on both the client and server are the same.
-
-   ib-verbs is preferred over ib-sdp.
-
-
-File: user-guide.info,  Node: Client protocol,  Next: Server protocol,  Prev: Transport modules,  Up: Client and Server Translators
-
-4.2.2 Client
-------------
-
-     type procotol/client
-
-   The client translator enables the GlusterFS client to access a
-remote server's translator tree.
-
-`transport-type [tcp,ib-sdp,ib-verbs] (tcp)'
-     The transport type to use. You should use the client versions of
-     all the transport modules (`tcp', `ib-sdp', `ib-verbs').
-
-`remote-subvolume <volume_name> *'
-     The name of the volume on the remote host to attach to. Note that
-     this is _not_ the name of the `protocol/server' volume on the
-     server. It should be any volume under the server.
-
-`transport-timeout <n> (120- seconds)'
-     Inactivity timeout. If a reply is expected and no activity takes
-     place on the connection within this time, the transport connection
-     will be broken, and a new connection will be attempted.
-
-
-File: user-guide.info,  Node: Server protocol,  Prev: Client protocol,  Up: Client and Server Translators
-
-4.2.3 Server
-------------
-
-     type protocol/server
-
-   The server translator exports a translator tree and makes it
-accessible to remote GlusterFS clients.
-
-`client-volume-filename <path> (<CONFDIR>/glusterfs-client.vol)'
-     The volume specification file to use for the client. This is the
-     file the client will receive when it is invoked with the
-     `--server' option (*note Client::).
-
-`transport-type [tcp,ib-verbs,ib-sdp] (tcp)'
-     The transport to use. You should use the server versions of all
-     the transport modules (`tcp', `ib-sdp', `ib-verbs').
-
-`auth.addr.<volume name>.allow <IP address wildcard pattern>'
-     IP addresses of the clients that are allowed to attach to the
-     specified volume.  This can be a wildcard. For example, a wildcard
-     of the form `192.168.*.*' allows any host in the `192.168.x.x'
-     subnet to connect to the server.
-
-
-
-File: user-guide.info,  Node: Clustering Translators,  Next: Performance Translators,  Prev: Client and Server Translators,  Up: Translators
-
-4.3 Clustering Translators
-==========================
-
-The clustering translators are the most important GlusterFS
-translators, since it is these that make GlusterFS a cluster
-filesystem. These translators together enable GlusterFS to access an
-arbitrarily large amount of storage, and provide RAID-like redundancy
-and distribution over the entire cluster.
-
-   There are three clustering translators: *unify*, *replicate*, and
-*stripe*.  The unify translator aggregates storage from many server
-nodes. The replicate translator provides file replication. The stripe
-translator allows a file to be spread across many server nodes. The
-following sections look at each of these translators in detail.
-
-* Menu:
-
-* Unify::
-* Replicate::
-* Stripe::
-
-
-File: user-guide.info,  Node: Unify,  Next: Replicate,  Up: Clustering Translators
-
-4.3.1 Unify
------------
-
-     type cluster/unify
-
-   The unify translator presents a `unified' view of all its
-sub-volumes. That is, it makes the union of all its sub-volumes appear
-as a single volume. It is the unify translator that gives GlusterFS the
-ability to access an arbitrarily large amount of storage.
-
-   For unify to work correctly, certain invariants need to be
-maintained across the entire network. These are:
-
-   * The directory structure of all the sub-volumes must be identical.
-
-   * A particular file can exist on only one of the sub-volumes.
-     Phrasing it in another way, a pathname such as
-     `/home/calvin/homework.txt') is unique across the entire cluster.
-
-
-
-Looking at the second requirement, you might wonder how one can
-accomplish storing redundant copies of a file, if no file can exist
-multiple times.  To answer, we must remember that these invariants are
-from _unify's perspective_.  A translator such as replicate at a lower
-level in the translator tree than unify may subvert this picture.
-
-   The first invariant might seem quite tedious to ensure. We shall see
-later that this is not so, since unify's _self-heal_ mechanism takes
-care of maintaining it.
-
-   The second invariant implies that unify needs some way to decide
-which file goes where.  Unify makes use of _scheduler_ modules for this
-purpose.
-
-   When a file needs to be created, unify's scheduler decides upon the
-sub-volume to be used to store the file. There are many schedulers
-available, each using a different algorithm and suitable for different
-purposes.
-
-   The various schedulers are described in detail in the sections that
-follow.
-
-4.3.1.1 ALU
-...........
-
-       option scheduler alu
-
-   ALU stands for "Adaptive Least Usage". It is the most advanced
-scheduler available in GlusterFS. It balances the load across volumes
-taking several factors in account. It adapts itself to changing I/O
-patterns according to its configuration. When properly configured, it
-can eliminate the need for regular tuning of the filesystem to keep
-volume load nicely balanced.
-
-   The ALU scheduler is composed of multiple least-usage
-sub-schedulers. Each sub-scheduler keeps track of a certain type of
-load, for each of the sub-volumes, getting statistics from the
-sub-volumes themselves. The sub-schedulers are these:
-
-   * disk-usage: The used and free disk space on the volume.
-
-   * read-usage: The amount of reading done from this volume.
-
-   * write-usage: The amount of writing done to this volume.
-
-   * open-files-usage: The number of files currently open from this
-     volume.
-
-   * disk-speed-usage: The speed at which the disks are spinning. This
-     is a constant value and therefore not very useful.
-
-   The ALU scheduler needs to know which of these sub-schedulers to use,
-and in which order to evaluate them. This is done through the `option
-alu.order' configuration directive.
-
-   Each sub-scheduler needs to know two things: when to kick in (the
-entry-threshold), and how long to stay in control (the exit-threshold).
-For example: when unifying three disks of 100GB, keeping an exact
-balance of disk-usage is not necesary. Instead, there could be a 1GB
-margin, which can be used to nicely balance other factors, such as
-read-usage. The disk-usage scheduler can be told to kick in only when a
-certain threshold of discrepancy is passed, such as 1GB. When it
-assumes control under this condition, it will write all subsequent data
-to the least-used volume. If it is doing so, it is unwise to stop right
-after the values are below the entry-threshold again, since that would
-make it very likely that the situation will occur again very soon. Such
-a situation would cause the ALU to spend most of its time disk-usage
-scheduling, which is unfair to the other sub-schedulers. The
-exit-threshold therefore defines the amount of data that needs to be
-written to the least-used disk, before control is relinquished again.
-
-   In addition to the sub-schedulers, the ALU scheduler also has
-"limits" options. These can stop the creation of new files on a volume
-once values drop below a certain threshold. For example, setting
-`option alu.limits.min-free-disk 5GB' will stop the scheduling of files
-to volumes that have less than 5GB of free disk space, leaving the
-files on that disk some room to grow.
-
-   The actual values you assign to the thresholds for sub-schedulers and
-limits depend on your situation. If you have fast-growing files, you'll
-want to stop file-creation on a disk much earlier than when hardly any
-of your files are growing. If you care less about disk-usage balance
-than about read-usage balance, you'll want a bigger disk-usage
-scheduler entry-threshold and a smaller read-usage scheduler
-entry-threshold.
-
-   For thresholds defining a size, values specifying "KB", "MB" and "GB"
-are allowed. For example: `option alu.limits.min-free-disk 5GB'.
-
-`alu.order <order> * ("disk-usage:write-usage:read-usage:open-files-usage:disk-speed")'
-
-`alu.disk-usage.entry-threshold <size> (1GB)'
-
-`alu.disk-usage.exit-threshold <size> (512MB)'
-
-`alu.write-usage.entry-threshold <%> (25)'
-
-`alu.write-usage.exit-threshold <%> (5)'
-
-`alu.read-usage.entry-threshold <%> (25)'
-
-`alu.read-usage.exit-threshold <%> (5)'
-
-`alu.open-files-usage.entry-threshold <n> (1000)'
-
-`alu.open-files-usage.exit-threshold <n> (100)'
-
-`alu.limits.min-free-disk <%>'
-
-`alu.limits.max-open-files <n>'
-
-4.3.1.2 Round Robin (RR)
-........................
-
-       option scheduler rr
-
-   Round-Robin (RR) scheduler creates files in a round-robin fashion.
-Each client will have its own round-robin loop. When your files are
-mostly similar in size and I/O access pattern, this scheduler is a good
-choice. RR scheduler checks for free disk space on the server before
-scheduling, so you can know when to add another server node. The
-default value of min-free-disk is 5% and is checked on file creation
-calls, with atleast 10 seconds (by default) elapsing between two checks.
-
-   Options:
-`rr.limits.min-free-disk <%> (5)'
-     Minimum free disk space a node must have for RR to schedule a file
-     to it.
-
-`rr.refresh-interval <t> (10 seconds)'
-     Time between two successive free disk space checks.
-
-4.3.1.3 Random
-..............
-
-       option scheduler random
-
-   The random scheduler schedules file creation randomly among its
-child nodes.  Like the round-robin scheduler, it also checks for a
-minimum amount of free disk space before scheduling a file to a node.
-
-`random.limits.min-free-disk <%> (5)'
-     Minimum free disk space a node must have for random to schedule a
-     file to it.
-
-`random.refresh-interval <t> (10 seconds)'
-     Time between two successive free disk space checks.
-
-4.3.1.4 NUFA
-............
-
-       option scheduler nufa
-
-   It is common in many GlusterFS computing environments for all
-deployed machines to act as both servers and clients. For example, a
-research lab may have 40 workstations each with its own storage. All of
-these workstations might act as servers exporting a volume as well as
-clients accessing the entire cluster's storage.  In such a situation,
-it makes sense to store locally created files on the local workstation
-itself (assuming files are accessed most by the workstation that
-created them). The Non-Uniform File Allocation (NUFA) scheduler
-accomplishes that.
-
-   NUFA gives the local system first priority for file creation over
-other nodes. If the local volume does not have more free disk space
-than a specified amount (5% by default) then NUFA schedules files among
-the other child volumes in a round-robin fashion.
-
-   NUFA is named after the similar strategy used for memory access,
-NUMA(1).
-
-`nufa.limits.min-free-disk <%> (5)'
-     Minimum disk space that must be free (local or remote) for NUFA to
-     schedule a file to it.
-
-`nufa.refresh-interval <t> (10 seconds)'
-     Time between two successive free disk space checks.
-
-`nufa.local-volume-name <volume>'
-     The name of the volume corresponding to the local system. This
-     volume must be one of the children of the unify volume. This
-     option is mandatory.
-
-4.3.1.5 Namespace
-.................
-
-Namespace volume needed because:  - persistent inode numbers.   - file
-exists even when node is down.
-
-   namespace files are simply touched. on every lookup it is checked.
-
-`namespace <volume> *'
-     Name of the namespace volume (which should be one of the unify
-     volume's children).
-
-`self-heal [on|off] (on)'
-     Enable/disable self-heal. Unless you know what you are doing, do
-     not disable self-heal.
-
-4.3.1.6 Self Heal
-.................
-
-* When a 'lookup()/stat()' call is made on directory for the first
-time, a self-heal call is made, which checks for the consistancy of its
-child nodes. If an entry is present in storage node, but not in
-namespace, that entry is created in namespace, and vica-versa. There is
-an writedir() API introduced which is used for the same. It also checks
-for permissions, and uid/gid consistencies.
-
-   * This check is also done when an server goes down and comes up.
-
-   * If one starts with an empty namespace export, but has data in
-storage nodes, a 'find .>/dev/null' or 'ls -lR >/dev/null' should help
-to build namespace in one shot. Even otherwise, namespace is built on
-demand when a file is looked up for the first time.
-
-   NOTE: There are some issues (Kernel 'Oops' msgs) seen with
-fuse-2.6.3, when someone deletes namespace in backend, when glusterfs is
-running. But with fuse-2.6.5, this issue is not there.
-
-   ---------- Footnotes ----------
-
-   (1) Non-Uniform Memory Access:
-<http://en.wikipedia.org/wiki/Non-Uniform_Memory_Access>
-
-
-File: user-guide.info,  Node: Replicate,  Next: Stripe,  Prev: Unify,  Up: Clustering Translators
-
-4.3.2 Replicate (formerly AFR)
-------------------------------
-
-     type cluster/replicate
-
-   Replicate provides RAID-1 like functionality for GlusterFS.
-Replicate replicates files and directories across the subvolumes. Hence
-if Replicate has four subvolumes, there will be four copies of all
-files and directories. Replicate provides high-availability, i.e., in
-case one of the subvolumes go down (e. g. server crash, network
-disconnection) Replicate will still service the requests using the
-redundant copies.
-
-   Replicate also provides self-heal functionality, i.e., in case the
-crashed servers come up, the outdated files and directories will be
-updated with the latest versions. Replicate uses extended attributes of
-the backend file system to track the versioning of files and
-directories and provide the self-heal feature.
-
-     volume replicate-example
-      type cluster/replicate
-      subvolumes brick1 brick2 brick3
-     end-volume
-
-   This sample configuration will replicate all directories and files on
-brick1, brick2 and brick3.
-
-   All the read operations happen from the first alive child. If all the
-three sub-volumes are up, reads will be done from brick1; if brick1 is
-down read will be done from brick2. In case read() was being done on
-brick1 and it goes down, replicate transparently falls back to brick2.
-
-   The next release of GlusterFS will add the following features:
-   * Ability to specify the sub-volume from which read operations are
-     to be done (this will help users who have one of the sub-volumes
-     as a local storage volume).
-
-   * Allow scheduling of read operations amongst the sub-volumes in a
-     round-robin fashion.
-
-   The order of the subvolumes list should be same across all the
-'replicate's as they will be used for locking purposes.
-
-4.3.2.1 Self Heal
-.................
-
-Replicate has self-heal feature, which updates the outdated file and
-directory copies by the most recent versions. For example consider the
-following config:
-
-     volume replicate-example
-      type cluster/replicate
-      subvolumes brick1 brick2
-     end-volume
-
-4.3.2.2 File self-heal
-......................
-
-Now if we create a file foo.txt on replicate-example, the file will be
-created on brick1 and brick2. The file will have two extended
-attributes associated with it in the backend filesystem. One is
-trusted.afr.createtime and the other is trusted.afr.version. The
-trusted.afr.createtime xattr has the create time (in terms of seconds
-since epoch) and trusted.afr.version is a number that is incremented
-each time a file is modified. This increment happens during close
-(incase any write was done before close).
-
-   If brick1 goes down, we edit foo.txt the version gets incremented.
-Now the brick1 comes back up, when we open() on foo.txt replicate will
-check if their versions are same. If they are not same, the outdated
-copy is replaced by the latest copy and its version is updated. After
-the sync the open() proceeds in the usual manner and the application
-calling open() can continue on its access to the file.
-
-   If brick1 goes down, we delete foo.txt and create a file with the
-same name again i.e foo.txt. Now brick1 comes back up, clearly there is
-a chance that the version on brick1 being more than the version on
-brick2, this is where createtime extended attribute helps in deciding
-which the outdated copy is. Hence we need to consider both createtime
-and version to decide on the latest copy.
-
-   The version attribute is incremented during the close() call. Version
-will not be incremented in case there was no write() done. In case the
-fd that the close() gets was got by create() call, we also create the
-createtime extended attribute.
-
-4.3.2.3 Directory self-heal
-...........................
-
-Suppose brick1 goes down, we delete foo.txt, brick1 comes back up, now
-we should not create foo.txt on brick2 but we should delete foo.txt on
-brick1. We handle this situation by having the createtime and version
-attribute on the directory similar to the file. when lookup() is done
-on the directory, we compare the createtime/version attributes of the
-copies and see which files needs to be deleted and delete those files
-and update the extended attributes of the outdated directory copy.
-Each time a directory is modified (a file or a subdirectory is created
-or deleted inside the directory) and one of the subvols is down, we
-increment the directory's version.
-
-   lookup() is a call initiated by the kernel on a file or directory
-just before any access to that file or directory. In glusterfs, by
-default, lookup() will not be called in case it was called in the past
-one second on that particular file or directory.
-
-   The extended attributes can be seen in the backend filesystem using
-the `getfattr' command. (`getfattr -n trusted.afr.version <file>')
-
-`debug [on|off]  (off)'
-
-`self-heal [on|off] (on)'
-
-`replicate <pattern> (*:1)'
-
-`lock-node <child_volume> (first child is used by default)'
-
-
-File: user-guide.info,  Node: Stripe,  Prev: Replicate,  Up: Clustering Translators
-
-4.3.3 Stripe
-------------
-
-     type cluster/stripe
-
-   The stripe translator distributes the contents of a file over its
-sub-volumes.  It does this by creating a file equal in size to the
-total size of the file on each of its sub-volumes. It then writes only
-a part of the file to each sub-volume, leaving the rest of it empty.
-These empty regions are called `holes' in Unix terminology. The holes
-do not consume any disk space.
-
-   The diagram below makes this clear.
-
-
-
-You can configure stripe so that only filenames matching a pattern are
-striped. You can also configure the size of the data to be stored on
-each sub-volume.
-
-`block-size <pattern>:<size>  (*:0 no striping)'
-     Distribute files matching `<pattern>' over the sub-volumes,
-     storing at least `<size>' on each sub-volume. For example,
-
-            option block-size *.mpg:1M
-
-     distributes all files ending in `.mpg', storing at least 1 MB on
-     each sub-volume.
-
-     Any number of `block-size' option lines may be present, specifying
-     different sizes for different file name patterns.
-
-
-File: user-guide.info,  Node: Performance Translators,  Next: Features Translators,  Prev: Clustering Translators,  Up: Translators
-
-4.4 Performance Translators
-===========================
-
-* Menu:
-
-* Read Ahead::
-* Write Behind::
-* IO Threads::
-* IO Cache::
-* Booster::
-
-
-File: user-guide.info,  Node: Read Ahead,  Next: Write Behind,  Up: Performance Translators
-
-4.4.1 Read Ahead
-----------------
-
-     type performance/read-ahead
-
-   The read-ahead translator pre-fetches data in advance on every read.
-This benefits applications that mostly process files in sequential
-order, since the next block of data will already be available by the
-time the application is done with the current one.
-
-   Additionally, the read-ahead translator also behaves as a
-read-aggregator.  Many small read operations are combined and issued as
-fewer, larger read requests to the server.
-
-   Read-ahead deals in "pages" as the unit of data fetched. The page
-size is configurable, as is the "page count", which is the number of
-pages that are pre-fetched.
-
-   Read-ahead is best used with InfiniBand (using the ib-verbs
-transport).  On FastEthernet and Gigabit Ethernet networks, GlusterFS
-can achieve the link-maximum throughput even without read-ahead, making
-it quite superflous.
-
-   Note that read-ahead only happens if the reads are perfectly
-sequential. If your application accesses data in a random fashion,
-using read-ahead might actually lead to a performance loss, since
-read-ahead will pointlessly fetch pages which won't be used by the
-application.
-
-   Options:
-`page-size <n> (256KB)'
-     The unit of data that is pre-fetched.
-
-`page-count <n> (2)'
-     The number of pages that are pre-fetched.
-
-`force-atime-update [on|off|yes|no] (off|no)'
-     Whether to force an access time (atime) update on the file on
-     every read. Without this, the atime will be slightly imprecise, as
-     it will reflect the time when the read-ahead translator read the
-     data, not when the application actually read it.
-
-
-File: user-guide.info,  Node: Write Behind,  Next: IO Threads,  Prev: Read Ahead,  Up: Performance Translators
-
-4.4.2 Write Behind
-------------------
-
-     type performance/write-behind
-
-   The write-behind translator improves the latency of a write
-operation.  It does this by relegating the write operation to the
-background and returning to the application even as the write is in
-progress. Using the write-behind translator, successive write requests
-can be pipelined.  This mode of write-behind operation is best used on
-the client side, to enable decreased write latency for the application.
-
-   The write-behind translator can also aggregate write requests. If the
-`aggregate-size' option is specified, then successive writes upto that
-size are accumulated and written in a single operation. This mode of
-operation is best used on the server side, as this will decrease the
-disk's head movement when multiple files are being written to in
-parallel.
-
-   The `aggregate-size' option has a default value of 128KB. Although
-this works well for most users, you should always experiment with
-different values to determine the one that will deliver maximum
-performance. This is because the performance of write-behind depends on
-your interconnect, size of RAM, and the work load.
-
-`aggregate-size <n> (128KB)'
-     Amount of data to accumulate before doing a write
-
-`flush-behind [on|yes|off|no] (off|no)'
-
-
-File: user-guide.info,  Node: IO Threads,  Next: IO Cache,  Prev: Write Behind,  Up: Performance Translators
-
-4.4.3 IO Threads
-----------------
-
-     type performance/io-threads
-
-   The IO threads translator is intended to increase the responsiveness
-of the server to metadata operations by doing file I/O (read, write) in
-a background thread.  Since the GlusterFS server is single-threaded,
-using the IO threads translator can significantly improve performance.
-This translator is best used on the server side, loaded just below the
-server protocol translator.
-
-   IO threads operates by handing out read and write requests to a
-separate thread.  The total number of threads in existence at a time is
-constant, and configurable.
-
-`thread-count <n> (1)'
-     Number of threads to use.
-
-
-File: user-guide.info,  Node: IO Cache,  Next: Booster,  Prev: IO Threads,  Up: Performance Translators
-
-4.4.4 IO Cache
---------------
-
-     type performance/io-cache
-
-   The IO cache translator caches data that has been read. This is
-useful if many applications read the same data multiple times, and if
-reads are much more frequent than writes (for example, IO caching may be
-useful in a web hosting environment, where most clients will simply
-read some files and only a few will write to them).
-
-   The IO cache translator reads data from its child in `page-size'
-chunks.  It caches data upto `cache-size' bytes. The cache is
-maintained as a prioritized least-recently-used (LRU) list, with
-priorities determined by user-specified patterns to match filenames.
-
-   When the IO cache translator detects a write operation, the cache
-for that file is flushed.
-
-   The IO cache translator periodically verifies the consistency of
-cached data, using the modification times on the files. The
-verification timeout is configurable.
-
-`page-size <n> (128KB)'
-     Size of a page.
-
-`cache-size (n) (32MB)'
-     Total amount of data to be cached.
-
-`force-revalidate-timeout <n> (1)'
-     Timeout to force a cache consistency verification, in seconds.
-
-`priority <pattern> (*:0)'
-     Filename patterns listed in order of priority.
-
-
-File: user-guide.info,  Node: Booster,  Prev: IO Cache,  Up: Performance Translators
-
-4.4.5 Booster
--------------
-
-       type performance/booster
-
-   The booster translator gives applications a faster path to
-communicate read and write requests to GlusterFS. Normally, all
-requests to GlusterFS from applications go through FUSE, as indicated
-in *note Filesystems in Userspace::.  Using the booster translator in
-conjunction with the GlusterFS booster shared library, an application
-can bypass the FUSE path and send read/write requests directly to the
-GlusterFS client process.
-
-   The booster mechanism consists of two parts: the booster translator,
-and the booster shared library. The booster translator is meant to be
-loaded on the client side, usually at the root of the translator tree.
-The booster shared library should be `LD_PRELOAD'ed with the
-application.
-
-   The booster translator when loaded opens a Unix domain socket and
-listens for read/write requests on it. The booster shared library
-intercepts read and write system calls and sends the requests to the
-GlusterFS process directly using the Unix domain socket, bypassing FUSE.
-This leads to superior performance.
-
-   Once you've loaded the booster translator in your volume
-specification file, you can start your application as:
-
-       $ LD_PRELOAD=/usr/local/bin/glusterfs-booster.so your_app
-
-   The booster translator accepts no options.
-
-
-File: user-guide.info,  Node: Features Translators,  Next: Miscellaneous Translators,  Prev: Performance Translators,  Up: Translators
-
-4.5 Features Translators
-========================
-
-* Menu:
-
-* POSIX Locks::
-* Fixed ID::
-
-
-File: user-guide.info,  Node: POSIX Locks,  Next: Fixed ID,  Up: Features Translators
-
-4.5.1 POSIX Locks
------------------
-
-     type features/posix-locks
-
-   This translator provides storage independent POSIX record locking
-support (`fcntl' locking). Typically you'll want to load this on the
-server side, just above the POSIX storage translator. Using this
-translator you can get both advisory locking and mandatory locking
-support.  It also handles `flock()' locks properly.
-
-   Caveat: Consider a file that does not have its mandatory locking bits
-(+setgid, -group execution) turned on. Assume that this file is now
-opened by a process on a client that has the write-behind xlator
-loaded. The write-behind xlator does not cache anything for files which
-have mandatory locking enabled, to avoid incoherence. Let's say that
-mandatory locking is now enabled on this file through another client.
-The former client will not know about this change, and write-behind may
-erroneously report a write as being successful when in fact it would
-fail due to the region it is writing to being locked.
-
-   There seems to be no easy way to fix this. To work around this
-problem, it is recommended that you never enable the mandatory bits on
-a file while it is open.
-
-`mandatory [on|off] (on)'
-     Turns mandatory locking on.
-
-
-File: user-guide.info,  Node: Fixed ID,  Prev: POSIX Locks,  Up: Features Translators
-
-4.5.2 Fixed ID
---------------
-
-     type features/fixed-id
-
-   The fixed ID translator makes all filesystem requests from the client
-to appear to be coming from a fixed, specified UID/GID, regardless of
-which user actually initiated the request.
-
-`fixed-uid <n> [if not set, not used]'
-     The UID to send to the server
-
-`fixed-gid <n> [if not set, not used]'
-     The GID to send to the server
-
-
-File: user-guide.info,  Node: Miscellaneous Translators,  Prev: Features Translators,  Up: Translators
-
-4.6 Miscellaneous Translators
-=============================
-
-* Menu:
-
-* ROT-13::
-* Trace::
-
-
-File: user-guide.info,  Node: ROT-13,  Next: Trace,  Up: Miscellaneous Translators
-
-4.6.1 ROT-13
-------------
-
-     type encryption/rot-13
-
-   ROT-13 is a toy translator that can "encrypt" and "decrypt" file
-contents using the ROT-13 algorithm. ROT-13 is a trivial algorithm that
-rotates each alphabet by thirteen places. Thus, 'A' becomes 'N', 'B'
-becomes 'O', and 'Z' becomes 'M'.
-
-   It goes without saying that you shouldn't use this translator if you
-need _real_ encryption (a future release of GlusterFS will have real
-encryption translators).
-
-`encrypt-write [on|off] (on)'
-     Whether to encrypt on write
-
-`decrypt-read [on|off] (on)'
-     Whether to decrypt on read
-
-
-File: user-guide.info,  Node: Trace,  Prev: ROT-13,  Up: Miscellaneous Translators
-
-4.6.2 Trace
------------
-
-     type debug/trace
-
-   The trace translator is intended for debugging purposes. When
-loaded, it logs all the system calls received by the server or client
-(wherever trace is loaded), their arguments, and the results. You must
-use a GlusterFS log level of DEBUG (See *note Running GlusterFS::) for
-trace to work.
-
-   Sample trace output (lines have been wrapped for readability):
-     2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68
-     (*this=0x8059e40, loc=0x8091984 {path=/iozone3_283, inode=0x8091f00},
-      fd=0x8091d50)
-
-     2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace:
-     (*this=0x8059e40, op_ret=4, op_errno=1, fd=0x8091d50)
-
-     2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69
-     (*this=0x8059e40, size=4096, offset=0 fd=0x8091d50)
-
-     2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace:
-     (*this=0x8059e40, op_ret=0, op_errno=0, count=4)
-
-     2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71
-     (*this=0x8059e40, *fd=0x8091d50)
-
-     2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace:
-     (*this=0x8059e40, op_ret=0, op_errno=1)
-
-
-File: user-guide.info,  Node: Usage Scenarios,  Next: Troubleshooting,  Prev: Translators,  Up: Top
-
-5 Usage Scenarios
-*****************
-
-5.1 Advanced Striping
-=====================
-
-This section is based on the Advanced Striping tutorial written by
-Anand Avati on the GlusterFS wiki (1).
-
-5.1.1 Mixed Storage Requirements
---------------------------------
-
-There are two ways of scheduling the I/O. One at file level (using
-unify translator) and other at block level (using stripe translator).
-Striped I/O is good for files that are potentially large and require
-high parallel throughput (for example, a single file of 400GB being
-accessed by 100s and 1000s of systems simultaneously and randomly). For
-most of the cases, file level scheduling works best.
-
-   In the real world, it is desirable to mix file level and block level
-scheduling on a single storage volume. Alternatively users can choose
-to have two separate volumes and hence two mount points, but the
-applications may demand a single storage system to host both.
-
-   This document explains how to mix file level scheduling with stripe.
-
-5.1.2 Configuration Brief
--------------------------
-
-This setup demonstrates how users can configure unify translator with
-appropriate I/O scheduler for file level scheduling and strip for only
-matching patterns. This way, GlusterFS chooses appropriate I/O profile
-and knows how to efficiently handle both the types of data.
-
-   A simple technique to achieve this effect is to create a stripe set
-of unify and stripe blocks, where unify is the first sub-volume. Files
-that do not match the stripe policy passed on to first unify sub-volume
-and inturn scheduled arcoss the cluster using its file level I/O
-scheduler.
-
- 5.1.3 Preparing GlusterFS Envoronment
--------------------------------------
-
-Create the directories /export/namespace, /export/unify and
-/export/stripe on all the storage bricks.
-
-   Place the following server and client volume spec file under
-/etc/glusterfs (or appropriate installed path) and replace the IP
-addresses / access control fields to match your environment.
-
-       ## file: /etc/glusterfs/glusterfsd.vol
-        volume posix-unify
-                type storage/posix
-                option directory /export/for-unify
-        end-volume
-
-        volume posix-stripe
-                type storage/posix
-                option directory /export/for-stripe
-        end-volume
-
-        volume posix-namespace
-                type storage/posix
-                option directory /export/for-namespace
-        end-volume
-
-        volume server
-                type protocol/server
-                option transport-type tcp
-                option auth.addr.posix-unify.allow 192.168.1.*
-                option auth.addr.posix-stripe.allow 192.168.1.*
-                option auth.addr.posix-namespace.allow 192.168.1.*
-                subvolumes posix-unify posix-stripe posix-namespace
-        end-volume
-
-      ## file: /etc/glusterfs/glusterfs.vol
-        volume client-namespace
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.1
-          option remote-subvolume posix-namespace
-        end-volume
-
-        volume client-unify-1
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.1
-          option remote-subvolume posix-unify
-        end-volume
-
-        volume client-unify-2
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.2
-          option remote-subvolume posix-unify
-        end-volume
-
-        volume client-unify-3
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.3
-          option remote-subvolume posix-unify
-        end-volume
-
-        volume client-unify-4
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.4
-          option remote-subvolume posix-unify
-        end-volume
-
-        volume client-stripe-1
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.1
-          option remote-subvolume posix-stripe
-        end-volume
-
-        volume client-stripe-2
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.2
-          option remote-subvolume posix-stripe
-        end-volume
-
-        volume client-stripe-3
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.3
-          option remote-subvolume posix-stripe
-        end-volume
-
-        volume client-stripe-4
-          type protocol/client
-          option transport-type tcp
-          option remote-host 192.168.1.4
-          option remote-subvolume posix-stripe
-        end-volume
-
-        volume unify
-          type cluster/unify
-          option scheduler rr
-          subvolumes cluster-unify-1 cluster-unify-2 cluster-unify-3 cluster-unify-4
-        end-volume
-
-        volume stripe
-          type cluster/stripe
-          option block-size *.img:2MB # All files ending with .img are striped with 2MB stripe block size.
-          subvolumes unify cluster-stripe-1 cluster-stripe-2 cluster-stripe-3 cluster-stripe-4
-        end-volume
-
-   Bring up the Storage
-
-   Starting GlusterFS Server: If you have installed through binary
-package, you can start the service through init.d startup script. If
-not:
-
-     [root@server]# glusterfsd
-
-   Mounting GlusterFS Volumes:
-
-     [root@client]# glusterfs -s [BRICK-IP-ADDRESS] /mnt/cluster
-
-   Improving upon this Setup
-
-   Infiniband Verbs RDMA transport is much faster than TCP/IP GigE
-transport.
-
-   Use of performance translators such as read-ahead, write-behind,
-io-cache, io-threads, booster is recommended.
-
-   Replace round-robin (rr) scheduler with ALU to handle more dynamic
-storage environments.
-
-   ---------- Footnotes ----------
-
-   (1)
-http://gluster.org/docs/index.php/Mixing_Striped_and_Regular_Files
-
-
-File: user-guide.info,  Node: Troubleshooting,  Next: GNU Free Documentation Licence,  Prev: Usage Scenarios,  Up: Top
-
-6 Troubleshooting
-*****************
-
-This chapter is a general troubleshooting guide to GlusterFS. It lists
-common GlusterFS server and client error messages, debugging hints, and
-concludes with the suggested procedure to report bugs in GlusterFS.
-
-6.1 GlusterFS error messages
-============================
-
-6.1.1 Server errors
--------------------
-
-     glusterfsd: FATAL: could not open specfile:
-     '/etc/glusterfs/glusterfsd.vol'
-
-   The GlusterFS server expects the volume specification file to be at
-`/etc/glusterfs/glusterfsd.vol'. The example specification file will be
-installed as `/etc/glusterfs/glusterfsd.vol.sample'. You need to edit
-it and rename it, or provide a different specification file using the
-`--spec-file' command line option (See *note Server::).
-
-     gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log"
-                  (Permission denied)
-
-   You don't have permission to create files in the
-`/usr/var/log/glusterfs' directory. Make sure you are running GlusterFS
-as root. Alternatively, specify a different path for the log file using
-the `--log-file' option (See *note Server::).
-
-6.1.2 Client errors
--------------------
-
-     fusermount: failed to access mountpoint /mnt:
-                 Transport endpoint is not connected
-
-   A previous failed (or hung) mount of GlusterFS is preventing it from
-being mounted again in the same location. The fix is to do:
-
-     # umount /mnt
-
-   and try mounting again.
-
-   *"Transport endpoint is not connected".*
-
-   If you get this error when you try a command such as `ls' or `cat',
-it means the GlusterFS mount did not succeed. Try running GlusterFS in
-`DEBUG' logging level and study the log messages to discover the cause.
-
-   *"Connect to server failed", "SERVER-ADDRESS: Connection refused".*
-
-   GluserFS Server is not running or dead. Check your network
-connections and firewall settings. To check if the server is reachable,
-try:
-
-     telnet IP-ADDRESS 24007
-
-   If the server is accessible, your `telnet' command should connect and
-block. If not you will see an error message such as `telnet: Unable to
-connect to remote host: Connection refused'. 24007 is the default
-GlusterFS port. If you have changed it, then use the corresponding port
-instead.
-
-     gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log"
-                  (Permission denied)
-
-   You don't have permission to create files in the
-`/usr/var/log/glusterfs' directory. Make sure you are running GlusterFS
-as root. Alternatively, specify a different path for the log file using
-the `--log-file' option (See *note Client::).
-
-6.2 FUSE error messages
-=======================
-
-`modprobe fuse' fails with: "Unknown symbol in module, or unknown
-parameter".  
-
-   If you are using fuse-2.6.x on Redhat Enterprise Linux Work Station 4
-and Advanced Server 4 with 2.6.9-42.ELlargesmp, 2.6.9-42.ELsmp,
-2.6.9-42.EL kernels and get this error while loading FUSE kernel
-module, you need to apply the following patch.
-
-   For fuse-2.6.2:
-
-<http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.2-rhel-build.patch>
-
-   For fuse-2.6.3:
-
-<http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.3-rhel-build.patch>
-
-6.3 AppArmour and GlusterFS
-===========================
-
-Under OpenSuSE GNU/Linux, the AppArmour security feature does not allow
-GlusterFS to create temporary files or network socket connections even
-while running as root. You will see error messages like `Unable to open
-log file: Operation not permitted' or `Connection refused'. Disabling
-AppArmour using YaST or properly configuring AppArmour to recognize
-`glusterfsd' or `glusterfs'/`fusermount' should solve the problem.
-
-6.4 Reporting a bug
-===================
-
-If you encounter a bug in GlusterFS, please follow the below guidelines
-when you report it to the mailing list. Be sure to report it! User
-feedback is crucial to the health of the project and we value it highly.
-
-6.4.1 General instructions
---------------------------
-
-When running GlusterFS in a non-production environment, be sure to
-build it with the following command:
-
-      $ make CFLAGS='-g -O0 -DDEBUG'
-
-   This includes debugging information which will be helpful in getting
-backtraces (see below) and also disable optimization. Enabling
-optimization can result in incorrect line numbers being reported to gdb.
-
-6.4.2 Volume specification files
---------------------------------
-
-Attach all relevant server and client spec files you were using when
-you encountered the bug. Also tell us details of your setup, i.e., how
-many clients and how many servers.
-
-6.4.3 Log files
----------------
-
-Set the loglevel of your client and server programs to DEBUG (by
-passing the -L DEBUG option) and attach the log files with your bug
-report. Obviously, if only the client is failing (for example), you
-only need to send us the client log file.
-
-6.4.4 Backtrace
----------------
-
-If GlusterFS has encountered a segmentation fault or has crashed for
-some other reason, include the backtrace with the bug report. You can
-get the backtrace using the following procedure.
-
-   Run the GlusterFS client or server inside gdb.
-
-      $ gdb ./glusterfs
-      (gdb) set args -f client.spec -N -l/path/to/log/file -LDEBUG /mnt/point
-      (gdb) run
-
-   Now when the process segfaults, you can get the backtrace by typing:
-
-      (gdb) bt
-
-   If the GlusterFS process has crashed and dumped a core file (you can
-find this in / if running as a daemon and in the current directory
-otherwise), you can do:
-
-      $ gdb /path/to/glusterfs /path/to/core.<pid>
-
-   and then get the backtrace.
-
-   If the GlusterFS server or client seems to be hung, then you can get
-the backtrace by attaching gdb to the process. First get the `PID' of
-the process (using ps), and then do:
-
-      $ gdb ./glusterfs <pid>
-
-   Press Ctrl-C to interrupt the process and then generate the
-backtrace.
-
-6.4.5 Reproducing the bug
--------------------------
-
-If the bug is reproducible, please include the steps necessary to do
-so. If the bug is not reproducible, send us the bug report anyway.
-
-6.4.6 Other information
------------------------
-
-If you think it is relevant, send us also the version of FUSE you're
-using, the kernel version, platform.
-
-
-File: user-guide.info,  Node: GNU Free Documentation Licence,  Next: Index,  Prev: Troubleshooting,  Up: Top
-
-Appendix A GNU Free Documentation Licence
-*****************************************
-
-                      Version 1.2, November 2002
-
-     Copyright (C) 2000,2001,2002 Free Software Foundation, Inc.
-     59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
-
-     Everyone is permitted to copy and distribute verbatim copies
-     of this license document, but changing it is not allowed.
-
-  0. PREAMBLE
-
-     The purpose of this License is to make a manual, textbook, or other
-     functional and useful document "free" in the sense of freedom: to
-     assure everyone the effective freedom to copy and redistribute it,
-     with or without modifying it, either commercially or
-     noncommercially.  Secondarily, this License preserves for the
-     author and publisher a way to get credit for their work, while not
-     being considered responsible for modifications made by others.
-
-     This License is a kind of "copyleft", which means that derivative
-     works of the document must themselves be free in the same sense.
-     It complements the GNU Affero General Public License, which is a copyleft
-     license designed for free software.
-
-     We have designed this License in order to use it for manuals for
-     free software, because free software needs free documentation: a
-     free program should come with manuals providing the same freedoms
-     that the software does.  But this License is not limited to
-     software manuals; it can be used for any textual work, regardless
-     of subject matter or whether it is published as a printed book.
-     We recommend this License principally for works whose purpose is
-     instruction or reference.
-
-  1. APPLICABILITY AND DEFINITIONS
-
-     This License applies to any manual or other work, in any medium,
-     that contains a notice placed by the copyright holder saying it
-     can be distributed under the terms of this License.  Such a notice
-     grants a world-wide, royalty-free license, unlimited in duration,
-     to use that work under the conditions stated herein.  The
-     "Document", below, refers to any such manual or work.  Any member
-     of the public is a licensee, and is addressed as "you".  You
-     accept the license if you copy, modify or distribute the work in a
-     way requiring permission under copyright law.
-
-     A "Modified Version" of the Document means any work containing the
-     Document or a portion of it, either copied verbatim, or with
-     modifications and/or translated into another language.
-
-     A "Secondary Section" is a named appendix or a front-matter section
-     of the Document that deals exclusively with the relationship of the
-     publishers or authors of the Document to the Document's overall
-     subject (or to related matters) and contains nothing that could
-     fall directly within that overall subject.  (Thus, if the Document
-     is in part a textbook of mathematics, a Secondary Section may not
-     explain any mathematics.)  The relationship could be a matter of
-     historical connection with the subject or with related matters, or
-     of legal, commercial, philosophical, ethical or political position
-     regarding them.
-
-     The "Invariant Sections" are certain Secondary Sections whose
-     titles are designated, as being those of Invariant Sections, in
-     the notice that says that the Document is released under this
-     License.  If a section does not fit the above definition of
-     Secondary then it is not allowed to be designated as Invariant.
-     The Document may contain zero Invariant Sections.  If the Document
-     does not identify any Invariant Sections then there are none.
-
-     The "Cover Texts" are certain short passages of text that are
-     listed, as Front-Cover Texts or Back-Cover Texts, in the notice
-     that says that the Document is released under this License.  A
-     Front-Cover Text may be at most 5 words, and a Back-Cover Text may
-     be at most 25 words.
-
-     A "Transparent" copy of the Document means a machine-readable copy,
-     represented in a format whose specification is available to the
-     general public, that is suitable for revising the document
-     straightforwardly with generic text editors or (for images
-     composed of pixels) generic paint programs or (for drawings) some
-     widely available drawing editor, and that is suitable for input to
-     text formatters or for automatic translation to a variety of
-     formats suitable for input to text formatters.  A copy made in an
-     otherwise Transparent file format whose markup, or absence of
-     markup, has been arranged to thwart or discourage subsequent
-     modification by readers is not Transparent.  An image format is
-     not Transparent if used for any substantial amount of text.  A
-     copy that is not "Transparent" is called "Opaque".
-
-     Examples of suitable formats for Transparent copies include plain
-     ASCII without markup, Texinfo input format, LaTeX input format,
-     SGML or XML using a publicly available DTD, and
-     standard-conforming simple HTML, PostScript or PDF designed for
-     human modification.  Examples of transparent image formats include
-     PNG, XCF and JPG.  Opaque formats include proprietary formats that
-     can be read and edited only by proprietary word processors, SGML or
-     XML for which the DTD and/or processing tools are not generally
-     available, and the machine-generated HTML, PostScript or PDF
-     produced by some word processors for output purposes only.
-
-     The "Title Page" means, for a printed book, the title page itself,
-     plus such following pages as are needed to hold, legibly, the
-     material this License requires to appear in the title page.  For
-     works in formats which do not have any title page as such, "Title
-     Page" means the text near the most prominent appearance of the
-     work's title, preceding the beginning of the body of the text.
-
-     A section "Entitled XYZ" means a named subunit of the Document
-     whose title either is precisely XYZ or contains XYZ in parentheses
-     following text that translates XYZ in another language.  (Here XYZ
-     stands for a specific section name mentioned below, such as
-     "Acknowledgements", "Dedications", "Endorsements", or "History".)
-     To "Preserve the Title" of such a section when you modify the
-     Document means that it remains a section "Entitled XYZ" according
-     to this definition.
-
-     The Document may include Warranty Disclaimers next to the notice
-     which states that this License applies to the Document.  These
-     Warranty Disclaimers are considered to be included by reference in
-     this License, but only as regards disclaiming warranties: any other
-     implication that these Warranty Disclaimers may have is void and
-     has no effect on the meaning of this License.
-
-  2. VERBATIM COPYING
-
-     You may copy and distribute the Document in any medium, either
-     commercially or noncommercially, provided that this License, the
-     copyright notices, and the license notice saying this License
-     applies to the Document are reproduced in all copies, and that you
-     add no other conditions whatsoever to those of this License.  You
-     may not use technical measures to obstruct or control the reading
-     or further copying of the copies you make or distribute.  However,
-     you may accept compensation in exchange for copies.  If you
-     distribute a large enough number of copies you must also follow
-     the conditions in section 3.
-
-     You may also lend copies, under the same conditions stated above,
-     and you may publicly display copies.
-
-  3. COPYING IN QUANTITY
-
-     If you publish printed copies (or copies in media that commonly
-     have printed covers) of the Document, numbering more than 100, and
-     the Document's license notice requires Cover Texts, you must
-     enclose the copies in covers that carry, clearly and legibly, all
-     these Cover Texts: Front-Cover Texts on the front cover, and
-     Back-Cover Texts on the back cover.  Both covers must also clearly
-     and legibly identify you as the publisher of these copies.  The
-     front cover must present the full title with all words of the
-     title equally prominent and visible.  You may add other material
-     on the covers in addition.  Copying with changes limited to the
-     covers, as long as they preserve the title of the Document and
-     satisfy these conditions, can be treated as verbatim copying in
-     other respects.
-
-     If the required texts for either cover are too voluminous to fit
-     legibly, you should put the first ones listed (as many as fit
-     reasonably) on the actual cover, and continue the rest onto
-     adjacent pages.
-
-     If you publish or distribute Opaque copies of the Document
-     numbering more than 100, you must either include a
-     machine-readable Transparent copy along with each Opaque copy, or
-     state in or with each Opaque copy a computer-network location from
-     which the general network-using public has access to download
-     using public-standard network protocols a complete Transparent
-     copy of the Document, free of added material.  If you use the
-     latter option, you must take reasonably prudent steps, when you
-     begin distribution of Opaque copies in quantity, to ensure that
-     this Transparent copy will remain thus accessible at the stated
-     location until at least one year after the last time you
-     distribute an Opaque copy (directly or through your agents or
-     retailers) of that edition to the public.
-
-     It is requested, but not required, that you contact the authors of
-     the Document well before redistributing any large number of
-     copies, to give them a chance to provide you with an updated
-     version of the Document.
-
-  4. MODIFICATIONS
-
-     You may copy and distribute a Modified Version of the Document
-     under the conditions of sections 2 and 3 above, provided that you
-     release the Modified Version under precisely this License, with
-     the Modified Version filling the role of the Document, thus
-     licensing distribution and modification of the Modified Version to
-     whoever possesses a copy of it.  In addition, you must do these
-     things in the Modified Version:
-
-       A. Use in the Title Page (and on the covers, if any) a title
-          distinct from that of the Document, and from those of
-          previous versions (which should, if there were any, be listed
-          in the History section of the Document).  You may use the
-          same title as a previous version if the original publisher of
-          that version gives permission.
-
-       B. List on the Title Page, as authors, one or more persons or
-          entities responsible for authorship of the modifications in
-          the Modified Version, together with at least five of the
-          principal authors of the Document (all of its principal
-          authors, if it has fewer than five), unless they release you
-          from this requirement.
-
-       C. State on the Title page the name of the publisher of the
-          Modified Version, as the publisher.
-
-       D. Preserve all the copyright notices of the Document.
-
-       E. Add an appropriate copyright notice for your modifications
-          adjacent to the other copyright notices.
-
-       F. Include, immediately after the copyright notices, a license
-          notice giving the public permission to use the Modified
-          Version under the terms of this License, in the form shown in
-          the Addendum below.
-
-       G. Preserve in that license notice the full lists of Invariant
-          Sections and required Cover Texts given in the Document's
-          license notice.
-
-       H. Include an unaltered copy of this License.
-
-       I. Preserve the section Entitled "History", Preserve its Title,
-          and add to it an item stating at least the title, year, new
-          authors, and publisher of the Modified Version as given on
-          the Title Page.  If there is no section Entitled "History" in
-          the Document, create one stating the title, year, authors,
-          and publisher of the Document as given on its Title Page,
-          then add an item describing the Modified Version as stated in
-          the previous sentence.
-
-       J. Preserve the network location, if any, given in the Document
-          for public access to a Transparent copy of the Document, and
-          likewise the network locations given in the Document for
-          previous versions it was based on.  These may be placed in
-          the "History" section.  You may omit a network location for a
-          work that was published at least four years before the
-          Document itself, or if the original publisher of the version
-          it refers to gives permission.
-
-       K. For any section Entitled "Acknowledgements" or "Dedications",
-          Preserve the Title of the section, and preserve in the
-          section all the substance and tone of each of the contributor
-          acknowledgements and/or dedications given therein.
-
-       L. Preserve all the Invariant Sections of the Document,
-          unaltered in their text and in their titles.  Section numbers
-          or the equivalent are not considered part of the section
-          titles.
-
-       M. Delete any section Entitled "Endorsements".  Such a section
-          may not be included in the Modified Version.
-
-       N. Do not retitle any existing section to be Entitled
-          "Endorsements" or to conflict in title with any Invariant
-          Section.
-
-       O. Preserve any Warranty Disclaimers.
-
-     If the Modified Version includes new front-matter sections or
-     appendices that qualify as Secondary Sections and contain no
-     material copied from the Document, you may at your option
-     designate some or all of these sections as invariant.  To do this,
-     add their titles to the list of Invariant Sections in the Modified
-     Version's license notice.  These titles must be distinct from any
-     other section titles.
-
-     You may add a section Entitled "Endorsements", provided it contains
-     nothing but endorsements of your Modified Version by various
-     parties--for example, statements of peer review or that the text
-     has been approved by an organization as the authoritative
-     definition of a standard.
-
-     You may add a passage of up to five words as a Front-Cover Text,
-     and a passage of up to 25 words as a Back-Cover Text, to the end
-     of the list of Cover Texts in the Modified Version.  Only one
-     passage of Front-Cover Text and one of Back-Cover Text may be
-     added by (or through arrangements made by) any one entity.  If the
-     Document already includes a cover text for the same cover,
-     previously added by you or by arrangement made by the same entity
-     you are acting on behalf of, you may not add another; but you may
-     replace the old one, on explicit permission from the previous
-     publisher that added the old one.
-
-     The author(s) and publisher(s) of the Document do not by this
-     License give permission to use their names for publicity for or to
-     assert or imply endorsement of any Modified Version.
-
-  5. COMBINING DOCUMENTS
-
-     You may combine the Document with other documents released under
-     this License, under the terms defined in section 4 above for
-     modified versions, provided that you include in the combination
-     all of the Invariant Sections of all of the original documents,
-     unmodified, and list them all as Invariant Sections of your
-     combined work in its license notice, and that you preserve all
-     their Warranty Disclaimers.
-
-     The combined work need only contain one copy of this License, and
-     multiple identical Invariant Sections may be replaced with a single
-     copy.  If there are multiple Invariant Sections with the same name
-     but different contents, make the title of each such section unique
-     by adding at the end of it, in parentheses, the name of the
-     original author or publisher of that section if known, or else a
-     unique number.  Make the same adjustment to the section titles in
-     the list of Invariant Sections in the license notice of the
-     combined work.
-
-     In the combination, you must combine any sections Entitled
-     "History" in the various original documents, forming one section
-     Entitled "History"; likewise combine any sections Entitled
-     "Acknowledgements", and any sections Entitled "Dedications".  You
-     must delete all sections Entitled "Endorsements."
-
-  6. COLLECTIONS OF DOCUMENTS
-
-     You may make a collection consisting of the Document and other
-     documents released under this License, and replace the individual
-     copies of this License in the various documents with a single copy
-     that is included in the collection, provided that you follow the
-     rules of this License for verbatim copying of each of the
-     documents in all other respects.
-
-     You may extract a single document from such a collection, and
-     distribute it individually under this License, provided you insert
-     a copy of this License into the extracted document, and follow
-     this License in all other respects regarding verbatim copying of
-     that document.
-
-  7. AGGREGATION WITH INDEPENDENT WORKS
-
-     A compilation of the Document or its derivatives with other
-     separate and independent documents or works, in or on a volume of
-     a storage or distribution medium, is called an "aggregate" if the
-     copyright resulting from the compilation is not used to limit the
-     legal rights of the compilation's users beyond what the individual
-     works permit.  When the Document is included in an aggregate, this
-     License does not apply to the other works in the aggregate which
-     are not themselves derivative works of the Document.
-
-     If the Cover Text requirement of section 3 is applicable to these
-     copies of the Document, then if the Document is less than one half
-     of the entire aggregate, the Document's Cover Texts may be placed
-     on covers that bracket the Document within the aggregate, or the
-     electronic equivalent of covers if the Document is in electronic
-     form.  Otherwise they must appear on printed covers that bracket
-     the whole aggregate.
-
-  8. TRANSLATION
-
-     Translation is considered a kind of modification, so you may
-     distribute translations of the Document under the terms of section
-     4.  Replacing Invariant Sections with translations requires special
-     permission from their copyright holders, but you may include
-     translations of some or all Invariant Sections in addition to the
-     original versions of these Invariant Sections.  You may include a
-     translation of this License, and all the license notices in the
-     Document, and any Warranty Disclaimers, provided that you also
-     include the original English version of this License and the
-     original versions of those notices and disclaimers.  In case of a
-     disagreement between the translation and the original version of
-     this License or a notice or disclaimer, the original version will
-     prevail.
-
-     If a section in the Document is Entitled "Acknowledgements",
-     "Dedications", or "History", the requirement (section 4) to
-     Preserve its Title (section 1) will typically require changing the
-     actual title.
-
-  9. TERMINATION
-
-     You may not copy, modify, sublicense, or distribute the Document
-     except as expressly provided for under this License.  Any other
-     attempt to copy, modify, sublicense or distribute the Document is
-     void, and will automatically terminate your rights under this
-     License.  However, parties who have received copies, or rights,
-     from you under this License will not have their licenses
-     terminated so long as such parties remain in full compliance.
-
- 10. FUTURE REVISIONS OF THIS LICENSE
-
-     The Free Software Foundation may publish new, revised versions of
-     the GNU Free Documentation License from time to time.  Such new
-     versions will be similar in spirit to the present version, but may
-     differ in detail to address new problems or concerns.  See
-     `http://www.gnu.org/copyleft/'.
-
-     Each version of the License is given a distinguishing version
-     number.  If the Document specifies that a particular numbered
-     version of this License "or any later version" applies to it, you
-     have the option of following the terms and conditions either of
-     that specified version or of any later version that has been
-     published (not as a draft) by the Free Software Foundation.  If
-     the Document does not specify a version number of this License,
-     you may choose any version ever published (not as a draft) by the
-     Free Software Foundation.
-
-A.0.1 ADDENDUM: How to use this License for your documents
-----------------------------------------------------------
-
-To use this License in a document you have written, include a copy of
-the License in the document and put the following copyright and license
-notices just after the title page:
-
-       Copyright (C)  YEAR  YOUR NAME.
-       Permission is granted to copy, distribute and/or modify this document
-       under the terms of the GNU Free Documentation License, Version 1.2
-       or any later version published by the Free Software Foundation;
-       with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
-       Texts.  A copy of the license is included in the section entitled ``GNU
-       Free Documentation License''.
-
-   If you have Invariant Sections, Front-Cover Texts and Back-Cover
-Texts, replace the "with...Texts." line with this:
-
-         with the Invariant Sections being LIST THEIR TITLES, with
-         the Front-Cover Texts being LIST, and with the Back-Cover Texts
-         being LIST.
-
-   If you have Invariant Sections without Cover Texts, or some other
-combination of the three, merge those two alternatives to suit the
-situation.
-
-   If your document contains nontrivial examples of program code, we
-recommend releasing these examples in parallel under your choice of
-free software license, such as the GNU Affero General Public License, to
-permit their use in free software.
-
-
-File: user-guide.info,  Node: Index,  Prev: GNU Free Documentation Licence,  Up: Top
-
-Index
-*****
-
-
-* Menu:
-
-* alu (scheduler):                       Unify.               (line  49)
-* AppArmour:                             Troubleshooting.     (line  96)
-* arch:                                  Getting GlusterFS.   (line   6)
-* booster:                               Booster.             (line   6)
-* commercial support:                    Introduction.        (line  36)
-* DNS round robin:                       Transport modules.   (line  29)
-* fcntl:                                 POSIX Locks.         (line   6)
-* FDL, GNU Free Documentation License:   GNU Free Documentation Licence.
-                                                              (line   6)
-* fixed-id (translator):                 Fixed ID.            (line   6)
-* GlusterFS client:                      Client.              (line   6)
-* GlusterFS mailing list:                Introduction.        (line  28)
-* GlusterFS server:                      Server.              (line   6)
-* infiniband transport:                  Transport modules.   (line  58)
-* InfiniBand, installation:              Pre requisites.      (line  51)
-* io-cache (translator):                 IO Cache.            (line   6)
-* io-threads (translator):               IO Threads.          (line   6)
-* IRC channel, #gluster:                 Introduction.        (line  31)
-* libibverbs:                            Pre requisites.      (line  51)
-* namespace:                             Unify.               (line 207)
-* nufa (scheduler):                      Unify.               (line 175)
-* OpenSuSE:                              Troubleshooting.     (line  96)
-* posix-locks (translator):              POSIX Locks.         (line   6)
-* random (scheduler):                    Unify.               (line 159)
-* read-ahead (translator):               Read Ahead.          (line   6)
-* record locking:                        POSIX Locks.         (line   6)
-* Redhat Enterprise Linux:               Troubleshooting.     (line  78)
-* Replicate:                             Replicate.           (line   6)
-* rot-13 (translator):                   ROT-13.              (line   6)
-* rr (scheduler):                        Unify.               (line 138)
-* scheduler (unify):                     Unify.               (line   6)
-* self heal (replicate):                 Replicate.           (line  46)
-* self heal (unify):                     Unify.               (line 223)
-* stripe (translator):                   Stripe.              (line   6)
-* trace (translator):                    Trace.               (line   6)
-* unify (translator):                    Unify.               (line   6)
-* unify invariants:                      Unify.               (line  16)
-* write-behind (translator):             Write Behind.        (line   6)
-* Gluster, Inc.:                      Introduction.        (line  36)
-
-
-
-Tag Table:
-Node: Top704
-Node: Acknowledgements2304
-Node: Introduction3214
-Node: Installation and Invocation4649
-Node: Pre requisites4933
-Node: Getting GlusterFS7023
-Ref: Getting GlusterFS-Footnote-17809
-Node: Building7857
-Node: Running GlusterFS9559
-Node: Server9770
-Node: Client11358
-Node: A Tutorial Introduction13564
-Node: Concepts17101
-Node: Filesystems in Userspace17316
-Node: Translator18457
-Node: Volume specification file21160
-Node: Translators23632
-Node: Storage Translators24201
-Ref: Storage Translators-Footnote-125008
-Node: POSIX25142
-Node: BDB25765
-Node: Client and Server Translators26822
-Node: Transport modules27298
-Node: Client protocol31445
-Node: Server protocol32384
-Node: Clustering Translators33373
-Node: Unify34260
-Ref: Unify-Footnote-143859
-Node: Replicate43951
-Node: Stripe49006
-Node: Performance Translators50164
-Node: Read Ahead50438
-Node: Write Behind52170
-Node: IO Threads53579
-Node: IO Cache54367
-Node: Booster55691
-Node: Features Translators57105
-Node: POSIX Locks57333
-Node: Fixed ID58650
-Node: Miscellaneous Translators59136
-Node: ROT-1359334
-Node: Trace60013
-Node: Usage Scenarios61282
-Ref: Usage Scenarios-Footnote-167215
-Node: Troubleshooting67290
-Node: GNU Free Documentation Licence73638
-Node: Index96087
-
-End Tag Table
diff --git a/doc/user-guide/user-guide.pdf b/doc/user-guide/user-guide.pdf
deleted file mode 100644
index ed7bd2a9907..00000000000
--- a/doc/user-guide/user-guide.pdf
+++ /dev/null
diff --git a/doc/user-guide/user-guide.texi b/doc/user-guide/user-guide.texi
deleted file mode 100644
index b679dee024d..00000000000
--- a/doc/user-guide/user-guide.texi
+++ /dev/null
@@ -1,2246 +0,0 @@
-\input texinfo
-@setfilename user-guide.info
-@settitle GlusterFS 2.0 User Guide
-@afourpaper
-
-@direntry
-* GlusterFS: (user-guide). GlusterFS distributed filesystem user guide
-@end direntry
-
-@copying
-This is the user manual for GlusterFS 2.0.
-
-Copyright @copyright{} 2007-2010 @email{@b{Gluster}} , Inc. Permission is granted to
-copy, distribute and/or modify this document under the terms of the
-@acronym{GNU} Free Documentation License, Version 1.2 or any later
-version published by the Free Software Foundation; with no Invariant
-Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the
-license is included in the chapter entitled ``@acronym{GNU} Free
-Documentation License''.
-@end copying
-
-@titlepage
-@title GlusterFS 2.0 User Guide [DRAFT]
-@subtitle January 15, 2008
-@author http://gluster.org/core-team.php
-@author @email{@b{Gluster}} 
-@page
-@vskip 0pt plus 1filll
-@insertcopying
-@end titlepage
-
-@c Info stuff
-@ifnottex
-@node Top
-@top GlusterFS 2.0 User Guide
-
-@insertcopying
-@menu
-* Acknowledgements::            
-* Introduction::                
-* Installation and Invocation::  
-* Concepts::                    
-* Translators::                 
-* Usage Scenarios::             
-* Troubleshooting::             
-* GNU Free Documentation Licence::  
-* Index::                       
-
-@detailmenu
- --- The Detailed Node Listing ---
-
-Installation and Invocation
-
-* Pre requisites::              
-* Getting GlusterFS::           
-* Building::                    
-* Running GlusterFS::           
-* A Tutorial Introduction::     
-
-Running GlusterFS
-
-* Server::                      
-* Client::                      
-
-Concepts
-
-* Filesystems in Userspace::                
-* Translator::                  
-* Volume specification file::   
-
-Translators
-
-* Storage Translators::         
-* Client and Server Translators::  
-* Clustering Translators::      
-* Performance Translators::     
-* Features Translators::        
-
-Storage Translators
-
-* POSIX::        
-
-Client and Server Translators
-
-* Transport modules::           
-* Client protocol::             
-* Server protocol::             
-
-Clustering Translators
-
-* Unify::                       
-* Replicate::  
-* Stripe::                      
-
-Performance Translators
-
-* Read Ahead::                  
-* Write Behind::                
-* IO Threads::                  
-* IO Cache::                    
-
-Features Translators 
-
-* POSIX Locks::                 
-* Fixed ID::                    
-
-Miscellaneous Translators
-
-* ROT-13::                      
-* Trace::                       
-
-@end detailmenu
-@end menu
-
-@end ifnottex
-@c Info stuff end
-
-@contents
-
-@node Acknowledgements
-@unnumbered Acknowledgements
-GlusterFS continues to be a wonderful and enriching experience for all
-of us involved. 
-
-GlusterFS development would not have been possible at this pace if
-not for our enthusiastic users. People from around the world have
-helped us with bug reports, performance numbers, and feature suggestions.
-A huge thanks to them all.
-
-Matthew Paine - for RPMs & general enthu
-
-Leonardo Rodrigues de Mello - for DEBs
-
-Julian Perez & Adam D'Auria - for multi-server tutorial
-
-Paul England - for HA spec
-
-Brent Nelson - for many bug reports
-
-Jacques Mattheij - for Europe mirror.
-
-Patrick Negri - for TCP non-blocking connect.
-@flushright
-http://gluster.org/core-team.php (@email{list-hacking@@gluster.com})
-@email{@b{Gluster}} 
-@end flushright
-
-@node Introduction
-@chapter Introduction
-
-GlusterFS is a distributed filesystem. It works at the file level,
-not block level.
-
-A network filesystem is one which allows us to access remote files. A
-distributed filesystem is one that stores data on multiple machines
-and makes them all appear to be a part of the same filesystem.
-
-Need for distributed filesystems
-
-@itemize @bullet
-@item Scalability: A distributed filesystem allows us to store more data than what can be stored on a single machine.
-
-@item Redundancy: We might want to replicate crucial data on to several machines.
-
-@item Uniform access: One can mount a remote volume (for example your home directory) from any machine and access the same data.
-@end itemize
-
-@section Contacting us
-You can reach us through the mailing list @strong{gluster-devel} 
-(@email{gluster-devel@@nongnu.org}).
-@cindex GlusterFS mailing list
-
-You can also find many of the developers on @acronym{IRC}, on the @code{#gluster}
-channel on Freenode (@indicateurl{irc.freenode.net}).
-@cindex IRC channel, #gluster
-
-The GlusterFS documentation wiki is also useful: @*
-@indicateurl{http://gluster.org/docs/index.php/GlusterFS}
-
-For commercial support, you can contact @email{@b{Gluster}} at:
-@cindex commercial support
-@cindex Gluster, Inc.
-
-@display
-3194 Winding Vista Common
-Fremont, CA 94539
-USA.
-
-Phone: +1 (510) 354 6801
-Toll free: +1 (888) 813 6309
-Fax: +1 (510) 372 0604
-@end display
-
-You can also email us at @email{support@@gluster.com}.
-
-@node Installation and Invocation
-@chapter Installation and Invocation
-
-@menu
-* Pre requisites::              
-* Getting GlusterFS::           
-* Building::                    
-* Running GlusterFS::           
-* A Tutorial Introduction::     
-@end menu
-
-@node Pre requisites
-@section Pre requisites
-
-Before installing GlusterFS make sure you have the
-following components installed.
-
-@subsection @acronym{FUSE}
-GlusterFS has now built-in support for the @acronym{FUSE} protocol.
-You need a kernel with @acronym{FUSE} support to mount GlusterFS.
-You do not need the @acronym{FUSE} package (library and utilities),
-but be aware of the following issues:
-
-@itemize
-@item If you want unprivileged users to be able to mount GlusterFS filesystems,
-you need a recent version of the @command{fusermount} utility. You already have
-it if you have @acronym{FUSE} version 2.7.0 or higher installed; if that's not
-the case, one will be compiled along with GlusterFS if you pass
-@command{--enable-fusermount} to the @command{configure} script.  @item You
-need to ensure @acronym{FUSE} support is configured properly on your system. In
-details:
-@itemize
-@item If your kernel has @acronym{FUSE} as a loadable module, make sure it's
-loaded.
-@item Create @command{/dev/fuse} (major 10, minor 229) either by means of udev
-rules or by hand.
-@item Optionally, if you want runtime control over your @acronym{FUSE} mounts,
-mount the fusectl auxiliary filesystem:
-
-@example
-# mount -t fusectl none /sys/fs/fuse/connections
-@end example
-@end itemize
-
-The @acronym{FUSE} packages shipped by the various distributions usually take care
-about these things, so the easiest way to get the above tasks handled is still
-installing the @acronym{FUSE} package(s).
-@end itemize
-
-To get the best performance from GlusterFS,it is recommended that you use
-our patched version of the @acronym{FUSE} kernel module. See Patched FUSE for details.
-
-@subsection Patched FUSE
-
-The GlusterFS project maintains a patched version of @acronym{FUSE} meant to be used 
-with GlusterFS. The patches increase GlusterFS performance. It is recommended that
-all users use the patched @acronym{FUSE}.
-
-The patched @acronym{FUSE} tarball can be downloaded from:
-
-@indicateurl{ftp://ftp.gluster.com/pub/gluster/glusterfs/fuse/}
-
-The specific changes made to @acronym{FUSE} are:
-
-@itemize
-@item The communication channel size between @acronym{FUSE} kernel module and GlusterFS has been increased to 1MB, permitting large reads and writes to be sent in bigger chunks.
-
-@item The kernel's read-ahead boundry has been extended upto 1MB.
-
-@item Block size returned in the @command{stat()}/@command{fstat()} calls tuned to 1MB, to make cp and similar commands perform I/O using that block size.
-
-@item @command{flock()} locking support has been added (although some rework in GlusterFS is needed for perfect compliance).
-@end itemize
-
-@subsection libibverbs (optional)
-@cindex InfiniBand, installation
-@cindex libibverbs
-This is only needed if you want GlusterFS to use InfiniBand as the
-interconnect mechanism between server and client. You can get it from:
-
-@indicateurl{http://www.openfabrics.org/downloads.htm}.
-
-@subsection Bison and Flex
-These should be already installed on most Linux systems. If not, use your distribution's
-normal software installation procedures to install them. Make sure you install the
-relevant developer packages also.
-
-@node Getting GlusterFS
-@section Getting GlusterFS
-@cindex arch
-There are many ways to get hold of GlusterFS. For a production deployment,
-the recommended method is to download the latest release tarball.
-Release tarballs are available at: @indicateurl{http://gluster.org/download.php}.
-
-If you want the bleeding edge development source, you can get them
-from the Git
-@footnote{@indicateurl{http://git-scm.com}}
-repository. First you must install Git itself. Then
-you can check out the source
-
-@example
-$ git clone git://git.sv.gnu.org/gluster.git glusterfs
-@end example
-
-@node Building
-@section Building
-You can skip this section if you're installing from @acronym{RPM}s
-or @acronym{DEB}s.
-
-GlusterFS uses the Autotools mechanism to build. As such, the procedure
-is straight-forward. First, change into the GlusterFS source directory.
-
-@example
-$ cd glusterfs-<version>
-@end example
-
-If you checked out the source from the Arch repository, you'll need
-to run @command{./autogen.sh} first. Note that you'll need to have
-Autoconf and Automake installed for this. 
-
-Run @command{configure}.
-
-@example
-$ ./configure
-@end example
-
-The configure script accepts the following options:
-
-@cartouche
-@table @code
-
-@item --disable-ibverbs
-Disable the InfiniBand transport mechanism.
-
-@item --disable-fuse-client
-Disable the @acronym{FUSE} client.
-
-@item --disable-server
-Disable building of the GlusterFS server.
-
-@item --disable-bdb
-Disable building of Berkeley DB based storage translator.
-
-@item --disable-mod_glusterfs
-Disable building of Apache/lighttpd glusterfs plugins.
-
-@item --disable-epoll
-Use poll instead of epoll.
-
-@item --disable-libglusterfsclient
-Disable building of libglusterfsclient
-
-@item --enable-fusermount
-Build fusermount
-
-@end table
-@end cartouche
-
-Build and install GlusterFS.
-
-@example
-# make install
-@end example
-
-The binaries (@command{glusterfsd} and @command{glusterfs}) will be by
-default installed in @command{/usr/local/sbin/}. Translator,
-scheduler, and transport shared libraries will be installed in
-@command{/usr/local/lib/glusterfs/<version>/}. Sample volume
-specification files will be in @command{/usr/local/etc/glusterfs/}.
-This document itself can be found in
-@command{/usr/local/share/doc/glusterfs/}. If you passed the @command{--prefix}
-argument to the configure script, then replace @command{/usr/local} in the preceding
-paths with the prefix.
-
-@node Running GlusterFS
-@section Running GlusterFS
-
-@menu
-* Server::                      
-* Client::                      
-@end menu
-
-@node Server
-@subsection Server
-@cindex GlusterFS server
-
-The GlusterFS server is necessary to export storage volumes to remote clients
-(See @ref{Server protocol} for more info). This section documents the invocation
-of the GlusterFS server program and all the command-line options accepted by it.
-
-@cartouche
-@table @code
-Basic Options
-@item -f, --volfile=<path>   
-      Use the volume file as the volume specification.
-
-@item -s, --volfile-server=<hostname>
-      Server to get volume file from. This option overrides --volfile option.
-
-@item -l, --log-file=<path>
-      Specify the path for the log file.
-
-@item -L, --log-level=<level>
-      Set the log level for the server. Log level should be one of @acronym{DEBUG}, 
-@acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}.
-
-Advanced Options
-@item --debug
-      Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and
-      --log-file to console.
-
-@item  -N, --no-daemon            
-      Run glusterfsd as a foreground process.
-
-@item  -p, --pid-file=<path>      
-      Path for the @acronym{PID} file.
-
-@item --volfile-id=<key>
-      'key' of the volfile to be fetched from server.
-
-@item --volfile-server-port=<port-number>
-      Listening port number of volfile server.
-
-@item --volfile-server-transport=[socket|ib-verbs]
-      Transport type to get volfile from server. [default: @command{socket}]
-
-@item --xlator-options=<volume-name.option=value>
-      Add/override a translator option for a volume with specified value.
-
-Miscellaneous Options
-@item  -?, --help                 
-       Show this help text.
-
-@item  --usage                
-       Display a short usage message.
-
-@item  -V, --version              
-       Show version information.
-@end table
-@end cartouche
-
-@node Client
-@subsection Client
-@cindex GlusterFS client
-
-The GlusterFS client process is necessary to access remote storage volumes and
-mount them locally using @acronym{FUSE}. This section documents the invocation of the
-client process and all its command-line arguments.
-
-@example
-  # glusterfs [options] <mountpoint>
-@end example
-
-The @command{mountpoint} is the directory where you want the GlusterFS
-filesystem to appear. Example:
-
-@example
-  # glusterfs -f /usr/local/etc/glusterfs-client.vol /mnt
-@end example
-
-The command-line options are detailed below.
-
-@tex
-\vfill
-@end tex
-@page
-
-@cartouche
-@table @code
-
-Basic Options
-@item -f, --volfile=<path>   
-      Use the volume file as the volume specification.
-
-@item -s, --volfile-server=<hostname>
-      Server to get volume file from. This option overrides --volfile option.
-
-@item -l, --log-file=<path>
-      Specify the path for the log file.
-
-@item -L, --log-level=<level>
-      Set the log level for the server. Log level should be one of @acronym{DEBUG}, 
-@acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}.
-
-Advanced Options
-@item --debug
-      Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and
-      --log-file to console.
-
-@item  -N, --no-daemon            
-      Run @command{glusterfs} as a foreground process.
-
-@item  -p, --pid-file=<path>      
-      Path for the @acronym{PID} file.
-
-@item --volfile-id=<key>
-      'key' of the volfile to be fetched from server.
-
-@item --volfile-server-port=<port-number>
-      Listening port number of volfile server.
-
-@item --volfile-server-transport=[socket|ib-verbs]
-      Transport type to get volfile from server. [default: @command{socket}]
-
-@item --xlator-options=<volume-name.option=value>
-      Add/override a translator option for a volume with specified value.
-
-@item  --volume-name=<volume name>
-      Volume name in client spec to use. Defaults to the root volume.
-
-@acronym{FUSE} Options
-@item  --attribute-timeout=<n>
-       Attribute timeout for inodes in the kernel, in seconds. Defaults to 1 second.
-
-@item  --disable-direct-io-mode
-       Disable direct @acronym{I/O} mode in @acronym{FUSE} kernel module. This is set
-       automatically if kernel supports big writes (>= 2.6.26).
-
-@item  -e, --entry-timeout=<n>
-       Entry timeout for directory entries in the kernel, in seconds. 
-       Defaults to 1 second.
-
-Missellaneous Options
-@item  -?, --help                 
-       Show this help information.
-
-@item  -V, --version              
-       Show version information.
-@end table
-@end cartouche
-
-@node A Tutorial Introduction
-@section A Tutorial Introduction
-
-This section will show you how to quickly get GlusterFS up and running. We'll 
-configure GlusterFS as a simple network filesystem, with one server and one client.
-In this mode of usage, GlusterFS can serve as a replacement for NFS.
-
-We'll make use of two machines; call them @emph{server} and
-@emph{client} (If you don't want to setup two machines, just run
-everything that follows on the same machine).  In the examples that
-follow, the shell prompts will use these names to clarify the machine
-on which the command is being run. For example, a command that should
-be run on the server will be shown with the prompt:
-
-@example
-[root@@server]#
-@end example
-
-Our goal is to make a directory on the @emph{server} (say, @command{/export})
-accessible to the @emph{client}.
-
-First of all, get GlusterFS installed on both the machines, as described in the 
-previous sections. Make sure you have the @acronym{FUSE} kernel module loaded. You
-can ensure this by running: 
-
-@example
-[root@@server]# modprobe fuse
-@end example
-
-Before we can run the GlusterFS client or server programs, we need to write
-two files called @emph{volume specifications} (equivalently refered to as @emph{volfiles}). 
-The volfile describes the @emph{translator tree} on a node. The next chapter will
-explain the concepts of `translator' and `volume specification' in detail. For now, 
-just assume that the volfile is like an NFS @command{/etc/export} file.
-
-On the server, create a text file somewhere (we'll assume the path
-@command{/tmp/glusterfsd.vol}) with the following contents.
-
-@cartouche
-@example
-volume colon-o
-  type storage/posix
-  option directory /export
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes colon-o
-  option transport-type tcp     
-  option auth.addr.colon-o.allow *
-end-volume
-@end example
-@end cartouche
-
-A brief explanation of the file's contents. The first section defines a storage
-volume, named ``colon-o'' (the volume names are arbitrary), which exports the
-@command{/export} directory. The second section defines options for the translator
-which will make the storage volume accessible remotely. It specifies @command{colon-o} as
-a subvolume. This defines the @emph{translator tree}, about which more will be said
-in the next chapter. The two options specify that the @acronym{TCP} protocol is to be
-used (as opposed to InfiniBand, for example), and that access to the storage volume
-is to be provided to clients with any @acronym{IP} address at all. If you wanted to
-restrict access to this server to only your subnet for example, you'd specify
-something like @command{192.168.1.*} in the second option line.
-
-On the client machine, create the following text file (again, we'll assume
-the path to be @command{/tmp/glusterfs-client.vol}). Replace
-@emph{server-ip-address} with the @acronym{IP} address of your server machine. If you
-are doing all this on a single machine, use @command{127.0.0.1}.
-
-@cartouche
-@example
-volume client
-  type protocol/client
-  option transport-type tcp
-  option remote-host @emph{server-ip-address}
-  option remote-subvolume colon-o
-end-volume
-@end example
-@end cartouche
-
-Now we need to start both the server and client programs. To start the server:
-
-@example
-[root@@server]# glusterfsd -f /tmp/glusterfs-server.vol
-@end example
-
-To start the client:
-
-@example
-[root@@client]# glusterfs -f /tmp/glusterfs-client.vol /mnt/glusterfs
-@end example
-
-You should now be able to see the files under the server's @command{/export} directory
-in the @command{/mnt/glusterfs} directory on the client. That's it; GlusterFS is now
-working as a network file system.
-
-@node Concepts
-@chapter Concepts
-
-@menu
-* Filesystems in Userspace::                
-* Translator::                  
-* Volume specification file::   
-@end menu
-
-@node Filesystems in Userspace
-@section Filesystems in Userspace
-
-A filesystem is usually implemented in kernel space. Kernel space
-development is much harder than userspace development. @acronym{FUSE}
-is a kernel module/library that allows us to write a filesystem
-completely in userspace.
-
-@acronym{FUSE} consists of a kernel module which interacts with the userspace
-implementation using a device file @code{/dev/fuse}. When a process 
-makes a syscall on a @acronym{FUSE} filesystem, @acronym{VFS} hands the request to the
-@acronym{FUSE} module, which writes the request to @code{/dev/fuse}. The
-userspace implementation polls @code{/dev/fuse}, and when a request arrives,
-processes it and writes the result back to @code{/dev/fuse}. The kernel then
-reads from the device file and returns the result to the user process. 
-
-In case of GlusterFS, the userspace program is the GlusterFS client.
-The control flow is shown in the diagram below. The GlusterFS client
-services the request by sending it to the server, which in turn 
-hands it to the local @acronym{POSIX} filesystem.
-
-@center @image{fuse,44pc,,,.pdf}
-@center Fig 1. Control flow in GlusterFS
-
-@node Translator
-@section Translator
-
-The @emph{translator} is the most important concept in GlusterFS. In
-fact, GlusterFS is nothing but a collection of translators working
-together, forming a translator @emph{tree}.
-
-The idea of a translator is perhaps best understood using an
-analogy. Consider the @acronym{VFS} in the Linux kernel. The
-@acronym{VFS} abstracts the various filesystem implementations (such
-as @acronym{EXT3}, ReiserFS, @acronym{XFS}, etc.) supported by the
-kernel. When an application calls the kernel to perform an operation
-on a file, the kernel passes the request on to the appropriate
-filesystem implementation.
-
-For example, let's say there are two partitions on a Linux machine:
-@command{/}, which is an @acronym{EXT3} partition, and @command{/usr},
-which is a ReiserFS partition. Now if an application wants to open a
-file called, say, @command{/etc/fstab}, then the kernel will
-internally pass the request to the @acronym{EXT3} implementation.  If
-on the other hand, an application wants to read a file called
-@command{/usr/src/linux/CREDITS}, then the kernel will call upon the
-ReiserFS implementation to do the job.
-
-The ``filesystem implementation'' objects are analogous to GlusterFS
-translators. A GlusterFS translator implements all the filesystem
-operations.  Whereas in @acronym{VFS} there is a two-level tree (with
-the kernel at the root and all the filesystem implementation as its
-children), in GlusterFS there exists a more elaborate tree structure.
-
-We can now define translators more precisely. A GlusterFS translator
-is a shared object (@command{.so}) that implements every filesystem
-call. GlusterFS translators can be arranged in an arbitrary tree
-structure (subject to constraints imposed by the translators). When
-GlusterFS receives a filesystem call, it passes it on to the
-translator at the root of the translator tree. The root translator may
-in turn pass it on to any or all of its children, and so on, until the
-leaf nodes are reached. The result of a filesystem call is
-communicated in the reverse fashion, from the leaf nodes up to the
-root node, and then on to the application.
-
-So what might a translator tree look like?
-
-@tex
-\vfill
-@end tex
-@page
-
-@center @image{xlator,44pc,,,.pdf}
-@center Fig 2. A sample translator tree
-
-The diagram depicts three servers and one GlusterFS client. It is important
-to note that conceptually, the translator tree spans machine boundaries.
-Thus, the client machine in the diagram, @command{10.0.0.1}, can access
-the aggregated storage of the filesystems on the server machines @command{10.0.0.2},
-@command{10.0.0.3}, and @command{10.0.0.4}. The translator diagram will make more
-sense once you've read the next chapter and understood the functions of the
-various translators.
-
-@node Volume specification file
-@section Volume specification file
-The volume specification file describes the translator tree for both the
-server and client programs.
-
-A volume specification file is a sequence of volume definitions.
-The syntax of a volume definition is explained below:
-
-@cartouche
-@example
-@strong{volume} @emph{volume-name}
-  @strong{type} @emph{translator-name}
-  @strong{option} @emph{option-name} @emph{option-value}
-  @dots{}
-  @strong{subvolumes} @emph{subvolume1} @emph{subvolume2} @dots{}
-@strong{end-volume}
-@end example
-
-@dots{}
-@end cartouche
-
-@table @asis
-@item @emph{volume-name}
-  An identifier for the volume. This is just a human-readable name,
-and can contain any alphanumeric character. For instance, ``storage-1'', ``colon-o'',
-or ``forty-two''.
-
-@item @emph{translator-name}
-  Name of one of the available translators. Example: @command{protocol/client},
-@command{cluster/unify}.
-
-@item @emph{option-name}
-  Name of a valid option for the translator.
-
-@item @emph{option-value}
-  Value for the option. Everything following the ``option'' keyword to the end of the
-line is considered the value; it is up to the translator to parse it.
-
-@item @emph{subvolume1}, @emph{subvolume2}, @dots{}
-  Volume names of sub-volumes. The sub-volumes must already have been defined earlier 
-in the file.
-@end table
-
-There are a few rules you must follow when writing a volume specification file:
-
-@itemize
-@item Everything following a `@command{#}' is considered a comment and is ignored. Blank lines are also ignored.
-@item All names and keywords are case-sensitive.
-@item The order of options inside a volume definition does not matter.
-@item An option value may not span multiple lines.
-@item If an option is not specified, it will assume its default value.
-@item A sub-volume must have already been defined before it can be referenced. This means you have to write the specification file ``bottom-up'', starting from the leaf nodes of the translator tree and moving up to the root.
-@end itemize
-
-A simple example volume specification file is shown below:
-
-@cartouche
-@example
-# This is a comment line
-volume client
- type protocol/client
- option transport-type tcp
- option remote-host localhost      # Also a comment
- option remote-subvolume brick
-# The subvolumes line may be absent
-end-volume
-
-volume iot
- type performance/io-threads
- option thread-count 4
- subvolumes client
-end-volume
-
-volume wb
- type performance/write-behind
- subvolumes iot
-end-volume
-@end example
-@end cartouche
-
-@node Translators
-@chapter Translators
-
-@menu
-* Storage Translators::         
-* Client and Server Translators::  
-* Clustering Translators::      
-* Performance Translators::     
-* Features Translators::        
-* Miscellaneous Translators::
-@end menu
-
-This chapter documents all the available GlusterFS translators in detail.
-Each translator section will show its name (for example, @command{cluster/unify}),
-briefly describe its purpose and workings, and list every option accepted by
-that translator and their meaning.
-
-@node Storage Translators
-@section Storage Translators
-
-The storage translators form the ``backend'' for GlusterFS. Currently,
-the only available storage translator is the @acronym{POSIX}
-translator, which stores files on a normal @acronym{POSIX}
-filesystem. A pleasant consequence of this is that your data will
-still be accessible if GlusterFS crashes or cannot be started.
-
-Other storage backends are planned for the future. One of the possibilities is an
-Amazon S3 translator. Amazon S3 is an unlimited online storage service accessible
-through a web services @acronym{API}. The S3 translator will allow you to access
-the storage as a normal @acronym{POSIX} filesystem.
-@footnote{Some more discussion about this can be found at: 
-
-http://developer.amazonwebservices.com/connect/message.jspa?messageID=52873}
-
-@menu
-* POSIX::        
-* BDB::
-@end menu
-
-@node POSIX
-@subsection POSIX
-@example
-type storage/posix
-@end example
-
-The @command{posix} translator uses a normal @acronym{POSIX}
-filesystem as its ``backend'' to actually store files and
-directories. This can be any filesystem that supports extended
-attributes (@acronym{EXT3}, ReiserFS, @acronym{XFS}, ...). Extended
-attributes are used by some translators to store metadata, for
-example, by the replicate and stripe translators. See 
-@ref{Replicate} and @ref{Stripe}, respectively for details.
-
-@cartouche
-@table @code
-@item directory <path>
-The directory on the local filesystem which is to be used for storage.
-@end table               
-@end cartouche
-
-@node BDB
-@subsection BDB
-@example
-type storage/bdb
-@end example
-
-The @command{BDB} translator uses a @acronym{Berkeley DB} database as its
-``backend'' to actually store files as key-value pair in the database and
-directories as regular @acronym{POSIX} directories. Note that @acronym{BDB}
-does not provide extended attribute support for regular files. Do not use 
-@acronym{BDB} as storage translator while using any translator that demands
-extended attributes on ``backend''.
-
-@cartouche
-@table @code
-@item directory <path>
-The directory on the local filesystem which is to be used for storage.
-@item mode [cache|persistent] (cache)
-When @acronym{BDB} is run in @command{cache} mode, recovery of back-end is not completely
-guaranteed. @command{persistent} guarantees that @acronym{BDB} can recover back-end from
-@acronym{Berkeley DB} even if GlusterFS crashes.
-@item errfile <path>
-The path of the file to be used as @command{errfile} for @acronym{Berkeley DB} to report
-detailed error messages, if any. Note that all the contents of this file will be written
-by @acronym{Berkeley DB}, not GlusterFS.
-@item logdir <path>
-
-
-@end table
-@end cartouche
-
-@node Client and Server Translators, Clustering Translators, Storage Translators, Translators
-@section Client and Server Translators
-
-The client and server translator enable GlusterFS to export a
-translator tree over the network or access a remote GlusterFS
-server. These two translators implement GlusterFS's network protocol.
-
-@menu
-* Transport modules::           
-* Client protocol::             
-* Server protocol::             
-@end menu
-
-@node Transport modules
-@subsection Transport modules
-The client and server translators are capable of using any of the
-pluggable transport modules. Currently available transport modules are
-@command{tcp}, which uses a @acronym{TCP} connection between client
-and server to communicate; @command{ib-sdp}, which uses a
-@acronym{TCP} connection over InfiniBand, and @command{ibverbs}, which
-uses high-speed InfiniBand connections.
-
-Each transport module comes in two different versions, one to be used on
-the server side and the other on the client side.
-
-@subsubsection TCP
-
-The @acronym{TCP} transport module uses a @acronym{TCP/IP} connection between
-the server and the client.
-
-@example
-  option transport-type tcp
-@end example
-
-The @acronym{TCP} client module accepts the following options:
-
-@cartouche
-@table @code
-@item non-blocking-connect [no|off|on|yes] (on)
-Whether to make the connection attempt asynchronous.
-@item remote-port <n> (24007)
-Server port to connect to.
-@cindex DNS round robin
-@item remote-host <hostname> *
-Hostname or @acronym{IP} address of the server. If the host name resolves to
-multiple IP addresses, all of them will be tried in a round-robin fashion. This
-feature can be used to implement fail-over.
-@end table
-@end cartouche
-
-The @acronym{TCP} server module accepts the following options:
-
-@cartouche
-@table @code
-@item bind-address <address> (0.0.0.0)
-The local interface on which the server should listen to requests. Default is to
-listen on all interfaces.
-@item listen-port <n> (24007)
-The local port to listen on.
-@end table
-@end cartouche
-
-@subsubsection IB-SDP
-@example
-  option transport-type ib-sdp
-@end example
-
-kernel implements socket interface for ib hardware. SDP is over ib-verbs.
-This module accepts the same options as @command{tcp}
-
-@subsubsection ibverbs
-
-@example
-  option transport-type tcp
-@end example
-
-@cindex infiniband transport
-
-InfiniBand is a scalable switched fabric interconnect mechanism 
-primarily used in high-performance computing. InfiniBand can deliver
-data throughput of the order of 10 Gbit/s, with latencies of 4-5 ms.
-
-The @command{ib-verbs} transport accesses the InfiniBand hardware through
-the ``verbs'' @acronym{API}, which is the lowest level of software access possible
-and which gives the highest performance. On InfiniBand hardware, it is always
-best to use @command{ib-verbs}. Use @command{ib-sdp} only if you cannot get
-@command{ib-verbs} working for some reason. 
-
-The @command{ib-verbs} client module accepts the following options:
-
-@cartouche
-@table @code
-@item non-blocking-connect [no|off|on|yes] (on)
-Whether to make the connection attempt asynchronous.
-@item remote-port <n> (24007)
-Server port to connect to.
-@cindex DNS round robin
-@item remote-host <hostname> *
-Hostname or @acronym{IP} address of the server. If the host name resolves to
-multiple IP addresses, all of them will be tried in a round-robin fashion. This
-feature can be used to implement fail-over.
-@end table
-@end cartouche
-
-The @command{ib-verbs} server module accepts the following options:
-
-@cartouche
-@table @code
-@item bind-address <address> (0.0.0.0)
-The local interface on which the server should listen to requests. Default is to
-listen on all interfaces.
-@item listen-port <n> (24007)
-The local port to listen on.
-@end table
-@end cartouche
-
-The following options are common to both the client and server modules:
-
-If you are familiar with InfiniBand jargon,
-the mode is used by GlusterFS is ``reliable connection-oriented channel transfer''.
-
-@cartouche
-@table @code
-@item ib-verbs-work-request-send-count <n> (64)
-Length of the send queue in datagrams. [Reason to increase/decrease?]
-
-@item ib-verbs-work-request-recv-count <n> (64)
-Length of the receive queue in datagrams. [Reason to increase/decrease?]
-
-@item ib-verbs-work-request-send-size <size> (128KB)
-Size of each datagram that is sent. [Reason to increase/decrease?]
-
-@item ib-verbs-work-request-recv-size <size> (128KB)
-Size of each datagram that is received. [Reason to increase/decrease?]
-
-@item ib-verbs-port <n> (1)
-Port number for ib-verbs.
-
-@item ib-verbs-mtu [256|512|1024|2048|4096] (2048)
-The Maximum Transmission Unit [Reason to increase/decrease?]
-
-@item ib-verbs-device-name <device-name> (first device in the list)
-InfiniBand device to be used.
-@end table
-@end cartouche
-
-For maximum performance, you should ensure that the send/receive counts on both
-the client and server are the same.
-
-ib-verbs is preferred over ib-sdp.
-
-@node Client protocol
-@subsection Client
-@example
-type procotol/client
-@end example
-
-The client translator enables the GlusterFS client to access a remote server's
-translator tree.
-
-@cartouche
-@table @code
-
-@item transport-type [tcp,ib-sdp,ib-verbs] (tcp)
-The transport type to use. You should use the client versions of all the
-transport modules (@command{tcp}, @command{ib-sdp}, 
-@command{ib-verbs}).
-@item remote-subvolume <volume_name> *
-The name of the volume on the remote host to attach to. Note that
-this is @emph{not} the name of the @command{protocol/server} volume on the
-server. It should be any volume under the server.
-@item transport-timeout <n> (120- seconds)
-Inactivity timeout. If a reply is expected and no activity takes place
-on the connection within this time, the transport connection will be
-broken, and a new connection will be attempted.
-@end table
-@end cartouche
-
-@node Server protocol
-@subsection Server
-@example
-type protocol/server
-@end example
-
-The server translator exports a translator tree and makes it accessible to
-remote GlusterFS clients.
-
-@cartouche
-@table @code
-@item client-volume-filename <path> (<CONFDIR>/glusterfs-client.vol)
-The volume specification file to use for the client. This is the file the
-client will receive when it is invoked with the @command{--server} option 
-(@ref{Client}).
-
-@item transport-type [tcp,ib-verbs,ib-sdp] (tcp)
-The transport to use. You should use the server versions of all the transport
-modules (@command{tcp}, @command{ib-sdp}, @command{ib-verbs}).
-
-@item auth.addr.<volume name>.allow <IP address wildcard pattern>
-IP addresses of the clients that are allowed to attach to the specified volume.
-This can be a wildcard. For example, a wildcard of the form @command{192.168.*.*}
-allows any host in the @command{192.168.x.x} subnet to connect to the server.
-
-@end table
-@end cartouche
-
-@node Clustering Translators
-@section Clustering Translators
-
-The clustering translators are the most important GlusterFS
-translators, since it is these that make GlusterFS a cluster
-filesystem. These translators together enable GlusterFS to access an
-arbitrarily large amount of storage, and provide @acronym{RAID}-like
-redundancy and distribution over the entire cluster.
-
-There are three clustering translators: @strong{unify}, @strong{replicate},
-and @strong{stripe}.  The unify translator aggregates storage from
-many server nodes. The replicate translator provides file replication. The stripe
-translator allows a file to be spread across many server nodes. The following sections
-look at each of these translators in detail.
-
-@menu
-* Unify::                       
-* Replicate::  
-* Stripe::                      
-@end menu
-
-@node Unify
-@subsection Unify
-@cindex unify (translator)
-@cindex scheduler (unify)
-@example
-type cluster/unify
-@end example
-
-The unify translator presents a `unified' view of all its sub-volumes. That is,
-it makes the union of all its sub-volumes appear as a single volume. It is the
-unify translator that gives GlusterFS the ability to access an arbitrarily 
-large amount of storage.
-
-For unify to work correctly, certain invariants need to be maintained across
-the entire network. These are:
-
-@cindex unify invariants
-@itemize
-@item The directory structure of all the sub-volumes must be identical.
-@item A particular file can exist on only one of the sub-volumes. Phrasing it in another way, a pathname such as @command{/home/calvin/homework.txt}) is unique across the entire cluster.
-@end itemize
-
-@tex
-\vfill
-@end tex
-@page
-
-@center @image{unify,44pc,,,.pdf}
-
-Looking at the second requirement, you might wonder how one can
-accomplish storing redundant copies of a file, if no file can exist
-multiple times.  To answer, we must remember that these invariants are
-from @emph{unify's perspective}.  A translator such as replicate at a lower
-level in the translator tree than unify may subvert this picture. 
-
-The first invariant might seem quite tedious to ensure. We shall see
-later that this is not so, since unify's @emph{self-heal} mechanism
-takes care of maintaining it.
-
-The second invariant implies that unify needs some way to decide which file goes where.
-Unify makes use of @emph{scheduler} modules for this purpose.
-
-When a file needs to be created, unify's scheduler decides upon the
-sub-volume to be used to store the file. There are many schedulers
-available, each using a different algorithm and suitable for different
-purposes.
-
-The various schedulers are described in detail in the sections that follow.
-
-@subsubsection ALU
-@cindex alu (scheduler)
-
-@example
-  option scheduler alu
-@end example
-
-ALU stands for "Adaptive Least Usage". It is the most advanced
-scheduler available in GlusterFS. It balances the load across volumes
-taking several factors in account. It adapts itself to changing I/O
-patterns according to its configuration. When properly configured, it
-can eliminate the need for regular tuning of the filesystem to keep
-volume load nicely balanced.
-
-The ALU scheduler is composed of multiple least-usage
-sub-schedulers. Each sub-scheduler keeps track of a certain type of
-load, for each of the sub-volumes, getting statistics from
-the sub-volumes themselves. The sub-schedulers are these:
-
-@itemize
-@item disk-usage: The used and free disk space on the volume.
-
-@item read-usage: The amount of reading done from this volume.
-
-@item write-usage: The amount of writing done to this volume.
-
-@item open-files-usage: The number of files currently open from this volume.
-
-@item disk-speed-usage: The speed at which the disks are spinning. This is a constant value and therefore not very useful.
-@end itemize
-
-The ALU scheduler needs to know which of these sub-schedulers to use,
-and in which order to evaluate them. This is done through the
-@command{option alu.order} configuration directive.
-
-Each sub-scheduler needs to know two things: when to kick in (the
-entry-threshold), and how long to stay in control (the
-exit-threshold). For example: when unifying three disks of 100GB,
-keeping an exact balance of disk-usage is not necesary. Instead, there
-could be a 1GB margin, which can be used to nicely balance other
-factors, such as read-usage. The disk-usage scheduler can be told to
-kick in only when a certain threshold of discrepancy is passed, such
-as 1GB. When it assumes control under this condition, it will write
-all subsequent data to the least-used volume. If it is doing so, it is
-unwise to stop right after the values are below the entry-threshold
-again, since that would make it very likely that the situation will
-occur again very soon. Such a situation would cause the ALU to spend
-most of its time disk-usage scheduling, which is unfair to the other
-sub-schedulers. The exit-threshold therefore defines the amount of
-data that needs to be written to the least-used disk, before control
-is relinquished again.
-
-In addition to the sub-schedulers, the ALU scheduler also has "limits"
-options. These can stop the creation of new files on a volume once
-values drop below a certain threshold. For example, setting
-@command{option alu.limits.min-free-disk 5GB} will stop the scheduling
-of files to volumes that have less than 5GB of free disk space,
-leaving the files on that disk some room to grow.
-
-The actual values you assign to the thresholds for sub-schedulers and
-limits depend on your situation. If you have fast-growing files,
-you'll want to stop file-creation on a disk much earlier than when
-hardly any of your files are growing. If you care less about
-disk-usage balance than about read-usage balance, you'll want a bigger
-disk-usage scheduler entry-threshold and a smaller read-usage
-scheduler entry-threshold.
-
-For thresholds defining a size, values specifying "KB", "MB" and "GB"
-are allowed. For example: @command{option alu.limits.min-free-disk 5GB}.
-
-@cartouche
-@table @code
-@item alu.order <order> * ("disk-usage:write-usage:read-usage:open-files-usage:disk-speed")
-@item alu.disk-usage.entry-threshold <size> (1GB)
-@item alu.disk-usage.exit-threshold <size> (512MB)
-@item alu.write-usage.entry-threshold <%> (25)
-@item alu.write-usage.exit-threshold <%> (5)
-@item alu.read-usage.entry-threshold <%> (25)
-@item alu.read-usage.exit-threshold <%> (5)
-@item alu.open-files-usage.entry-threshold <n> (1000)
-@item alu.open-files-usage.exit-threshold <n> (100)
-@item alu.limits.min-free-disk <%> 
-@item alu.limits.max-open-files <n> 
-@end table
-@end cartouche
-
-@subsubsection Round Robin (RR)
-@cindex rr (scheduler)
-
-@example
-  option scheduler rr
-@end example
-
-Round-Robin (RR) scheduler creates files in a round-robin
-fashion. Each client will have its own round-robin loop. When your
-files are mostly similar in size and I/O access pattern, this
-scheduler is a good choice. RR scheduler checks for free disk space
-on the server before scheduling, so you can know when to add
-another server node. The default value of min-free-disk is 5% and is
-checked on file creation calls, with atleast 10 seconds (by default) 
-elapsing between two checks.
-
-Options:
-@cartouche
-@table @code
-@item rr.limits.min-free-disk <%> (5)
-Minimum free disk space a node must have for RR to schedule a file to it.
-@item rr.refresh-interval <t> (10 seconds)
-Time between two successive free disk space checks.
-@end table
-@end cartouche
-
-@subsubsection Random
-@cindex random (scheduler)
-
-@example
-  option scheduler random
-@end example
-
-The random scheduler schedules file creation randomly among its child nodes.
-Like the round-robin scheduler, it also checks for a minimum amount of free disk
-space before scheduling a file to a node.
-
-@cartouche
-@table @code
-@item random.limits.min-free-disk <%> (5)
-Minimum free disk space a node must have for random to schedule a file to it.
-@item random.refresh-interval <t> (10 seconds)
-Time between two successive free disk space checks.
-@end table
-@end cartouche
-
-@subsubsection NUFA
-@cindex nufa (scheduler)
-
-@example
-  option scheduler nufa
-@end example
-
-It is common in many GlusterFS computing environments for all deployed
-machines to act as both servers and clients. For example, a
-research lab may have 40 workstations each with its own storage. All
-of these workstations might act as servers exporting a volume as well
-as clients accessing the entire cluster's storage.  In such a
-situation, it makes sense to store locally created files on the local
-workstation itself (assuming files are accessed most by the
-workstation that created them). The Non-Uniform File Allocation (@acronym{NUFA})
-scheduler accomplishes that.
-
-@acronym{NUFA} gives the local system first priority for file creation
-over other nodes. If the local volume does not have more free disk space
-than a specified amount (5% by default) then @acronym{NUFA} schedules files
-among the other child volumes in a round-robin fashion.
-
-@acronym{NUFA} is named after the similar strategy used for memory access,
-@acronym{NUMA}@footnote{Non-Uniform Memory Access: 
-@indicateurl{http://en.wikipedia.org/wiki/Non-Uniform_Memory_Access}}.
-
-@cartouche
-@table @code
-@item nufa.limits.min-free-disk <%> (5)
-Minimum disk space that must be free (local or remote) for @acronym{NUFA} to schedule a
-file to it.
-@item nufa.refresh-interval <t> (10 seconds)
-Time between two successive free disk space checks.
-@item nufa.local-volume-name <volume> 
-The name of the volume corresponding to the local system. This volume must be
-one of the children of the unify volume. This option is mandatory.
-@end table
-@end cartouche
-
-@cindex namespace
-@subsubsection Namespace
-Namespace volume needed because:
- - persistent inode numbers.
- - file exists even when node is down.
-
-namespace files are simply touched. on every lookup it is checked.
-
-@cartouche
-@table @code
-@item namespace <volume> *
-Name of the namespace volume (which should be one of the unify volume's children).
-@item self-heal [on|off] (on)
-Enable/disable self-heal. Unless you know what you are doing, do not disable self-heal.
-@end table
-@end cartouche
-
-@cindex self heal (unify)
-@subsubsection Self Heal
-    * When a 'lookup()/stat()' call is made on directory for the first
-time, a self-heal call is made, which checks for the consistancy of
-its child nodes. If an entry is present in storage node, but not in
-namespace, that entry is created in namespace, and vica-versa. There
-is an writedir() API introduced which is used for the same. It also
-checks for permissions, and uid/gid consistencies.
-
-    * This check is also done when an server goes down and comes up.
-
-    * If one starts with an empty namespace export, but has data in
-storage nodes, a 'find .>/dev/null' or 'ls -lR >/dev/null' should help
-to build namespace in one shot. Even otherwise, namespace is built on
-demand when a file is looked up for the first time.
-
-NOTE: There are some issues (Kernel 'Oops' msgs) seen with fuse-2.6.3,
-when someone deletes namespace in backend, when glusterfs is
-running. But with fuse-2.6.5, this issue is not there.
-
-@node Replicate
-@subsection Replicate (formerly AFR)
-@cindex Replicate
-@example
-type cluster/replicate
-@end example
-
-Replicate provides @acronym{RAID}-1 like functionality for
-GlusterFS. Replicate replicates files and directories across the
-subvolumes. Hence if Replicate has four subvolumes, there will be
-four copies of all files and directories. Replicate provides
-high-availability, i.e., in case one of the subvolumes go down
-(e. g. server crash, network disconnection) Replicate will still
-service the requests using the redundant copies.
-
-Replicate also provides self-heal functionality, i.e., in case the
-crashed servers come up, the outdated files and directories will be
-updated with the latest versions. Replicate uses extended
-attributes of the backend file system to track the versioning of files
-and directories and provide the self-heal feature.
-
-@example
-volume replicate-example
- type cluster/replicate
- subvolumes brick1 brick2 brick3
-end-volume
-@end example
-
-This sample configuration will replicate all directories and files on
-brick1, brick2 and brick3. 
-
-All the read operations happen from the first alive child. If all the
-three sub-volumes are up, reads will be done from brick1; if brick1 is
-down read will be done from brick2. In case read() was being done on
-brick1 and it goes down, replicate transparently falls back to
-brick2. 
-
-The next release of GlusterFS will add the following features:
-@itemize
-@item Ability to specify the sub-volume from which read operations are to be done (this will help users who have one of the sub-volumes as a local storage volume).
-@item Allow scheduling of read operations amongst the sub-volumes in a round-robin fashion.
-@end itemize
-
-The order of the subvolumes list should be same across all the 'replicate's as
-they will be used for locking purposes.
-
-@cindex self heal (replicate)
-@subsubsection Self Heal
-Replicate has self-heal feature, which updates the outdated file and
-directory copies by the most recent versions. For example consider the
-following config:
-
-@example
-volume replicate-example
- type cluster/replicate
- subvolumes brick1 brick2
-end-volume
-@end example
-
-@subsubsection File self-heal
-
-Now if we create a file foo.txt on replicate-example, the file will be created
-on brick1 and brick2. The file will have two extended attributes associated
-with it in the backend filesystem. One is trusted.afr.createtime and the
-other is trusted.afr.version. The trusted.afr.createtime xattr has the
-create time (in terms of seconds since epoch) and trusted.afr.version
-is a number that is incremented each time a file is modified. This increment
-happens during close (incase any write was done before close).
-
-If brick1 goes down, we edit foo.txt the version gets incremented. Now
-the brick1 comes back up, when we open() on foo.txt replicate will check if
-their versions are same. If they are not same, the outdated copy is
-replaced by the latest copy and its version is updated. After the sync
-the open() proceeds in the usual manner and the application calling open()
-can continue on its access to the file.
-
-If brick1 goes down, we delete foo.txt and create a file with the same
-name again i.e foo.txt. Now brick1 comes back up, clearly there is a
-chance that the version on brick1 being more than the version on brick2,
-this is where createtime extended attribute helps in deciding which
-the outdated copy is. Hence we need to consider both createtime and
-version to decide on the latest copy.
-
-The version attribute is incremented during the close() call. Version
-will not be incremented in case there was no write() done. In case the
-fd that the close() gets was got by create() call, we also create
-the createtime extended attribute.
-
-@subsubsection Directory self-heal
-
-Suppose brick1 goes down, we delete foo.txt, brick1 comes back up, now
-we should not create foo.txt on brick2 but we should delete foo.txt
-on brick1. We handle this situation by having the createtime and version
-attribute on the directory similar to the file. when lookup() is done
-on the directory, we compare the createtime/version attributes of the
-copies and see which files needs to be deleted and delete those files
-and update the extended attributes of the outdated directory copy.
-Each time a directory is modified (a file or a subdirectory is created
-or deleted inside the directory) and one of the subvols is down, we
-increment the directory's version.
-
-lookup() is a call initiated by the kernel on a file or directory
-just before any access to that file or directory. In glusterfs, by
-default, lookup() will not be called in case it was called in the
-past one second on that particular file or directory.
-
-The extended attributes can be seen in the backend filesystem using
-the @command{getfattr} command. (@command{getfattr -n trusted.afr.version <file>})
-
-@cartouche
-@table @code
-@item debug [on|off]  (off)
-@item self-heal [on|off] (on)
-@item replicate <pattern> (*:1)
-@item lock-node <child_volume> (first child is used by default)
-@end table
-@end cartouche
-
-@node Stripe
-@subsection Stripe
-@cindex stripe (translator)
-@example
-type cluster/stripe
-@end example
-
-The stripe translator distributes the contents of a file over its
-sub-volumes.  It does this by creating a file equal in size to the
-total size of the file on each of its sub-volumes. It then writes only
-a part of the file to each sub-volume, leaving the rest of it empty. 
-These empty regions are called `holes' in Unix terminology. The holes
-do not consume any disk space.
-
-The diagram below makes this clear.
-
-@center @image{stripe,44pc,,,.pdf}
-
-You can configure stripe so that only filenames matching a pattern 
-are striped. You can also configure the size of the data to be stored 
-on each sub-volume.
-
-@cartouche
-@table @code
-@item block-size <pattern>:<size>  (*:0 no striping)
-Distribute files matching @command{<pattern>} over the sub-volumes, 
-storing at least @command{<size>} on each sub-volume. For example,
-
-@example
-  option block-size *.mpg:1M
-@end example
-
-distributes all files ending in @command{.mpg}, storing at least 1 MB on
-each sub-volume.
-
-Any number of @command{block-size} option lines may be present, specifying
-different sizes for different file name patterns.
-@end table
-@end cartouche
-
-@node Performance Translators
-@section Performance Translators
-
-@menu
-* Read Ahead::                  
-* Write Behind::                
-* IO Threads::                  
-* IO Cache::
-* Booster::
-@end menu
-
-@node Read Ahead
-@subsection Read Ahead
-@cindex read-ahead (translator)
-@example
-type performance/read-ahead
-@end example
-
-The read-ahead translator pre-fetches data in advance on every read.
-This benefits applications that mostly process files in sequential order,
-since the next block of data will already be available by the time the
-application is done with the current one. 
-
-Additionally, the read-ahead translator also behaves as a read-aggregator. 
-Many small read operations are combined and issued as fewer, larger read
-requests to the server.
-
-Read-ahead deals in ``pages'' as the unit of data fetched. The page size
-is configurable, as is the ``page count'', which is the number of pages
-that are pre-fetched.
-
-Read-ahead is best used with InfiniBand (using the ib-verbs transport). 
-On FastEthernet and Gigabit Ethernet networks,
-GlusterFS can achieve the link-maximum throughput even without
-read-ahead, making it quite superflous.
-
-Note that read-ahead only happens if the reads are perfectly
-sequential. If your application accesses data in a random fashion,
-using read-ahead might actually lead to a performance loss, since
-read-ahead will pointlessly fetch pages which won't be used by the
-application.
-
-@cartouche
-Options:
-@table @code
-@item page-size <n> (256KB)
-The unit of data that is pre-fetched.
-@item page-count <n> (2)
-The number of pages that are pre-fetched.
-@item force-atime-update [on|off|yes|no] (off|no)
-Whether to force an access time (atime) update on the file on every read. Without
-this, the atime will be slightly imprecise, as it will reflect the time when 
-the read-ahead translator read the data, not when the application actually read it.
-@end table
-@end cartouche
-
-@node Write Behind
-@subsection Write Behind
-@cindex write-behind (translator)
-@example
-type performance/write-behind
-@end example
-
-The write-behind translator improves the latency of a write operation.
-It does this by relegating the write operation to the background and
-returning to the application even as the write is in progress. Using the
-write-behind translator, successive write requests can be pipelined.
-This mode of write-behind operation is best used on the client side, to
-enable decreased write latency for the application.
-
-The write-behind translator can also aggregate write requests. If the 
-@command{aggregate-size} option is specified, then successive writes upto that
-size are accumulated and written in a single operation. This mode of operation
-is best used on the server side, as this will decrease the disk's head movement
-when multiple files are being written to in parallel.
-
-The @command{aggregate-size} option has a default value of 128KB. Although
-this works well for most users, you should always experiment with different values
-to determine the one that will deliver maximum performance. This is because the
-performance of write-behind depends on your interconnect, size of RAM, and the
-work load.
-
-@cartouche
-@table @code
-@item aggregate-size <n> (128KB)
-Amount of data to accumulate before doing a write
-@item flush-behind [on|yes|off|no] (off|no)
-
-@end table
-@end cartouche
-
-@node IO Threads
-@subsection IO Threads
-@cindex io-threads (translator)
-@example
-type performance/io-threads
-@end example
-
-The IO threads translator is intended to increase the responsiveness
-of the server to metadata operations by doing file I/O (read, write)
-in a background thread.  Since the GlusterFS server is
-single-threaded, using the IO threads translator can significantly
-improve performance. This translator is best used on the server side,
-loaded just below the server protocol translator.
-
-IO threads operates by handing out read and write requests to a separate thread.
-The total number of threads in existence at a time is constant, and configurable.
-
-@cartouche
-@table @code
-@item thread-count <n> (1)
-Number of threads to use.
-@end table
-@end cartouche
-
-@node IO Cache
-@subsection IO Cache
-@cindex io-cache (translator)
-@example
-type performance/io-cache
-@end example
-
-The IO cache translator caches data that has been read. This is useful
-if many applications read the same data multiple times, and if reads
-are much more frequent than writes (for example, IO caching may be
-useful in a web hosting environment, where most clients will simply
-read some files and only a few will write to them).
-
-The IO cache translator reads data from its child in @command{page-size} chunks.
-It caches data upto @command{cache-size} bytes. The cache is maintained as
-a prioritized least-recently-used (@acronym{LRU}) list, with priorities determined
-by user-specified patterns to match filenames.
-
-When the IO cache translator detects a write operation, the 
-cache for that file is flushed.
-
-The IO cache translator periodically verifies the consistency of
-cached data, using the modification times on the files. The verification timeout
-is configurable.
-
-@cartouche
-@table @code
-@item page-size <n> (128KB)
-Size of a page.
-@item cache-size (n) (32MB)
-Total amount of data to be cached.
-@item force-revalidate-timeout <n> (1)
-Timeout to force a cache consistency verification, in seconds.
-@item priority <pattern> (*:0)
-Filename patterns listed in order of priority.
-@end table
-@end cartouche
-
-@node Booster
-@subsection Booster
-@cindex booster
-@example
-  type performance/booster
-@end example
-
-The booster translator gives applications a faster path to communicate
-read and write requests to GlusterFS. Normally, all requests to GlusterFS from
-applications go through FUSE, as indicated in @ref{Filesystems in Userspace}.
-Using the booster translator in conjunction with the GlusterFS booster shared
-library, an application can bypass the FUSE path and send read/write requests
-directly to the GlusterFS client process.
-
-The booster mechanism consists of two parts: the booster translator,
-and the booster shared library. The booster translator is meant to be
-loaded on the client side, usually at the root of the translator tree.
-The booster shared library should be @command{LD_PRELOAD}ed with the
-application.
-
-The booster translator when loaded opens a Unix domain socket and
-listens for read/write requests on it. The booster shared library
-intercepts read and write system calls and sends the requests to the
-GlusterFS process directly using the Unix domain socket, bypassing FUSE.
-This leads to superior performance.
-
-Once you've loaded the booster translator in your volume specification file, you
-can start your application as:
-
-@example
-  $ LD_PRELOAD=/usr/local/bin/glusterfs-booster.so your_app
-@end example
-
-The booster translator accepts no options.
-
-@node Features Translators
-@section Features Translators 
-
-@menu
-* POSIX Locks::                 
-* Fixed ID::                    
-@end menu
-
-@node POSIX Locks
-@subsection POSIX Locks
-@cindex record locking
-@cindex fcntl
-@cindex posix-locks (translator)
-@example
-type features/posix-locks
-@end example
-
-This translator provides storage independent POSIX record locking
-support (@command{fcntl} locking). Typically you'll want to load this on the
-server side, just above the @acronym{POSIX} storage translator. Using this
-translator you can get both advisory locking and mandatory locking
-support.  It also handles @command{flock()} locks properly.
-
-Caveat: Consider a file that does not have its mandatory locking bits
-(+setgid, -group execution) turned on. Assume that this file is now
-opened by a process on a client that has the write-behind xlator
-loaded. The write-behind xlator does not cache anything for files
-which have mandatory locking enabled, to avoid incoherence. Let's say
-that mandatory locking is now enabled on this file through another
-client. The former client will not know about this change, and
-write-behind may erroneously report a write as being successful when
-in fact it would fail due to the region it is writing to being locked.
-
-There seems to be no easy way to fix this. To work around this
-problem, it is recommended that you never enable the mandatory bits on
-a file while it is open.
-
-@cartouche
-@table @code
-@item mandatory [on|off] (on)
-Turns mandatory locking on.
-@end table
-@end cartouche
-
-@node Fixed ID
-@subsection Fixed ID
-@cindex fixed-id (translator)
-@example
-type features/fixed-id
-@end example
-
-The fixed ID translator makes all filesystem requests from the client
-to appear to be coming from a fixed, specified
-@acronym{UID}/@acronym{GID}, regardless of which user actually
-initiated the request.
-
-@cartouche
-@table @code
-@item fixed-uid <n> [if not set, not used]
-The @acronym{UID} to send to the server
-@item fixed-gid <n> [if not set, not used]
-The @acronym{GID} to send to the server
-@end table
-@end cartouche
-
-@node Miscellaneous Translators
-@section Miscellaneous Translators
-
-@menu
-* ROT-13::                      
-* Trace::                       
-@end menu
-
-@node ROT-13
-@subsection ROT-13
-@cindex rot-13 (translator)
-@example
-type encryption/rot-13
-@end example
-
-@acronym{ROT-13} is a toy translator that can ``encrypt'' and ``decrypt'' file
-contents using the @acronym{ROT-13} algorithm. @acronym{ROT-13} is a trivial
-algorithm that rotates each alphabet by thirteen places. Thus, 'A' becomes 'N',
-'B' becomes 'O', and 'Z' becomes 'M'.
-
-It goes without saying that you shouldn't use this translator if you need 
-@emph{real} encryption (a future release of GlusterFS will have real encryption
-translators).
-
-@cartouche
-@table @code
-@item encrypt-write [on|off] (on)
-Whether to encrypt on write
-@item decrypt-read [on|off] (on)
-Whether to decrypt on read
-@end table
-@end cartouche
-
-@node Trace
-@subsection Trace
-@cindex trace (translator)
-@example
-type debug/trace     
-@end example
-
-The trace translator is intended for debugging purposes. When loaded, it
-logs all the system calls received by the server or client (wherever
-trace is loaded), their arguments, and the results. You must use a GlusterFS log
-level of DEBUG (See @ref{Running GlusterFS}) for trace to work.
-
-Sample trace output (lines have been wrapped for readability):
-@cartouche
-@example
-2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68 
-(*this=0x8059e40, loc=0x8091984 @{path=/iozone3_283, inode=0x8091f00@}, 
- fd=0x8091d50)
-
-2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace: 
-(*this=0x8059e40, op_ret=4, op_errno=1, fd=0x8091d50)
-
-2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69 
-(*this=0x8059e40, size=4096, offset=0 fd=0x8091d50)
-
-2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace: 
-(*this=0x8059e40, op_ret=0, op_errno=0, count=4)
-
-2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71 
-(*this=0x8059e40, *fd=0x8091d50)
-
-2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace: 
-(*this=0x8059e40, op_ret=0, op_errno=1)
-@end example
-@end cartouche
-
-@node Usage Scenarios
-@chapter Usage Scenarios
-
-@section Advanced Striping
-
-This section is based on the Advanced Striping tutorial written by
-Anand Avati on the GlusterFS wiki
-@footnote{http://gluster.org/docs/index.php/Mixing_Striped_and_Regular_Files}.
-
-@subsection Mixed Storage Requirements
-
-There are two ways of scheduling the I/O. One at file level (using
-unify translator) and other at block level (using stripe
-translator). Striped I/O is good for files that are potentially large
-and require high parallel throughput (for example, a single file of
-400GB being accessed by 100s and 1000s of systems simultaneously and
-randomly). For most of the cases, file level scheduling works best.
-
-In the real world, it is desirable to mix file level and block level
-scheduling on a single storage volume. Alternatively users can choose
-to have two separate volumes and hence two mount points, but the
-applications may demand a single storage system to host both.
-
-This document explains how to mix file level scheduling with stripe. 
-
-@subsection Configuration Brief
-
-This setup demonstrates how users can configure unify translator with
-appropriate I/O scheduler for file level scheduling and strip for only
-matching patterns. This way, GlusterFS chooses appropriate I/O profile
-and knows how to efficiently handle both the types of data.
-
-A simple technique to achieve this effect is to create a stripe set of
-unify and stripe blocks, where unify is the first sub-volume. Files
-that do not match the stripe policy passed on to first unify
-sub-volume and inturn scheduled arcoss the cluster using its file
-level I/O scheduler.
-
-@image{advanced-stripe,44pc,,,.pdf}
-
-@subsection Preparing GlusterFS Envoronment
-
-Create the directories /export/namespace, /export/unify and
-/export/stripe on all the storage bricks.
-
- Place the following server and client volume spec file under
-/etc/glusterfs (or appropriate installed path) and replace the IP
-addresses / access control fields to match your environment.
-
-@cartouche
-@example
-  ## file: /etc/glusterfs/glusterfsd.vol
-   volume posix-unify
-           type storage/posix
-           option directory /export/for-unify
-   end-volume
- 
-   volume posix-stripe
-           type storage/posix
-           option directory /export/for-stripe
-   end-volume
- 
-   volume posix-namespace
-           type storage/posix
-           option directory /export/for-namespace
-   end-volume
-  
-   volume server
-           type protocol/server
-           option transport-type tcp
-           option auth.addr.posix-unify.allow 192.168.1.*
-           option auth.addr.posix-stripe.allow 192.168.1.*
-           option auth.addr.posix-namespace.allow 192.168.1.*
-           subvolumes posix-unify posix-stripe posix-namespace
-   end-volume
-@end example
-@end cartouche
-
-@cartouche
-@example
- ## file: /etc/glusterfs/glusterfs.vol
-   volume client-namespace
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.1
-     option remote-subvolume posix-namespace
-   end-volume
-
-   volume client-unify-1
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.1
-     option remote-subvolume posix-unify
-   end-volume
-
-   volume client-unify-2
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.2
-     option remote-subvolume posix-unify
-   end-volume
-
-   volume client-unify-3
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.3
-     option remote-subvolume posix-unify
-   end-volume
-
-   volume client-unify-4
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.4
-     option remote-subvolume posix-unify
-   end-volume
- 
-   volume client-stripe-1
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.1
-     option remote-subvolume posix-stripe
-   end-volume
-
-   volume client-stripe-2
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.2
-     option remote-subvolume posix-stripe
-   end-volume
-
-   volume client-stripe-3
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.3
-     option remote-subvolume posix-stripe
-   end-volume
-
-   volume client-stripe-4
-     type protocol/client
-     option transport-type tcp
-     option remote-host 192.168.1.4
-     option remote-subvolume posix-stripe
-   end-volume
-  
-   volume unify
-     type cluster/unify
-     option scheduler rr
-     subvolumes cluster-unify-1 cluster-unify-2 cluster-unify-3 cluster-unify-4
-   end-volume
- 
-   volume stripe
-     type cluster/stripe
-     option block-size *.img:2MB # All files ending with .img are striped with 2MB stripe block size.
-     subvolumes unify cluster-stripe-1 cluster-stripe-2 cluster-stripe-3 cluster-stripe-4
-   end-volume
-@end example
-@end cartouche
-
-
-Bring up the Storage
-
-Starting GlusterFS Server: If you have installed through binary
-package, you can start the service through init.d startup script. If
-not:
-
-@example
-[root@@server]# glusterfsd
-@end example
-
-Mounting GlusterFS Volumes:
-
-@example
-[root@@client]# glusterfs -s [BRICK-IP-ADDRESS] /mnt/cluster
-@end example
-
-Improving upon this Setup
-
-Infiniband Verbs RDMA transport is much faster than TCP/IP GigE
-transport.
-
-Use of performance translators such as read-ahead, write-behind,
-io-cache, io-threads, booster is recommended.
-
-Replace round-robin (rr) scheduler with ALU to handle more dynamic
-storage environments.
-
-@node Troubleshooting
-@chapter Troubleshooting
-
-This chapter is a general troubleshooting guide to GlusterFS. It lists
-common GlusterFS server and client error messages, debugging hints, and
-concludes with the suggested procedure to report bugs in GlusterFS.
-
-@section GlusterFS error messages
-
-@subsection Server errors
-
-@example
-glusterfsd: FATAL: could not open specfile: 
-'/etc/glusterfs/glusterfsd.vol'
-@end example
-
-The GlusterFS server expects the volume specification file to be 
-at @command{/etc/glusterfs/glusterfsd.vol}. The example
-specification file will be installed as 
-@command{/etc/glusterfs/glusterfsd.vol.sample}. You need to edit
-it and rename it, or provide a different specification file using
-the @command{--spec-file} command line option (See @ref{Server}).
-
-@vskip 4ex
-
-@example
-gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log" 
-             (Permission denied)
-@end example
-
-You don't have permission to create files in the
-@command{/usr/var/log/glusterfs} directory. Make sure you are running
-GlusterFS as root. Alternatively, specify a different path for the log
-file using the @command{--log-file} option (See @ref{Server}).
-
-@subsection Client errors
-
-@example
-fusermount: failed to access mountpoint /mnt: 
-            Transport endpoint is not connected
-@end example
-
-A previous failed (or hung) mount of GlusterFS is preventing it from being
-mounted again in the same location. The fix is to do:
-
-@example
-# umount /mnt
-@end example
-
-and try mounting again.
-
-@vskip 4ex
-
-@strong{``Transport endpoint is not connected''.}
-
-If you get this error when you try a command such as @command{ls} or @command{cat},
-it means the GlusterFS mount did not succeed. Try running GlusterFS in @command{DEBUG}
-logging level and study the log messages to discover the cause.
-
-@vskip 4ex
-
-@strong{``Connect to server failed'', ``SERVER-ADDRESS: Connection refused''.}
-
-GluserFS Server is not running or dead. Check your network
-connections and firewall settings. To check if the server is reachable,
-try:
-
-@example
-telnet IP-ADDRESS 24007
-@end example
-
-If the server is accessible, your `telnet' command should connect and
-block. If not you will see an error message such as @command{telnet: Unable to
-connect to remote host: Connection refused}. 24007 is the default
-GlusterFS port. If you have changed it, then use the corresponding
-port instead.
-
-@vskip 4ex
-
-@example
-gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log" 
-             (Permission denied)
-@end example
-
-You don't have permission to create files in the
-@command{/usr/var/log/glusterfs} directory. Make sure you are running
-GlusterFS as root. Alternatively, specify a different path for the log
-file using the @command{--log-file} option (See @ref{Client}).
-
-@section FUSE error messages
-@command{modprobe fuse} fails with: ``Unknown symbol in module, or unknown parameter''.
-@cindex Redhat Enterprise Linux
-
-If you are using fuse-2.6.x on Redhat Enterprise Linux Work Station 4
-and Advanced Server 4 with 2.6.9-42.ELlargesmp, 2.6.9-42.ELsmp,
-2.6.9-42.EL kernels and get this error while loading @acronym{FUSE} kernel
-module, you need to apply the following patch.
-
-For fuse-2.6.2:
-
-@indicateurl{http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.2-rhel-build.patch}
-
-For fuse-2.6.3:
-
-@indicateurl{http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.3-rhel-build.patch}
-
-@section AppArmour and GlusterFS
-@cindex AppArmour
-@cindex OpenSuSE
-Under OpenSuSE GNU/Linux, the AppArmour security feature does not
-allow GlusterFS to create temporary files or network socket
-connections even while running as root. You will see error messages
-like `Unable to open log file: Operation not permitted' or `Connection
-refused'. Disabling AppArmour using YaST or properly configuring
-AppArmour to recognize @command{glusterfsd} or @command{glusterfs}/@command{fusermount}
-should solve the problem.
-
-@section Reporting a bug
-
-If you encounter a bug in GlusterFS, please follow the below
-guidelines when you report it to the mailing list. Be sure to report
-it! User feedback is crucial to the health of the project and we value
-it highly.
-
-@subsection General instructions
-
-When running GlusterFS in a non-production environment, be sure to
-build it with the following command:
-
-@example
- $ make CFLAGS='-g -O0 -DDEBUG'
-@end example
-
-This includes debugging information which will be helpful in getting
-backtraces (see below) and also disable optimization. Enabling
-optimization can result in incorrect line numbers being reported to
-gdb.
-
-@subsection Volume specification files
-
-Attach all relevant server and client spec files you were using when
-you encountered the bug. Also tell us details of your setup, i.e., how
-many clients and how many servers.
-
-@subsection Log files
-
-Set the loglevel of your client and server programs to @acronym{DEBUG} (by
-passing the -L @acronym{DEBUG} option) and attach the log files with your bug
-report. Obviously, if only the client is failing (for example), you
-only need to send us the client log file.
-
-@subsection Backtrace
-
-If GlusterFS has encountered a segmentation fault or has crashed for
-some other reason, include the backtrace with the bug report. You can
-get the backtrace using the following procedure.
-
-Run the GlusterFS client or server inside gdb.
-
-@example
- $ gdb ./glusterfs
- (gdb) set args -f client.spec -N -l/path/to/log/file -LDEBUG /mnt/point
- (gdb) run
-@end example
-
-Now when the process segfaults, you can get the backtrace by typing:
-
-@example
- (gdb) bt
-@end example
-
-If the GlusterFS process has crashed and dumped a core file (you can
-find this in / if running as a daemon and in the current directory
-otherwise), you can do:
-
-@example
- $ gdb /path/to/glusterfs /path/to/core.<pid>
-@end example
-
-and then get the backtrace.
-
-If the GlusterFS server or client seems to be hung, then you can get
-the backtrace by attaching gdb to the process. First get the @command{PID} of
-the process (using ps), and then do:
-
-@example
- $ gdb ./glusterfs <pid>
-@end example
-
-Press Ctrl-C to interrupt the process and then generate the backtrace.
-
-@subsection Reproducing the bug
-
-If the bug is reproducible, please include the steps necessary to do
-so. If the bug is not reproducible, send us the bug report anyway.
-
-@subsection Other information
-
-If you think it is relevant, send us also the version of @acronym{FUSE} you're
-using, the kernel version, platform.
-
-@node GNU Free Documentation Licence
-@appendix GNU Free Documentation Licence
-@include fdl.texi
-
-@node Index
-@unnumbered Index
-@printindex cp
-
-@bye
diff --git a/doc/user-guide/xlator.odg b/doc/user-guide/xlator.odg
deleted file mode 100644
index 179a65f6e26..00000000000
--- a/doc/user-guide/xlator.odg
+++ /dev/null
diff --git a/doc/user-guide/xlator.pdf b/doc/user-guide/xlator.pdf
deleted file mode 100644
index a07e14d67d2..00000000000
--- a/doc/user-guide/xlator.pdf
+++ /dev/null
diff --git a/events/Makefile.am b/events/Makefile.am
new file mode 100644
index 00000000000..264bb742a80
--- /dev/null
+++ b/events/Makefile.am
@@ -0,0 +1,8 @@
+SUBDIRS = src tools
+EXTRA_DIST = eventskeygen.py
+noinst_PYTHON = eventskeygen.py
+
+if BUILD_EVENTS
+install-data-hook:
+	$(INSTALL) -d -m 755 $(DESTDIR)@GLUSTERD_WORKDIR@/events
+endif
diff --git a/events/eventskeygen.py b/events/eventskeygen.py
new file mode 100644
index 00000000000..e28ebe9b7e6
--- /dev/null
+++ b/events/eventskeygen.py
@@ -0,0 +1,243 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+
+GLUSTER_SRC_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+eventtypes_h = os.path.join(GLUSTER_SRC_ROOT, "libglusterfs/src/eventtypes.h")
+eventtypes_py = os.path.join(GLUSTER_SRC_ROOT, "events/src/eventtypes.py")
+
+gen_header_type = sys.argv[1]
+
+# When adding new keys add it to the END
+keys = (
+    # user driven events
+    #peer and volume management events
+    "EVENT_PEER_ATTACH",
+    "EVENT_PEER_DETACH",
+    "EVENT_VOLUME_CREATE",
+    "EVENT_VOLUME_START",
+    "EVENT_VOLUME_STOP",
+    "EVENT_VOLUME_DELETE",
+    "EVENT_VOLUME_SET",
+    "EVENT_VOLUME_RESET",
+    "EVENT_BRICK_RESET_START",
+    "EVENT_BRICK_RESET_COMMIT",
+    "EVENT_BRICK_REPLACE",
+
+    #geo-rep events
+    "EVENT_GEOREP_CREATE",
+    "EVENT_GEOREP_START",
+    "EVENT_GEOREP_STOP",
+    "EVENT_GEOREP_PAUSE",
+    "EVENT_GEOREP_RESUME",
+    "EVENT_GEOREP_DELETE",
+    "EVENT_GEOREP_CONFIG_SET",
+    "EVENT_GEOREP_CONFIG_RESET",
+
+    #bitrot events
+    "EVENT_BITROT_ENABLE",
+    "EVENT_BITROT_DISABLE",
+    "EVENT_BITROT_SCRUB_THROTTLE",
+    "EVENT_BITROT_SCRUB_FREQ",
+    "EVENT_BITROT_SCRUB_OPTION",
+    "EVENT_BITROT_SCRUB_ONDEMAND",
+
+    #quota events
+    "EVENT_QUOTA_ENABLE",
+    "EVENT_QUOTA_DISABLE",
+    "EVENT_QUOTA_SET_USAGE_LIMIT",
+    "EVENT_QUOTA_SET_OBJECTS_LIMIT",
+    "EVENT_QUOTA_REMOVE_USAGE_LIMIT",
+    "EVENT_QUOTA_REMOVE_OBJECTS_LIMIT",
+    "EVENT_QUOTA_ALERT_TIME",
+    "EVENT_QUOTA_SOFT_TIMEOUT",
+    "EVENT_QUOTA_HARD_TIMEOUT",
+    "EVENT_QUOTA_DEFAULT_SOFT_LIMIT",
+
+    #snapshot events
+    "EVENT_SNAPSHOT_CREATED",
+    "EVENT_SNAPSHOT_CREATE_FAILED",
+    "EVENT_SNAPSHOT_ACTIVATED",
+    "EVENT_SNAPSHOT_ACTIVATE_FAILED",
+    "EVENT_SNAPSHOT_DEACTIVATED",
+    "EVENT_SNAPSHOT_DEACTIVATE_FAILED",
+    "EVENT_SNAPSHOT_SOFT_LIMIT_REACHED",
+    "EVENT_SNAPSHOT_HARD_LIMIT_REACHED",
+    "EVENT_SNAPSHOT_RESTORED",
+    "EVENT_SNAPSHOT_RESTORE_FAILED",
+    "EVENT_SNAPSHOT_DELETED",
+    "EVENT_SNAPSHOT_DELETE_FAILED",
+    "EVENT_SNAPSHOT_CLONED",
+    "EVENT_SNAPSHOT_CLONE_FAILED",
+    "EVENT_SNAPSHOT_CONFIG_UPDATED",
+    "EVENT_SNAPSHOT_CONFIG_UPDATE_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_INITIALISED",
+    "EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_ENABLED",
+    "EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_DISABLED",
+    "EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED",
+    "EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED",
+
+    #async events
+    #glusterd events
+    "EVENT_SVC_MANAGER_FAILED",
+    "EVENT_SVC_RECONFIGURE_FAILED",
+    "EVENT_SVC_CONNECTED",
+    "EVENT_SVC_DISCONNECTED",
+    "EVENT_PEER_STORE_FAILURE",
+    "EVENT_PEER_RPC_CREATE_FAILED",
+    "EVENT_PEER_REJECT",
+    "EVENT_PEER_CONNECT",
+    "EVENT_PEER_DISCONNECT",
+    "EVENT_PEER_NOT_FOUND",
+    "EVENT_UNKNOWN_PEER",
+    "EVENT_BRICK_START_FAILED",
+    "EVENT_BRICK_STOP_FAILED",
+    "EVENT_BRICK_DISCONNECTED",
+    "EVENT_BRICK_CONNECTED",
+    "EVENT_BRICKPATH_RESOLVE_FAILED",
+    "EVENT_NOTIFY_UNKNOWN_OP",
+    "EVENT_QUORUM_LOST",
+    "EVENT_QUORUM_REGAINED",
+    "EVENT_REBALANCE_START_FAILED",
+    "EVENT_REBALANCE_STATUS_UPDATE_FAILED",
+    "EVENT_IMPORT_QUOTA_CONF_FAILED",
+    "EVENT_IMPORT_VOLUME_FAILED",
+    "EVENT_IMPORT_BRICK_FAILED",
+    "EVENT_COMPARE_FRIEND_VOLUME_FAILED",
+    "EVENT_NFS_GANESHA_EXPORT_FAILED",
+    #ec events
+    "EVENT_EC_MIN_BRICKS_NOT_UP",
+    "EVENT_EC_MIN_BRICKS_UP",
+    #georep async events
+    "EVENT_GEOREP_FAULTY",
+    "EVENT_GEOREP_CHECKPOINT_COMPLETED",
+    "EVENT_GEOREP_ACTIVE",
+    "EVENT_GEOREP_PASSIVE",
+
+    #quota async events
+    "EVENT_QUOTA_CROSSED_SOFT_LIMIT",
+    #bitrot async events
+    "EVENT_BITROT_BAD_FILE",
+    #protocol-server events
+    "EVENT_CLIENT_CONNECT",
+    "EVENT_CLIENT_AUTH_REJECT",
+    "EVENT_CLIENT_DISCONNECT",
+    #posix events
+    "EVENT_POSIX_SAME_GFID",
+    "EVENT_POSIX_ALREADY_PART_OF_VOLUME",
+    "EVENT_POSIX_BRICK_NOT_IN_VOLUME",
+    "EVENT_POSIX_BRICK_VERIFICATION_FAILED",
+    "EVENT_POSIX_ACL_NOT_SUPPORTED",
+    "EVENT_POSIX_HEALTH_CHECK_FAILED",
+    #afr events
+    "EVENT_AFR_QUORUM_MET",
+    "EVENT_AFR_QUORUM_FAIL",
+    "EVENT_AFR_SUBVOL_UP",
+    "EVENT_AFR_SUBVOLS_DOWN",
+    "EVENT_AFR_SPLIT_BRAIN",
+
+    #tier events
+    "EVENT_TIER_ATTACH",
+    "EVENT_TIER_ATTACH_FORCE",
+    "EVENT_TIER_DETACH_START",
+    "EVENT_TIER_DETACH_STOP",
+    "EVENT_TIER_DETACH_COMMIT",
+    "EVENT_TIER_DETACH_FORCE",
+    "EVENT_TIER_PAUSE",
+    "EVENT_TIER_RESUME",
+    "EVENT_TIER_WATERMARK_HI",
+    "EVENT_TIER_WATERMARK_DROPPED_TO_MID",
+    "EVENT_TIER_WATERMARK_RAISED_TO_MID",
+    "EVENT_TIER_WATERMARK_DROPPED_TO_LOW",
+
+    #dht events
+    #add/remove brick events
+    "EVENT_VOLUME_ADD_BRICK",
+    "EVENT_VOLUME_ADD_BRICK_FAILED",
+    "EVENT_VOLUME_REMOVE_BRICK_START",
+    "EVENT_VOLUME_REMOVE_BRICK_START_FAILED",
+    "EVENT_VOLUME_REMOVE_BRICK_COMMIT",
+    "EVENT_VOLUME_REMOVE_BRICK_COMMIT_FAILED",
+    "EVENT_VOLUME_REMOVE_BRICK_STOP",
+    "EVENT_VOLUME_REMOVE_BRICK_STOP_FAILED",
+    "EVENT_VOLUME_REMOVE_BRICK_FORCE",
+    "EVENT_VOLUME_REMOVE_BRICK_FORCE_FAILED",
+    "EVENT_VOLUME_REMOVE_BRICK_FAILED",
+
+    #rebalance events
+    "EVENT_VOLUME_REBALANCE_START",
+    "EVENT_VOLUME_REBALANCE_STOP",
+    "EVENT_VOLUME_REBALANCE_FAILED",
+    "EVENT_VOLUME_REBALANCE_COMPLETE",
+
+    #tier events
+    "EVENT_TIER_START",
+    "EVENT_TIER_START_FORCE",
+
+    #brick/inodes events
+    "EVENT_DHT_DISK_USAGE",
+    "EVENT_DHT_INODES_USAGE",
+)
+
+LAST_EVENT = "EVENT_LAST"
+
+ERRORS = (
+    "EVENT_SEND_OK",
+    "EVENT_ERROR_INVALID_INPUTS",
+    "EVENT_ERROR_SOCKET",
+    "EVENT_ERROR_CONNECT",
+    "EVENT_ERROR_SEND",
+    "EVENT_ERROR_RESOLVE",
+    "EVENT_ERROR_MSG_FORMAT",
+)
+
+if gen_header_type == "C_HEADER":
+    # Generate eventtypes.h
+    with open(eventtypes_h, "w") as f:
+        f.write("#ifndef __EVENTTYPES_H__\n")
+        f.write("#define __EVENTTYPES_H__\n\n")
+        f.write("typedef enum {\n")
+        for k in ERRORS:
+            f.write("    {0},\n".format(k))
+        f.write("} event_errors_t;\n")
+
+        f.write("\n")
+
+        f.write("typedef enum {\n")
+        for k in keys:
+            f.write("    {0},\n".format(k))
+
+        f.write("    {0}\n".format(LAST_EVENT))
+        f.write("} eventtypes_t;\n")
+        f.write("\n#endif /* __EVENTTYPES_H__ */\n")
+
+if gen_header_type == "PY_HEADER":
+    # Generate eventtypes.py
+    with open(eventtypes_py, "w") as f:
+        f.write("# -*- coding: utf-8 -*-\n")
+        f.write("all_events = [\n")
+        for ev in keys:
+            f.write('    "{0}",\n'.format(ev))
+
+        f.write("]\n\n")
+
+        for idx, ev in enumerate(keys):
+            f.write("{0} = {1}\n".format(ev.replace("EVENT_", ""), idx))
diff --git a/events/src/Makefile.am b/events/src/Makefile.am
new file mode 100644
index 00000000000..3b229691897
--- /dev/null
+++ b/events/src/Makefile.am
@@ -0,0 +1,41 @@
+noinst_PYTHON = $(top_srcdir)/events/eventskeygen.py
+EXTRA_DIST = glustereventsd.py __init__.py  eventsapiconf.py.in \
+	handlers.py utils.py peer_eventsapi.py eventsconfig.json gf_event.py
+
+BUILT_SOURCES = eventtypes.py
+CLEANFILES = eventtypes.py
+
+eventsdir = $(GLUSTERFS_LIBEXECDIR)/gfevents
+if BUILD_EVENTS
+events_PYTHON = __init__.py gf_event.py eventsapiconf.py eventtypes.py \
+	utils.py
+endif
+# this does not work, see the Makefile.am in the root for a workaround
+#nodist_events_PYTHON = eventtypes.py
+
+eventtypes.py: $(top_srcdir)/events/eventskeygen.py
+	$(PYTHON) $(top_srcdir)/events/eventskeygen.py PY_HEADER
+
+if BUILD_EVENTS
+eventspeerscriptdir = $(GLUSTERFS_LIBEXECDIR)
+eventsconfdir = $(sysconfdir)/glusterfs
+eventsconf_DATA = eventsconfig.json
+
+events_PYTHON += handlers.py
+events_SCRIPTS = glustereventsd.py
+eventspeerscript_SCRIPTS = peer_eventsapi.py
+
+install-exec-hook:
+	$(mkdir_p) $(DESTDIR)$(sbindir)
+	rm -f $(DESTDIR)$(sbindir)/glustereventsd
+	ln -s $(GLUSTERFS_LIBEXECDIR)/gfevents/glustereventsd.py \
+		$(DESTDIR)$(sbindir)/glustereventsd
+	rm -f $(DESTDIR)$(sbindir)/gluster-eventsapi
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_eventsapi.py \
+		$(DESTDIR)$(sbindir)/gluster-eventsapi
+
+uninstall-hook:
+	rm -f $(DESTDIR)$(sbindir)/glustereventsd
+	rm -f $(DESTDIR)$(sbindir)/gluster-eventsapi
+
+endif
diff --git a/events/src/__init__.py b/events/src/__init__.py
new file mode 100644
index 00000000000..f27c53a4df4
--- /dev/null
+++ b/events/src/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
diff --git a/events/src/eventsapiconf.py.in b/events/src/eventsapiconf.py.in
new file mode 100644
index 00000000000..700093bee60
--- /dev/null
+++ b/events/src/eventsapiconf.py.in
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import subprocess
+glusterd_workdir = None
+
+# Methods
+def get_glusterd_workdir():
+    global glusterd_workdir
+    if glusterd_workdir is not None:
+        return glusterd_workdir
+    proc = subprocess.Popen(["gluster", "system::", "getwd"],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            universal_newlines = True)
+    out, err = proc.communicate()
+    if proc.returncode == 0:
+        glusterd_workdir = out.strip()
+    else:
+        glusterd_workdir = "@GLUSTERD_WORKDIR@"
+    return glusterd_workdir
+
+SERVER_ADDRESS = "0.0.0.0"
+SERVER_ADDRESSv4 = "0.0.0.0"
+SERVER_ADDRESSv6 = "::1"
+DEFAULT_CONFIG_FILE = "@SYSCONF_DIR@/glusterfs/eventsconfig.json"
+CUSTOM_CONFIG_FILE_TO_SYNC = "/events/config.json"
+CUSTOM_CONFIG_FILE = get_glusterd_workdir() + CUSTOM_CONFIG_FILE_TO_SYNC
+WEBHOOKS_FILE_TO_SYNC = "/events/webhooks.json"
+WEBHOOKS_FILE = get_glusterd_workdir() + WEBHOOKS_FILE_TO_SYNC
+LOG_FILE = "@localstatedir@/log/glusterfs/events.log"
+EVENTSD = "glustereventsd"
+CONFIG_KEYS = ["log-level", "port", "disable-events-log"]
+BOOL_CONFIGS = ["disable-events-log"]
+INT_CONFIGS = ["port"]
+RESTART_CONFIGS = ["port"]
+EVENTS_ENABLED = @EVENTS_ENABLED@
+UUID_FILE = get_glusterd_workdir() + "/glusterd.info"
+PID_FILE = "@localstatedir@/run/glustereventsd.pid"
+AUTO_BOOL_ATTRIBUTES = ["force", "push-pem", "no-verify"]
+AUTO_INT_ATTRIBUTES = ["ssh-port"]
+CERTS_DIR = get_glusterd_workdir() + "/events"
+
+# Errors
+ERROR_SAME_CONFIG = 2
+ERROR_ALL_NODES_STATUS_NOT_OK = 3
+ERROR_PARTIAL_SUCCESS = 4
+ERROR_WEBHOOK_ALREADY_EXISTS = 5
+ERROR_WEBHOOK_NOT_EXISTS = 6
+ERROR_INVALID_CONFIG = 7
+ERROR_WEBHOOK_SYNC_FAILED = 8
+ERROR_CONFIG_SYNC_FAILED = 9
diff --git a/events/src/eventsconfig.json b/events/src/eventsconfig.json
new file mode 100644
index 00000000000..89e5b9c1d68
--- /dev/null
+++ b/events/src/eventsconfig.json
@@ -0,0 +1,5 @@
+{
+    "log-level": "INFO",
+    "port": 24009,
+    "disable-events-log": false
+}
diff --git a/events/src/gf_event.py b/events/src/gf_event.py
new file mode 100644
index 00000000000..260b0d9aa48
--- /dev/null
+++ b/events/src/gf_event.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import socket
+import time
+
+from gfevents.eventsapiconf import SERVER_ADDRESS, EVENTS_ENABLED
+from gfevents.eventtypes import all_events
+
+from gfevents.utils import logger, setup_logger, get_config
+
+# Run this when this lib loads
+setup_logger()
+
+
+def gf_event(event_type, **kwargs):
+    if EVENTS_ENABLED == 0:
+        return
+
+    if not isinstance(event_type, int) or event_type >= len(all_events):
+        logger.error("Invalid Event Type: {0}".format(event_type))
+        return
+
+    try:
+        client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    except socket.error as e:
+        logger.error("Unable to connect to events Server: {0}".format(e))
+        return
+
+    port = get_config("port")
+    if port is None:
+        logger.error("Unable to get eventsd port details")
+        return
+
+    # Convert key value args into KEY1=VALUE1;KEY2=VALUE2;..
+    msg = ""
+    for k, v in kwargs.items():
+        msg += "{0}={1};".format(k, v)
+
+    # <TIMESTAMP> <EVENT_TYPE> <MSG>
+    msg = "{0} {1} {2}".format(int(time.time()), event_type, msg.strip(";")).encode()
+
+    try:
+        sent = client.sendto(msg, (SERVER_ADDRESS, port))
+        assert sent == len(msg)
+    except socket.error as e:
+        logger.error("Unable to Send message: {0}".format(e))
+    except AssertionError:
+        logger.error("Unable to send message. Sent: {0}, Actual: {1}".format(
+            sent, len(msg)))
+    finally:
+        client.close()
diff --git a/events/src/glustereventsd.py b/events/src/glustereventsd.py
new file mode 100644
index 00000000000..341a3b60947
--- /dev/null
+++ b/events/src/glustereventsd.py
@@ -0,0 +1,159 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+import sys
+import signal
+import threading
+try:
+    import socketserver
+except ImportError:
+    import SocketServer as socketserver
+import socket
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+from eventtypes import all_events
+import handlers
+import utils
+from eventsapiconf import SERVER_ADDRESSv4, SERVER_ADDRESSv6, PID_FILE
+from eventsapiconf import AUTO_BOOL_ATTRIBUTES, AUTO_INT_ATTRIBUTES
+from utils import logger, PidFile, PidFileLockFailed, boolify
+
+# Subclass so that specifically IPv4 packets are captured
+class UDPServerv4(socketserver.ThreadingUDPServer):
+    address_family = socket.AF_INET
+
+# Subclass so that specifically IPv6 packets are captured
+class UDPServerv6(socketserver.ThreadingUDPServer):
+    address_family = socket.AF_INET6
+
+class GlusterEventsRequestHandler(socketserver.BaseRequestHandler):
+
+    def handle(self):
+        data = self.request[0].strip()
+        if sys.version_info >= (3,):
+            data = self.request[0].strip().decode("utf-8")
+
+        logger.debug("EVENT: {0} from {1}".format(repr(data),
+                                                  self.client_address[0]))
+        try:
+            # Event Format <TIMESTAMP> <TYPE> <DETAIL>
+            ts, key, value = data.split(" ", 2)
+        except ValueError:
+            logger.warn("Invalid Event Format {0}".format(data))
+            return
+
+        data_dict = {}
+        try:
+            # Format key=value;key=value
+            data_dict = dict(x.split('=') for x in value.split(';'))
+        except ValueError:
+            logger.warn("Unable to parse Event {0}".format(data))
+            return
+
+        for k, v in data_dict.items():
+            try:
+                if k in AUTO_BOOL_ATTRIBUTES:
+                    data_dict[k] = boolify(v)
+                if k in AUTO_INT_ATTRIBUTES:
+                    data_dict[k] = int(v)
+            except ValueError:
+                # Auto Conversion failed, Retain the old value
+                continue
+
+        try:
+            # Event Type to Function Map, Received event data will be in
+            # the form <TIMESTAMP> <TYPE> <DETAIL>, Get Event name for the
+            # received Type/Key and construct a function name starting with
+            # handle_ For example: handle_event_volume_create
+            func_name = "handle_" + all_events[int(key)].lower()
+        except IndexError:
+            # This type of Event is not handled?
+            logger.warn("Unhandled Event: {0}".format(key))
+            func_name = None
+
+        if func_name is not None:
+            # Get function from handlers module
+            func = getattr(handlers, func_name, None)
+            # If func is None, then handler unimplemented for that event.
+            if func is not None:
+                func(ts, int(key), data_dict)
+            else:
+                # Generic handler, broadcast whatever received
+                handlers.generic_handler(ts, int(key), data_dict)
+
+
+def signal_handler_sigusr2(sig, frame):
+    utils.load_all()
+    utils.restart_webhook_pool()
+
+
+def UDP_server_thread(sock):
+    sock.serve_forever()
+
+
+def init_event_server():
+    utils.setup_logger()
+    utils.load_all()
+    utils.init_webhook_pool()
+
+    port = utils.get_config("port")
+    if port is None:
+        sys.stderr.write("Unable to get Port details from Config\n")
+        sys.exit(1)
+
+    # Creating the Eventing Server, UDP Server for IPv4 packets
+    try:
+        serverv4 = UDPServerv4((SERVER_ADDRESSv4, port),
+                   GlusterEventsRequestHandler)
+    except socket.error as e:
+        sys.stderr.write("Failed to start Eventsd for IPv4: {0}\n".format(e))
+        sys.exit(1)
+    # Creating the Eventing Server, UDP Server for IPv6 packets
+    try:
+        serverv6 = UDPServerv6((SERVER_ADDRESSv6, port),
+                   GlusterEventsRequestHandler)
+    except socket.error as e:
+        sys.stderr.write("Failed to start Eventsd for IPv6: {0}\n".format(e))
+        sys.exit(1)
+    server_thread1 = threading.Thread(target=UDP_server_thread,
+                     args=(serverv4,))
+    server_thread2 = threading.Thread(target=UDP_server_thread,
+                     args=(serverv6,))
+    server_thread1.start()
+    server_thread2.start()
+
+
+def get_args():
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
+                            description=__doc__)
+    parser.add_argument("-p", "--pid-file", help="PID File",
+                        default=PID_FILE)
+
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    try:
+        with PidFile(args.pid_file):
+            init_event_server()
+    except PidFileLockFailed as e:
+        sys.stderr.write("Failed to get lock for pid file({0}): {1}".format(
+            args.pid_file, e))
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGUSR2, signal_handler_sigusr2)
+    main()
diff --git a/events/src/handlers.py b/events/src/handlers.py
new file mode 100644
index 00000000000..7746d488bf3
--- /dev/null
+++ b/events/src/handlers.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import utils
+
+
+def generic_handler(ts, key, data):
+    """
+    Generic handler to broadcast message to all peers, custom handlers
+    can be created by func name handler_<event_name>
+    Ex: handle_event_volume_create(ts, key, data)
+    """
+    utils.publish(ts, key, data)
+
+
+def handle_event_volume_set(ts, key, data):
+    """
+    Received data will have all the options as one string, split into
+    list of options. "key1,value1,key2,value2" into
+    [[key1, value1], [key2, value2]]
+    """
+    opts = data.get("options", "").strip(",").split(",")
+    data["options"] = []
+    for i, opt in enumerate(opts):
+        if i % 2 == 0:
+            # Add new array with key
+            data["options"].append([opt])
+        else:
+            # Add to the last added array
+            data["options"][-1].append(opt)
+
+    utils.publish(ts, key, data)
diff --git a/events/src/peer_eventsapi.py b/events/src/peer_eventsapi.py
new file mode 100644
index 00000000000..4d2e5f35b1c
--- /dev/null
+++ b/events/src/peer_eventsapi.py
@@ -0,0 +1,669 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+import os
+import json
+from errno import EEXIST
+import fcntl
+from errno import EACCES, EAGAIN
+import signal
+import sys
+import time
+
+import requests
+from prettytable import PrettyTable
+
+from gluster.cliutils import (Cmd, node_output_ok, node_output_notok,
+                              sync_file_to_peers, GlusterCmdException,
+                              output_error, execute_in_peers, runcli,
+                              set_common_args_func)
+from gfevents.utils import LockedOpen, get_jwt_token, save_https_cert
+
+from gfevents.eventsapiconf import (WEBHOOKS_FILE_TO_SYNC,
+                                    WEBHOOKS_FILE,
+                                    DEFAULT_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE_TO_SYNC,
+                                    EVENTSD,
+                                    CONFIG_KEYS,
+                                    BOOL_CONFIGS,
+                                    INT_CONFIGS,
+                                    PID_FILE,
+                                    RESTART_CONFIGS,
+                                    ERROR_INVALID_CONFIG,
+                                    ERROR_WEBHOOK_NOT_EXISTS,
+                                    ERROR_CONFIG_SYNC_FAILED,
+                                    ERROR_WEBHOOK_ALREADY_EXISTS,
+                                    ERROR_PARTIAL_SUCCESS,
+                                    ERROR_ALL_NODES_STATUS_NOT_OK,
+                                    ERROR_SAME_CONFIG,
+                                    ERROR_WEBHOOK_SYNC_FAILED,
+                                    CERTS_DIR)
+
+
+def handle_output_error(err, errcode=1, json_output=False):
+    if json_output:
+        print (json.dumps({
+            "output": "",
+            "error": err
+            }))
+        sys.exit(errcode)
+    else:
+        output_error(err, errcode)
+
+
+def file_content_overwrite(fname, data):
+    with open(fname + ".tmp", "w") as f:
+        f.write(json.dumps(data))
+
+    os.rename(fname + ".tmp", fname)
+
+
+def create_custom_config_file_if_not_exists(args):
+    try:
+        config_dir = os.path.dirname(CUSTOM_CONFIG_FILE)
+        mkdirp(config_dir)
+    except OSError as e:
+        handle_output_error("Failed to create dir %s: %s" % (config_dir, e),
+                            json_output=args.json)
+
+    if not os.path.exists(CUSTOM_CONFIG_FILE):
+        with open(CUSTOM_CONFIG_FILE, "w") as f:
+            f.write("{}")
+
+
+def create_webhooks_file_if_not_exists(args):
+    try:
+        webhooks_dir = os.path.dirname(WEBHOOKS_FILE)
+        mkdirp(webhooks_dir)
+    except OSError as e:
+        handle_output_error("Failed to create dir %s: %s" % (webhooks_dir, e),
+                            json_output=args.json)
+
+    if not os.path.exists(WEBHOOKS_FILE):
+        with open(WEBHOOKS_FILE, "w") as f:
+            f.write("{}")
+
+
+def boolify(value):
+    val = False
+    if value.lower() in ["enabled", "true", "on", "yes"]:
+        val = True
+    return val
+
+
+def mkdirp(path, exit_on_err=False, logger=None):
+    """
+    Try creating required directory structure
+    ignore EEXIST and raise exception for rest of the errors.
+    Print error in stderr and exit
+    """
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != EEXIST or not os.path.isdir(path):
+            raise
+
+
+def is_active():
+    state = False
+    try:
+        with open(PID_FILE, "a+") as f:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            state = False
+    except (IOError, OSError) as e:
+        if e.errno in (EACCES, EAGAIN):
+            # cannot grab. so, process still running..move on
+            state = True
+        else:
+            state = False
+    return state
+
+
+def reload_service():
+    pid = None
+    if is_active():
+        with open(PID_FILE) as f:
+            try:
+                pid = int(f.read().strip())
+            except ValueError:
+                pid = None
+        if pid is not None:
+            os.kill(pid, signal.SIGUSR2)
+
+    return (0, "", "")
+
+
+def rows_to_json(json_out, column_name, rows):
+    num_ok_rows = 0
+    for row in rows:
+        num_ok_rows += 1 if row.ok else 0
+        json_out.append({
+            "node": row.hostname,
+            "node_status": "UP" if row.node_up else "DOWN",
+            column_name: "OK" if row.ok else "NOT OK",
+            "error": row.error
+        })
+    return num_ok_rows
+
+
+def rows_to_table(table, rows):
+    num_ok_rows = 0
+    for row in rows:
+        num_ok_rows += 1 if row.ok else 0
+        table.add_row([row.hostname,
+                       "UP" if row.node_up else "DOWN",
+                       "OK" if row.ok else "NOT OK: {0}".format(
+                           row.error)])
+    return num_ok_rows
+
+
+def sync_to_peers(args):
+    if os.path.exists(WEBHOOKS_FILE):
+        try:
+            sync_file_to_peers(WEBHOOKS_FILE_TO_SYNC)
+        except GlusterCmdException as e:
+            # Print stdout if stderr is empty
+            errmsg = e.message[2] if e.message[2] else e.message[1]
+            handle_output_error("Failed to sync Webhooks file: [Error: {0}]"
+                                "{1}".format(e.message[0], errmsg),
+                                errcode=ERROR_WEBHOOK_SYNC_FAILED,
+                                json_output=args.json)
+
+    if os.path.exists(CUSTOM_CONFIG_FILE):
+        try:
+            sync_file_to_peers(CUSTOM_CONFIG_FILE_TO_SYNC)
+        except GlusterCmdException as e:
+            # Print stdout if stderr is empty
+            errmsg = e.message[2] if e.message[2] else e.message[1]
+            handle_output_error("Failed to sync Config file: [Error: {0}]"
+                                "{1}".format(e.message[0], errmsg),
+                                errcode=ERROR_CONFIG_SYNC_FAILED,
+                                json_output=args.json)
+
+    out = execute_in_peers("node-reload")
+    if not args.json:
+        table = PrettyTable(["NODE", "NODE STATUS", "SYNC STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["SYNC STATUS"] = "r"
+
+    json_out = []
+    if args.json:
+        num_ok_rows = rows_to_json(json_out, "sync_status", out)
+    else:
+        num_ok_rows = rows_to_table(table, out)
+
+    ret = 0
+    if num_ok_rows == 0:
+        ret = ERROR_ALL_NODES_STATUS_NOT_OK
+    elif num_ok_rows != len(out):
+        ret = ERROR_PARTIAL_SUCCESS
+
+    if args.json:
+        print (json.dumps({
+            "output": json_out,
+            "error": ""
+        }))
+    else:
+        print (table)
+
+    # If sync status is not ok for any node set error code as partial success
+    sys.exit(ret)
+
+
+def node_output_handle(resp):
+    rc, out, err = resp
+    if rc == 0:
+        node_output_ok(out)
+    else:
+        node_output_notok(err)
+
+
+def action_handle(action, json_output=False):
+    out = execute_in_peers("node-" + action)
+    column_name = action.upper()
+    if action == "status":
+        column_name = EVENTSD.upper()
+
+    if not json_output:
+        table = PrettyTable(["NODE", "NODE STATUS", column_name + " STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align[column_name + " STATUS"] = "r"
+
+    json_out = []
+    if json_output:
+        rows_to_json(json_out, column_name.lower() + "_status", out)
+    else:
+        rows_to_table(table, out)
+
+    return json_out if json_output else table
+
+
+class NodeReload(Cmd):
+    name = "node-reload"
+
+    def run(self, args):
+        node_output_handle(reload_service())
+
+
+class ReloadCmd(Cmd):
+    name = "reload"
+
+    def run(self, args):
+        out = action_handle("reload", args.json)
+        if args.json:
+            print (json.dumps({
+                "output": out,
+                "error": ""
+            }))
+        else:
+            print (out)
+
+
+class NodeStatus(Cmd):
+    name = "node-status"
+
+    def run(self, args):
+        node_output_ok("UP" if is_active() else "DOWN")
+
+
+class StatusCmd(Cmd):
+    name = "status"
+
+    def run(self, args):
+        webhooks = {}
+        if os.path.exists(WEBHOOKS_FILE):
+            webhooks = json.load(open(WEBHOOKS_FILE))
+
+        json_out = {"webhooks": [], "data": []}
+        if args.json:
+            json_out["webhooks"] = webhooks.keys()
+        else:
+            print ("Webhooks: " + ("" if webhooks else "None"))
+            for w in webhooks:
+                print (w)
+
+            print ()
+
+        out = action_handle("status", args.json)
+        if args.json:
+            json_out["data"] = out
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
+        else:
+            print (out)
+
+
+class WebhookAddCmd(Cmd):
+    name = "webhook-add"
+
+    def args(self, parser):
+        parser.add_argument("url", help="URL of Webhook")
+        parser.add_argument("--bearer_token", "-t", help="Bearer Token",
+                            default="")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to add JWT Bearer Token", default="")
+
+    def run(self, args):
+        create_webhooks_file_if_not_exists(args)
+
+        with LockedOpen(WEBHOOKS_FILE, 'r+'):
+            data = json.load(open(WEBHOOKS_FILE))
+            if data.get(args.url, None) is not None:
+                handle_output_error("Webhook already exists",
+                                    errcode=ERROR_WEBHOOK_ALREADY_EXISTS,
+                                    json_output=args.json)
+
+            data[args.url] = {"token": args.bearer_token,
+                              "secret": args.secret}
+            file_content_overwrite(WEBHOOKS_FILE, data)
+
+        sync_to_peers(args)
+
+
+class WebhookModCmd(Cmd):
+    name = "webhook-mod"
+
+    def args(self, parser):
+        parser.add_argument("url", help="URL of Webhook")
+        parser.add_argument("--bearer_token", "-t", help="Bearer Token",
+                            default="")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to add JWT Bearer Token", default="")
+
+    def run(self, args):
+        create_webhooks_file_if_not_exists(args)
+
+        with LockedOpen(WEBHOOKS_FILE, 'r+'):
+            data = json.load(open(WEBHOOKS_FILE))
+            if data.get(args.url, None) is None:
+                handle_output_error("Webhook does not exists",
+                                    errcode=ERROR_WEBHOOK_NOT_EXISTS,
+                                    json_output=args.json)
+
+            if isinstance(data[args.url], str):
+                data[args.url]["token"] = data[args.url]
+
+            if args.bearer_token != "":
+                data[args.url]["token"] = args.bearer_token
+
+            if args.secret != "":
+                data[args.url]["secret"] = args.secret
+
+            file_content_overwrite(WEBHOOKS_FILE, data)
+
+        sync_to_peers(args)
+
+
+class WebhookDelCmd(Cmd):
+    name = "webhook-del"
+
+    def args(self, parser):
+        parser.add_argument("url", help="URL of Webhook")
+
+    def run(self, args):
+        create_webhooks_file_if_not_exists(args)
+
+        with LockedOpen(WEBHOOKS_FILE, 'r+'):
+            data = json.load(open(WEBHOOKS_FILE))
+            if data.get(args.url, None) is None:
+                handle_output_error("Webhook does not exists",
+                                    errcode=ERROR_WEBHOOK_NOT_EXISTS,
+                                    json_output=args.json)
+
+            del data[args.url]
+            file_content_overwrite(WEBHOOKS_FILE, data)
+
+        sync_to_peers(args)
+
+
+class NodeWebhookTestCmd(Cmd):
+    name = "node-webhook-test"
+
+    def args(self, parser):
+        parser.add_argument("url")
+        parser.add_argument("bearer_token")
+        parser.add_argument("secret")
+
+    def run(self, args):
+        http_headers = {}
+        hashval = ""
+        if args.bearer_token != ".":
+            hashval = args.bearer_token
+
+        if args.secret != ".":
+            hashval = get_jwt_token(args.secret, "TEST", int(time.time()))
+
+        if hashval:
+            http_headers["Authorization"] = "Bearer " + hashval
+
+        urldata = requests.utils.urlparse(args.url)
+        parts = urldata.netloc.split(":")
+        domain = parts[0]
+        # Default https port if not specified
+        port = 443
+        if len(parts) == 2:
+            port = int(parts[1])
+
+        cert_path = os.path.join(CERTS_DIR, args.url.replace("/", "_").strip())
+        verify = True
+        while True:
+            try:
+                resp = requests.post(args.url, headers=http_headers,
+                                     verify=verify)
+                # Successful webhook push
+                break
+            except requests.exceptions.SSLError as e:
+                # If verify is equal to cert path, but still failed with
+                # SSLError, Looks like some issue with custom downloaded
+                # certificate, Try with verify = false
+                if verify == cert_path:
+                    verify = False
+                    continue
+
+                # If verify is instance of bool and True, then custom cert
+                # is required, download the cert and retry
+                try:
+                    save_https_cert(domain, port, cert_path)
+                    verify = cert_path
+                except Exception:
+                    verify = False
+
+                # Done with collecting cert, continue
+                continue
+            except Exception as e:
+                node_output_notok("{0}".format(e))
+                break
+
+        if resp.status_code != 200:
+            node_output_notok("{0}".format(resp.status_code))
+
+        node_output_ok()
+
+
+class WebhookTestCmd(Cmd):
+    name = "webhook-test"
+
+    def args(self, parser):
+        parser.add_argument("url", help="URL of Webhook")
+        parser.add_argument("--bearer_token", "-t", help="Bearer Token")
+        parser.add_argument("--secret", "-s",
+                            help="Secret to generate Bearer Token")
+
+    def run(self, args):
+        url = args.url
+        bearer_token = args.bearer_token
+        secret = args.secret
+
+        if not args.url:
+            url = "."
+        if not args.bearer_token:
+            bearer_token = "."
+        if not args.secret:
+            secret = "."
+
+        out = execute_in_peers("node-webhook-test", [url, bearer_token,
+                                                     secret])
+
+        if not args.json:
+            table = PrettyTable(["NODE", "NODE STATUS", "WEBHOOK STATUS"])
+            table.align["NODE STATUS"] = "r"
+            table.align["WEBHOOK STATUS"] = "r"
+
+        num_ok_rows = 0
+        json_out = []
+        if args.json:
+            num_ok_rows = rows_to_json(json_out, "webhook_status", out)
+        else:
+            num_ok_rows = rows_to_table(table, out)
+
+        ret = 0
+        if num_ok_rows == 0:
+            ret = ERROR_ALL_NODES_STATUS_NOT_OK
+        elif num_ok_rows != len(out):
+            ret = ERROR_PARTIAL_SUCCESS
+
+        if args.json:
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
+        else:
+            print (table)
+
+        sys.exit(ret)
+
+
+class ConfigGetCmd(Cmd):
+    name = "config-get"
+
+    def args(self, parser):
+        parser.add_argument("--name", help="Config Name")
+
+    def run(self, args):
+        data = json.load(open(DEFAULT_CONFIG_FILE))
+        if os.path.exists(CUSTOM_CONFIG_FILE):
+            data.update(json.load(open(CUSTOM_CONFIG_FILE)))
+
+        if args.name is not None and args.name not in CONFIG_KEYS:
+            handle_output_error("Invalid Config item",
+                                errcode=ERROR_INVALID_CONFIG,
+                                json_output=args.json)
+
+        if args.json:
+            json_out = {}
+            if args.name is None:
+                json_out = data
+            else:
+                json_out[args.name] = data[args.name]
+
+            print (json.dumps({
+                "output": json_out,
+                "error": ""
+            }))
+        else:
+            table = PrettyTable(["NAME", "VALUE"])
+            if args.name is None:
+                for k, v in data.items():
+                    table.add_row([k, v])
+            else:
+                table.add_row([args.name, data[args.name]])
+
+            print (table)
+
+
+def read_file_content_json(fname):
+    content = "{}"
+    with open(fname) as f:
+        content = f.read()
+        if content.strip() == "":
+            content = "{}"
+
+    return json.loads(content)
+
+
+class ConfigSetCmd(Cmd):
+    name = "config-set"
+
+    def args(self, parser):
+        parser.add_argument("name", help="Config Name")
+        parser.add_argument("value", help="Config Value")
+
+    def run(self, args):
+        if args.name not in CONFIG_KEYS:
+            handle_output_error("Invalid Config item",
+                                errcode=ERROR_INVALID_CONFIG,
+                                json_output=args.json)
+
+        create_custom_config_file_if_not_exists(args)
+
+        with LockedOpen(CUSTOM_CONFIG_FILE, 'r+'):
+            data = json.load(open(DEFAULT_CONFIG_FILE))
+            if os.path.exists(CUSTOM_CONFIG_FILE):
+                config_json = read_file_content_json(CUSTOM_CONFIG_FILE)
+                data.update(config_json)
+
+            # Do Nothing if same as previous value
+            if data[args.name] == args.value:
+                handle_output_error("Config value not changed. Same config",
+                                    errcode=ERROR_SAME_CONFIG,
+                                    json_output=args.json)
+
+            # TODO: Validate Value
+            new_data = read_file_content_json(CUSTOM_CONFIG_FILE)
+
+            v = args.value
+            if args.name in BOOL_CONFIGS:
+                v = boolify(args.value)
+
+            if args.name in INT_CONFIGS:
+                v = int(args.value)
+
+            new_data[args.name] = v
+            file_content_overwrite(CUSTOM_CONFIG_FILE, new_data)
+
+            # If any value changed which requires restart of REST server
+            restart = False
+            if args.name in RESTART_CONFIGS:
+                restart = True
+
+            if restart:
+                print ("\nRestart glustereventsd in all nodes")
+
+            sync_to_peers(args)
+
+
+class ConfigResetCmd(Cmd):
+    name = "config-reset"
+
+    def args(self, parser):
+        parser.add_argument("name", help="Config Name or all")
+
+    def run(self, args):
+        create_custom_config_file_if_not_exists(args)
+
+        with LockedOpen(CUSTOM_CONFIG_FILE, 'r+'):
+            changed_keys = []
+            data = {}
+            if os.path.exists(CUSTOM_CONFIG_FILE):
+                data = read_file_content_json(CUSTOM_CONFIG_FILE)
+
+            # If No data available in custom config or, the specific config
+            # item is not available in custom config
+            if not data or \
+               (args.name != "all" and data.get(args.name, None) is None):
+                handle_output_error("Config value not reset. Already "
+                                    "set to default value",
+                                    errcode=ERROR_SAME_CONFIG,
+                                    json_output=args.json)
+
+            if args.name.lower() == "all":
+                for k, v in data.items():
+                    changed_keys.append(k)
+
+                # Reset all keys
+                file_content_overwrite(CUSTOM_CONFIG_FILE, {})
+            else:
+                changed_keys.append(args.name)
+                del data[args.name]
+                file_content_overwrite(CUSTOM_CONFIG_FILE, data)
+
+            # If any value changed which requires restart of REST server
+            restart = False
+            for key in changed_keys:
+                if key in RESTART_CONFIGS:
+                    restart = True
+                    break
+
+            if restart:
+                print ("\nRestart glustereventsd in all nodes")
+
+            sync_to_peers(args)
+
+
+class SyncCmd(Cmd):
+    name = "sync"
+
+    def run(self, args):
+        sync_to_peers(args)
+
+
+def common_args(parser):
+    parser.add_argument("--json", help="JSON Output", action="store_true")
+
+
+if __name__ == "__main__":
+    set_common_args_func(common_args)
+    runcli()
diff --git a/events/src/utils.py b/events/src/utils.py
new file mode 100644
index 00000000000..6d4e0791a2b
--- /dev/null
+++ b/events/src/utils.py
@@ -0,0 +1,445 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import sys
+import json
+import os
+import logging
+import logging.handlers
+import fcntl
+from errno import EBADF
+from threading import Thread
+import multiprocessing
+try:
+    from queue import Queue
+except ImportError:
+    from Queue import Queue
+from datetime import datetime, timedelta
+import base64
+import hmac
+from hashlib import sha256
+from calendar import timegm
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from gfevents.eventsapiconf import (LOG_FILE,
+                                    WEBHOOKS_FILE,
+                                    DEFAULT_CONFIG_FILE,
+                                    CUSTOM_CONFIG_FILE,
+                                    UUID_FILE,
+                                    CERTS_DIR)
+from gfevents import eventtypes
+
+
+# Webhooks list
+_webhooks = {}
+_webhooks_file_mtime = 0
+# Default Log Level
+_log_level = "INFO"
+# Config Object
+_config = {}
+
+# Init Logger instance
+logger = logging.getLogger(__name__)
+NodeID = None
+webhooks_pool = None
+
+
+def boolify(value):
+    value = str(value)
+    if value.lower() in ["1", "on", "true", "yes"]:
+        return True
+    else:
+        return False
+
+
+def log_event(data):
+    # Log all published events unless it is disabled
+    if not _config.get("disable-events-log", False):
+        logger.info(repr(data))
+
+
+def get_node_uuid():
+    val = None
+    with open(UUID_FILE) as f:
+        for line in f:
+            if line.startswith("UUID="):
+                val = line.strip().split("=")[-1]
+                break
+    return val
+
+
+def get_config(key, default_value=None):
+    if not _config:
+        load_config()
+    return _config.get(key, default_value)
+
+
+def get_event_type_name(idx):
+    """
+    Returns Event Type text from the index. For example, VOLUME_CREATE
+    """
+    return eventtypes.all_events[idx].replace("EVENT_", "")
+
+
+def setup_logger():
+    """
+    Logging initialization, Log level by default will be INFO, once config
+    file is read, respective log_level will be set.
+    """
+    global logger
+    logger.setLevel(logging.INFO)
+
+    # create the logging file handler
+    fh = logging.handlers.WatchedFileHandler(LOG_FILE)
+
+    formatter = logging.Formatter("[%(asctime)s] %(levelname)s "
+                                  "[%(module)s - %(lineno)s:%(funcName)s] "
+                                  "- %(message)s")
+
+    fh.setFormatter(formatter)
+
+    # add handler to logger object
+    logger.addHandler(fh)
+
+
+def load_config():
+    """
+    Load/Reload the config from REST Config files. This function will
+    be triggered during init and when SIGUSR2.
+    """
+    global _config
+    _config = {}
+    if os.path.exists(DEFAULT_CONFIG_FILE):
+        _config = json.load(open(DEFAULT_CONFIG_FILE))
+    if os.path.exists(CUSTOM_CONFIG_FILE):
+        _config.update(json.load(open(CUSTOM_CONFIG_FILE)))
+
+
+def load_log_level():
+    """
+    Reads log_level from Config file and sets accordingly. This function will
+    be triggered during init and when SIGUSR2.
+    """
+    global logger, _log_level
+    new_log_level = _config.get("log-level", "INFO")
+    if _log_level != new_log_level:
+        logger.setLevel(getattr(logging, new_log_level.upper()))
+        _log_level = new_log_level.upper()
+
+
+def load_webhooks():
+    """
+    Load/Reload the webhooks list. This function will
+    be triggered during init and when SIGUSR2.
+    """
+    global _webhooks, _webhooks_file_mtime
+    _webhooks = {}
+    if os.path.exists(WEBHOOKS_FILE):
+        _webhooks = json.load(open(WEBHOOKS_FILE))
+        st = os.lstat(WEBHOOKS_FILE)
+        _webhooks_file_mtime = st.st_mtime
+
+
+def load_all():
+    """
+    Wrapper function to call all load/reload functions. This function will
+    be triggered during init and when SIGUSR2.
+    """
+    load_config()
+    load_webhooks()
+    load_log_level()
+
+
+def publish(ts, event_key, data):
+    global NodeID
+    if NodeID is None:
+        NodeID = get_node_uuid()
+
+    autoload_webhooks()
+
+    message = {
+        "nodeid": NodeID,
+        "ts": int(ts),
+        "event": get_event_type_name(event_key),
+        "message": data
+    }
+
+    log_event(message)
+
+    if _webhooks:
+        plugin_webhook(message)
+    else:
+        # TODO: Default action?
+        pass
+
+
+def autoload_webhooks():
+    global _webhooks_file_mtime
+    try:
+        st = os.lstat(WEBHOOKS_FILE)
+    except OSError:
+        st = None
+
+    if st is not None:
+        # If Stat is available and mtime is not matching with
+        # previously recorded mtime, reload the webhooks file
+        if st.st_mtime != _webhooks_file_mtime:
+            load_webhooks()
+
+
+def base64_urlencode(inp):
+    return base64.urlsafe_b64encode(inp).replace("=", "").strip()
+
+
+def get_jwt_token(secret, event_type, event_ts, jwt_expiry_time_seconds=60):
+    exp = datetime.utcnow() + timedelta(seconds=jwt_expiry_time_seconds)
+    payload = {
+        "exp": timegm(exp.utctimetuple()),
+        "iss": "gluster",
+        "sub": event_type,
+        "iat": event_ts
+    }
+    header = '{"alg":"HS256","typ":"JWT"}'
+    payload = json.dumps(payload, separators=(',', ':'), sort_keys=True)
+    msg = base64_urlencode(header) + "." + base64_urlencode(payload)
+    return "%s.%s" % (
+        msg,
+        base64_urlencode(hmac.HMAC(str(secret), msg, sha256).digest())
+    )
+
+
+def save_https_cert(domain, port, cert_path):
+    import ssl
+
+    # Cert file already available for this URL
+    if os.path.exists(cert_path):
+        return
+
+    cert_data = ssl.get_server_certificate((domain, port))
+    with open(cert_path, "w") as f:
+        f.write(cert_data)
+
+
+def publish_to_webhook(url, token, secret, message_queue):
+    # Import requests here since not used in any other place
+    import requests
+
+    http_headers = {"Content-Type": "application/json"}
+    urldata = requests.utils.urlparse(url)
+    parts = urldata.netloc.split(":")
+    domain = parts[0]
+    # Default https port if not specified
+    port = 443
+    if len(parts) == 2:
+        port = int(parts[1])
+
+    cert_path = os.path.join(CERTS_DIR, url.replace("/", "_").strip())
+
+    while True:
+        hashval = ""
+        event_type, event_ts, message_json = message_queue.get()
+        if token != "" and token is not None:
+            hashval = token
+
+        if secret != "" and secret is not None:
+            hashval = get_jwt_token(secret, event_type, event_ts)
+
+        if hashval:
+            http_headers["Authorization"] = "Bearer " + hashval
+
+        verify = True
+        while True:
+            try:
+                resp = requests.post(url, headers=http_headers,
+                                     data=message_json,
+                                     verify=verify)
+                # Successful webhook push
+                message_queue.task_done()
+                if resp.status_code != 200:
+                    logger.warn("Event push failed to URL: {url}, "
+                                "Event: {event}, "
+                                "Status Code: {status_code}".format(
+                                    url=url,
+                                    event=message_json,
+                                    status_code=resp.status_code))
+                break
+            except requests.exceptions.SSLError as e:
+                # If verify is equal to cert path, but still failed with
+                # SSLError, Looks like some issue with custom downloaded
+                # certificate, Try with verify = false
+                if verify == cert_path:
+                    logger.warn("Event push failed with certificate, "
+                                "ignoring verification url={0} "
+                                "Error={1}".format(url, e))
+                    verify = False
+                    continue
+
+                # If verify is instance of bool and True, then custom cert
+                # is required, download the cert and retry
+                try:
+                    save_https_cert(domain, port, cert_path)
+                    verify = cert_path
+                except Exception as ex:
+                    verify = False
+                    logger.warn("Unable to get Server certificate, "
+                                "ignoring verification url={0} "
+                                "Error={1}".format(url, ex))
+
+                # Done with collecting cert, continue
+                continue
+            except Exception as e:
+                logger.warn("Event push failed to URL: {url}, "
+                            "Event: {event}, "
+                            "Status: {error}".format(
+                                url=url,
+                                event=message_json,
+                                error=e))
+                message_queue.task_done()
+                break
+
+
+def plugin_webhook(message):
+    message_json = json.dumps(message, sort_keys=True)
+    logger.debug("EVENT: {0}".format(message_json))
+    webhooks_pool.send(message["event"], message["ts"], message_json)
+
+
+class LockedOpen(object):
+
+    def __init__(self, filename, *args, **kwargs):
+        self.filename = filename
+        self.open_args = args
+        self.open_kwargs = kwargs
+        self.fileobj = None
+
+    def __enter__(self):
+        """
+        If two processes compete to update a file, The first process
+        gets the lock and the second process is blocked in the fcntl.flock()
+        call. When first process replaces the file and releases the lock,
+        the already open file descriptor in the second process now points
+        to a  "ghost" file(not reachable by any path name) with old contents.
+        To avoid that conflict, check the fd already opened is same or
+        not. Open new one if not same
+        """
+        f = open(self.filename, *self.open_args, **self.open_kwargs)
+        while True:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            fnew = open(self.filename, *self.open_args, **self.open_kwargs)
+            if os.path.sameopenfile(f.fileno(), fnew.fileno()):
+                fnew.close()
+                break
+            else:
+                f.close()
+                f = fnew
+        self.fileobj = f
+        return f
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        self.fileobj.close()
+
+
+class PidFileLockFailed(Exception):
+    pass
+
+
+class PidFile(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self.pid = os.getpid()
+        self.fh = None
+
+    def cleanup(self, remove_file=True):
+        try:
+            if self.fh is not None:
+                self.fh.close()
+        except IOError as exc:
+            if exc.errno != EBADF:
+                raise
+        finally:
+            if os.path.isfile(self.filename) and remove_file:
+                os.remove(self.filename)
+
+    def __enter__(self):
+        self.fh = open(self.filename, 'a+')
+        try:
+            fcntl.flock(self.fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError as exc:
+            self.cleanup(remove_file=False)
+            raise PidFileLockFailed(exc)
+
+        self.fh.seek(0)
+        self.fh.truncate()
+        self.fh.write("%d\n" % self.pid)
+        self.fh.flush()
+        self.fh.seek(0)
+        return self
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        self.cleanup()
+
+
+def webhook_monitor(proc_queue, webhooks):
+    queues = {}
+    for url, data in webhooks.items():
+        if isinstance(data, str):
+            token = data
+            secret = None
+        else:
+            token = data["token"]
+            secret = data["secret"]
+
+        queues[url] = Queue()
+        t = Thread(target=publish_to_webhook, args=(url, token, secret,
+                                                    queues[url]))
+        t.start()
+
+    # Get the message sent to Process queue and distribute to all thread queues
+    while True:
+        message = proc_queue.get()
+        for _, q in queues.items():
+            q.put(message)
+
+
+class WebhookThreadPool(object):
+    def start(self):
+        # Separate process to emit messages to webhooks
+        # which maintains one thread per webhook. Separate
+        # process is required since on reload we need to stop
+        # and start the thread pool. In Python Threads can't be stopped
+        # so terminate the process and start again. Note: In transit
+        # events will be missed during reload
+        self.queue = multiprocessing.Queue()
+        self.proc = multiprocessing.Process(target=webhook_monitor,
+                                            args=(self.queue, _webhooks))
+        self.proc.start()
+
+    def restart(self):
+        # In transit messages are skipped, since webhooks monitor process
+        # is terminated.
+        self.proc.terminate()
+        self.start()
+
+    def send(self, event_type, event_ts, message):
+        self.queue.put((event_type, event_ts, message))
+
+
+def init_webhook_pool():
+    global webhooks_pool
+    webhooks_pool = WebhookThreadPool()
+    webhooks_pool.start()
+
+
+def restart_webhook_pool():
+    global webhooks_pool
+    if webhooks_pool is not None:
+        webhooks_pool.restart()
diff --git a/events/tools/Makefile.am b/events/tools/Makefile.am
new file mode 100644
index 00000000000..fb6770cf070
--- /dev/null
+++ b/events/tools/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST = eventsdash.py
+
+if BUILD_EVENTS
+scriptsdir = $(datadir)/glusterfs/scripts
+scripts_SCRIPTS = eventsdash.py
+endif
diff --git a/events/tools/eventsdash.py b/events/tools/eventsdash.py
new file mode 100644
index 00000000000..6479ea59da6
--- /dev/null
+++ b/events/tools/eventsdash.py
@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+import logging
+from datetime import datetime
+
+from flask import Flask, request
+
+app = Flask(__name__)
+app.logger.disabled = True
+log = logging.getLogger('werkzeug')
+log.disabled = True
+
+
+def human_time(ts):
+    return datetime.fromtimestamp(float(ts)).strftime("%Y-%m-%d %H:%M:%S")
+
+
+@app.route("/")
+def home():
+    return "OK"
+
+
+@app.route("/listen", methods=["POST"])
+def listen():
+    data = request.json
+    if data is None:
+        return "OK"
+
+    message = []
+    for k, v in data.get("message", {}).items():
+        message.append("{0}={1}".format(k, v))
+
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
+        human_time(data.get("ts")),
+        data.get("event"),
+        data.get("nodeid"),
+        " ".join(message))))
+
+    return "OK"
+
+
+def main():
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
+                            description=__doc__)
+    parser.add_argument("--port", type=int, help="Port", default=9000)
+    parser.add_argument("--debug", help="Run Server in debug mode",
+                        action="store_true")
+    args = parser.parse_args()
+
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
+        "TIMESTAMP", "EVENT", "NODE ID", "MESSAGE"
+    )))
+    print(("{0:20s} {1:20s} {2:36} {3}".format(
+        "-"*20, "-"*20, "-"*36, "-"*20
+    )))
+    if args.debug:
+        app.debug = True
+
+    app.run(host="0.0.0.0", port=args.port)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/extras/LinuxRPM/Makefile.am b/extras/LinuxRPM/Makefile.am
new file mode 100644
index 00000000000..f02853798c0
--- /dev/null
+++ b/extras/LinuxRPM/Makefile.am
@@ -0,0 +1,55 @@
+
+GFS_TAR = ../../glusterfs-$(VERSION).tar.gz
+
+.PHONY: all
+
+all:
+	@echo "To build RPMS run 'make glusterrpms'"
+
+.PHONY: glusterrpms glusterrpms_without_autogen
+.PHONY: autogen prep srcrpm testsrpm clean
+
+glusterrpms: autogen glusterrpms_without_autogen
+
+glusterrpms_without_autogen: prep srcrpm rpms
+	-rm -rf rpmbuild
+
+autogen:
+	cd ../.. && \
+	rm -rf autom4te.cache && \
+	./autogen.sh && \
+	./configure --enable-gnfs --with-previous-options
+
+prep:
+	$(MAKE) -C ../.. dist;
+	-mkdir -p rpmbuild/BUILD
+	-mkdir -p rpmbuild/SPECS
+	-mkdir -p rpmbuild/RPMS
+	-mkdir -p rpmbuild/SRPMS
+	-mkdir -p rpmbuild/SOURCES
+	-rm -rf rpmbuild/SOURCES/*
+	cp ../../*.tar.gz ./rpmbuild/SOURCES
+	cp ../../glusterfs.spec ./rpmbuild/SPECS
+
+srcrpm:
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bs rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/SRPMS/* .
+
+rpms:
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' --with gnfs -bb rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/RPMS/*/* .
+
+# EPEL-5 does not like new versions of rpmbuild and requires some
+# _source_* defines
+
+testsrpm: prep
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' \
+		--define '_source_payload w9.gzdio' \
+		--define '_source_filedigest_algorithm 1' \
+		-bs rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/SRPMS/* ../..
+	-rm -rf rpmbuild
+
+clean:
+	-rm -rf rpmbuild
+	-rm -f *.rpm
diff --git a/extras/LinuxRPM/make_glusterrpms b/extras/LinuxRPM/make_glusterrpms
new file mode 100755
index 00000000000..3156af97870
--- /dev/null
+++ b/extras/LinuxRPM/make_glusterrpms
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+(
+    cd $(realpath $(dirname $0))/../.. || exit 1
+    rm -rf autom4te.cache
+    ./autogen.sh || exit 1
+    ./configure --with-previous-options || exit 1
+)
+make -C $(realpath $(dirname $0)) glusterrpms_without_autogen
diff --git a/extras/MacOSX/Portfile b/extras/MacOSX/Portfile
deleted file mode 100644
index 1262a44daea..00000000000
--- a/extras/MacOSX/Portfile
+++ /dev/null
@@ -1,26 +0,0 @@
-# $Id$
-
-PortSystem          1.0
-
-name                glusterfs
-version             2.0.0rc8
-categories          fuse
-maintainers         amar@gluster.com
-description         GlusterFS
-long_description    GlusterFS is a cluster file system, flexible to tune it for your needs.
-homepage            http://www.gluster.org/
-platforms           darwin
-master_sites        http://ftp.gluster.com/pub/gluster/glusterfs/2.0/2.0.0
-
-configure.args      --disable-bdb
-checksums           md5 33c2d02344d4fab422e80cfb637e0b48
-
-post-destroot {
-    file mkdir ${destroot}/Library/LaunchDaemons/
-    file copy ${worksrcpath}/extras/glusterfs-server.plist \
-        ${destroot}/Library/LaunchDaemons/com.gluster.glusterfs.plist
-
-    file mkdir ${destroot}/sbin/
-    file copy ${worksrcpath}/xlators/mount/fuse/utils/mount_glusterfs \
-	${destroot}/sbin/
-}
-\ No newline at end of file
diff --git a/extras/MacOSX/README.MacOSX b/extras/MacOSX/README.MacOSX
deleted file mode 100644
index ec7abc2cc78..00000000000
--- a/extras/MacOSX/README.MacOSX
+++ /dev/null
@@ -1,84 +0,0 @@
-
-Mostly the usage is over OS-X.
-
-Important links
-
-GlusterFS
-=========
-* http://www.gluster.org
-* http://gluster.org/docs/index.php/GlusterFS
-* http://gluster.org/docs/index.php/GlusterFS_on_MAC_OS_X
-
-MacFUSE
-=======
-* http://code.google.com/p/macfuse/
-* http://code.google.com/p/macfuse/wiki/FAQ
-
-
-Important steps:
-================
-
-Make sure that there is no previous installation of MacFUSE exists in the 
-system. Run,
-
-  bash# sudo /Library/Filesystems/fusefs.fs/Support/uninstall-macfuse-core.sh
-
-After this, install MacFUSE (mostly through .dmg available in macfuse homepage 
-or if Gluster Inc provides any custom built .dmg)
-
-Make sure the .dmg of glusterfs is built against the installed MacFUSE version
-(if not, any operations over mountpoint gives EIO ie, Input/Output Error). If 
-glusterfs tarball is used then compile it only after the MacFUSE installation 
-is complete.
-
-To make an entry in /etc/fstab for glusterfs mount, use 'vifs' command
-
-   bash# sudo vifs
-
-after the entry is added in /etc/fstab, it can be mounted by 'mount' command.
-
-To start the server process one can use the 'launchd' mechanism. Follow below 
-steps after installation
-
-   bash# launchctl load /Library/LaunchDaemons/com.gluster.glusterfs.plist
-   
-No need to run the command if the machine reboots, as it will be loaded 
-automatically by launchd process.
-
-Now copy the server volume file in the proper path
-
-   bash# sudo vi /opt/local/etc/glusterfs/server.vol
-
-NOTE: (If glusterfs is installed in different path other than '/opt/local' 
-      update the volume file at the corresponding  path, and  also need to 
-      update the /Library/LaunchDaemons/com.gluster.glusterfs.plist with 
-      the proper path)
-
-Once the volume file is updated, administrator can start the server process by
-running,
-
-  bash# launchctl start com.gluster.glusterfs
-
-and stop like
-
-  bash# launchctl stop com.gluster.glusterfs
-
-NOTE: To start the process by default when the loaded, one need to add the 
-following lines to .plist file
-      -----
-      <key>KeepAlive</key> 
-      <true/>
-      -----
-
-
-
-
-Install using .dmg in Terminal
-=================================
-
-  bash# hdiutil attach <package>.dmg
-  bash# cd /Volumes/<package>/
-  bash# installer -pkg <package>.pkg -installer /
-  bash# cd 
-  bash# hdiutil detach /Volumes/<package>/
-
diff --git a/extras/Makefile.am b/extras/Makefile.am
index 26e1e24b675..983f014cca6 100644
--- a/extras/Makefile.am
+++ b/extras/Makefile.am
@@ -1,11 +1,77 @@
+addonexecdir = $(GLUSTERFS_LIBEXECDIR)
+addonexec_SCRIPTS =
+if WITH_SERVER
+addonexec_SCRIPTS += peer_add_secret_pub
+if USE_SYSTEMD
+addonexec_SCRIPTS += mount-shared-storage.sh
+endif
+endif
 
-docdir = $(datadir)/doc/glusterfs/
-EditorModedir = $(docdir)/
+EditorModedir = $(docdir)
 EditorMode_DATA = glusterfs-mode.el glusterfs.vim
 
-dist_bin_SCRIPTS = glusterfs-defrag
+SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM \
+          $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils python \
+		  ganesha
 
-SUBDIRS = init.d benchmarking volgen
+confdir = $(sysconfdir)/glusterfs
+if WITH_SERVER
+conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
+	    logger.conf.example glusterfs-georep-logrotate group-virt.example \
+	    group-metadata-cache group-gluster-block group-nl-cache \
+	    group-db-workload group-distributed-virt group-samba
+endif
 
-EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh glusterfs-defrag disk_usage_sync.sh
+voldir = $(sysconfdir)/glusterfs
+vol_DATA = thin-arbiter/thin-arbiter.vol
+if WITH_SERVER
+vol_DATA += glusterd.vol
+endif
 
+scriptsdir = $(datadir)/glusterfs/scripts
+scripts_SCRIPTS = thin-arbiter/setup-thin-arbiter.sh
+if WITH_SERVER
+scripts_SCRIPTS += post-upgrade-script-for-quota.sh \
+	pre-upgrade-script-for-quota.sh stop-all-gluster-processes.sh
+if USE_SYSTEMD
+scripts_SCRIPTS += control-cpu-load.sh
+scripts_SCRIPTS += control-mem.sh
+endif
+endif
+
+EXTRA_DIST = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
+	logger.conf.example glusterfs-georep-logrotate group-virt.example \
+	group-metadata-cache group-gluster-block group-nl-cache \
+	group-db-workload group-samba specgen.scm glusterfs-mode.el glusterfs.vim \
+	migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \
+	backend-cleanup.sh disk_usage_sync.sh clear_xattrs.sh \
+	glusterd-sysconfig glusterd.vol post-upgrade-script-for-quota.sh \
+	pre-upgrade-script-for-quota.sh command-completion/gluster.bash \
+	command-completion/Makefile command-completion/README \
+	stop-all-gluster-processes.sh clang-checker.sh mount-shared-storage.sh \
+	control-cpu-load.sh control-mem.sh group-distributed-virt \
+	thin-arbiter/thin-arbiter.vol thin-arbiter/setup-thin-arbiter.sh
+
+if WITH_SERVER
+install-data-local:
+	if [ -n "$(tmpfilesdir)" ]; then \
+		$(mkdir_p) $(DESTDIR)$(tmpfilesdir); \
+		$(INSTALL_DATA) run-gluster.tmpfiles \
+			$(DESTDIR)$(tmpfilesdir)/gluster.conf; \
+	fi
+	$(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)/groups
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-virt.example \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/virt
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-metadata-cache \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/metadata-cache
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-gluster-block \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/gluster-block
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-nl-cache \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/nl-cache
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-db-workload \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/db-workload
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-distributed-virt \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/distributed-virt
+	$(INSTALL_DATA) $(top_srcdir)/extras/group-samba \
+		$(DESTDIR)$(GLUSTERD_WORKDIR)/groups/samba
+endif
diff --git a/extras/Solaris/README.solaris b/extras/Solaris/README.solaris
index ddb70c22c0e..5a4e4336d24 100644
--- a/extras/Solaris/README.solaris
+++ b/extras/Solaris/README.solaris
@@ -10,7 +10,7 @@ $ which glusterfs
 $ glusterfs --version
 glusterfs 2.0.0rc1 built on Jan 16 2009 03:36:59
 Repository revision: glusterfs--mainline--3.0--patch-844
-Copyright (c) 2006, 2007, 2008 Gluster Inc. <http://www.gluster.com>
+Copyright (c) 2006-2011 Gluster Inc. <http://www.gluster.com>
 GlusterFS comes with ABSOLUTELY NO WARRANTY.
 You may redistribute copies of GlusterFS under the terms of the GNU General Public License.
 
diff --git a/extras/Ubuntu/README.Ubuntu b/extras/Ubuntu/README.Ubuntu
new file mode 100644
index 00000000000..890da3ca614
--- /dev/null
+++ b/extras/Ubuntu/README.Ubuntu
@@ -0,0 +1,26 @@
+Bug 765014 - Mounting from localhost in fstab fails at boot on ubuntu
+(original bug: https://bugzilla.redhat.com/show_bug.cgi?id=765014)
+(updated in: https://bugzilla.redhat.com/show_bug.cgi?id=1047007)
+(https://bugs.launchpad.net/ubuntu/+source/glusterfs/+bug/876648)
+
+Ubuntu uses upstart instead of init to bootstrap the system and it has a unique
+way of handling fstab, using a program called mountall(8).  As a result of using
+a debian initscript to start glusterd, glusterfs mounts in fstab are tried before
+the glusterd service is running.  In the case where the client is also a server
+and the volume is mounted from localhost, the mount fails at boot time.  To
+correct this we need to launch glusterd using upstart and block the glusterfs
+mounting event until glusterd is started.
+
+The glusterfs-server.conf file contains the necessary configuration for upstart to
+manage the glusterd service.  It should be placed in /etc/init/glusterfs-server.conf
+on Ubuntu systems, and then the old initscript /etc/init.d/glusterd can be
+removed.  An additional upstart job, mounting-glusterfs.conf, is also required
+to block mounting glusterfs volumes until the network interfaces are available.
+Both of these upstart jobs need to be placed in /etc/init to resolve the issue.
+
+Starting with Ubuntu 14.04, Trusty Tahr, these upstart jobs will be included 
+with the glusterfs-server and glusterfs-client packages in the Ubuntu 
+universe repository.
+
+This affects all versions of glusterfs on the Ubuntu platform since at least
+10.04, Lucid Lynx.
diff --git a/extras/Ubuntu/glusterfs-server.conf b/extras/Ubuntu/glusterfs-server.conf
new file mode 100644
index 00000000000..aa99502b0a7
--- /dev/null
+++ b/extras/Ubuntu/glusterfs-server.conf
@@ -0,0 +1,10 @@
+author "Louis Zuckerman <me@louiszuckerman.com>"
+description "GlusterFS Management Daemon"
+
+start on runlevel [2345]
+stop on runlevel [016]
+
+expect fork
+
+exec /usr/sbin/glusterd -p /var/run/glusterd.pid
+
diff --git a/extras/Ubuntu/mounting-glusterfs.conf b/extras/Ubuntu/mounting-glusterfs.conf
new file mode 100644
index 00000000000..786ef16df04
--- /dev/null
+++ b/extras/Ubuntu/mounting-glusterfs.conf
@@ -0,0 +1,6 @@
+author "Louis Zuckerman <me@louiszuckerman.com>"
+description "Block the mounting event for glusterfs filesystems until the network interfaces are running"
+
+start on mounting TYPE=glusterfs
+task
+exec start wait-for-state WAIT_FOR=static-network-up WAITER=mounting-glusterfs
diff --git a/extras/backend-cleanup.sh b/extras/backend-cleanup.sh
index dc504860b73..cb0a1d8871f 100644
--- a/extras/backend-cleanup.sh
+++ b/extras/backend-cleanup.sh
@@ -17,7 +17,7 @@ export_directory="/export/glusterfs"
 clean_dir()
 {
     # Clean the 'link' files on backend
-    find "${export_directory}" -type f -perm +01000 -exec rm -v '{}' \; 
+    find "${export_directory}" -type f -perm /01000 -exec rm -v '{}' \;
 }
 
 main()
diff --git a/extras/benchmarking/Makefile.am b/extras/benchmarking/Makefile.am
index 04cc06182ab..bfcc59277a5 100644
--- a/extras/benchmarking/Makefile.am
+++ b/extras/benchmarking/Makefile.am
@@ -1,7 +1,5 @@
 
-docdir = $(datadir)/doc/$(PACKAGE_NAME)/benchmarking
-
-benchmarkingdir = $(docdir)
+benchmarkingdir = $(docdir)/benchmarking
 
 benchmarking_DATA = rdd.c glfs-bm.c README launch-script.sh local-script.sh
 
diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c
index 7a4eabcae2b..f7f5873f84d 100644
--- a/extras/benchmarking/glfs-bm.c
+++ b/extras/benchmarking/glfs-bm.c
@@ -1,21 +1,12 @@
-/*                                                                  
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com> 
-  This file is part of GlusterFS.                             
-  GlusterFS is free software; you can redistribute it and/or modify 
-  it under the terms of the GNU General Public License as published 
-  by the Free Software Foundation; either version 3 of the License, 
-  or (at your option) any later version.                            
-
-  GlusterFS is distributed in the hope that it will be useful, but 
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details. 
-                                                                                   
-  You should have received a copy of the GNU General Public License 
-  along with this program.  If not, see 
-  <http://www.gnu.org/licenses/>.  
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #define _GNU_SOURCE
 #define __USE_FILE_OFFSET64
 #define _FILE_OFFSET_BITS 64
@@ -34,365 +25,338 @@
 #include <sys/time.h>
 
 struct state {
-        char need_op_write:1;
-        char need_op_read:1;
+    char need_op_write : 1;
+    char need_op_read : 1;
 
-        char need_iface_fileio:1;
-        char need_iface_xattr:1;
+    char need_iface_fileio : 1;
+    char need_iface_xattr : 1;
 
-        char need_mode_posix:1;
+    char need_mode_posix : 1;
 
-        char prefix[512];
-        long int count;
+    char prefix[512];
+    long int count;
 
-        size_t block_size;
+    size_t block_size;
 
-        char *specfile;
+    char *specfile;
 
-        long int io_size;
+    long int io_size;
 };
 
-
-#define MEASURE(func, arg) measure (func, #func, arg)
-
+#define MEASURE(func, arg) measure(func, #func, arg)
 
 void
-tv_difference (struct timeval *tv_stop,
-               struct timeval *tv_start,
-               struct timeval *tv_diff)
+tv_difference(struct timeval *tv_stop, struct timeval *tv_start,
+              struct timeval *tv_diff)
 {
-        if (tv_stop->tv_usec < tv_start->tv_usec) {
-                tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec;
-                tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec);
-        } else {
-                tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec;
-                tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec;
-        }
+    if (tv_stop->tv_usec < tv_start->tv_usec) {
+        tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec;
+        tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec);
+    } else {
+        tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec;
+        tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec;
+    }
 }
 
-
 void
-measure (int (*func)(struct state *state),
-         char *func_name, struct state *state)
+measure(int (*func)(struct state *state), char *func_name, struct state *state)
 {
-        struct timeval tv_start, tv_stop, tv_diff;
-        state->io_size = 0;
-        long int count;
+    struct timeval tv_start, tv_stop, tv_diff;
+    state->io_size = 0;
+    long int count;
 
-        gettimeofday (&tv_start, NULL);
-        count = func (state);
-        gettimeofday (&tv_stop, NULL);
+    gettimeofday(&tv_start, NULL);
+    count = func(state);
+    gettimeofday(&tv_stop, NULL);
 
-        tv_difference (&tv_stop, &tv_start, &tv_diff);
+    tv_difference(&tv_stop, &tv_start, &tv_diff);
 
-        fprintf (stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n",
-                 func_name, count, state->io_size,
-                 tv_diff.tv_sec, tv_diff.tv_usec);
+    fprintf(stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", func_name, count,
+            state->io_size, tv_diff.tv_sec, tv_diff.tv_usec);
 }
 
-
 static error_t
-parse_opts (int key, char *arg,
-            struct argp_state *_state)
+parse_opts(int key, char *arg, struct argp_state *_state)
 {
-        struct state *state = _state->input;
+    struct state *state = _state->input;
 
-        switch (key)
-        {
+    switch (key) {
         case 'o':
-                if (strcasecmp (arg, "read") == 0) {
-                        state->need_op_write = 0;
-                        state->need_op_read = 1;
-                } else if (strcasecmp (arg, "write") == 0) {
-                        state->need_op_write = 1;
-                        state->need_op_read = 0;
-                } else if (strcasecmp (arg, "both") == 0) {
-                        state->need_op_write = 1;
-                        state->need_op_read = 1;
-                } else {
-                        fprintf (stderr, "unknown op: %s\n", arg);
-                        return -1;
-                }
-                break;
+            if (strcasecmp(arg, "read") == 0) {
+                state->need_op_write = 0;
+                state->need_op_read = 1;
+            } else if (strcasecmp(arg, "write") == 0) {
+                state->need_op_write = 1;
+                state->need_op_read = 0;
+            } else if (strcasecmp(arg, "both") == 0) {
+                state->need_op_write = 1;
+                state->need_op_read = 1;
+            } else {
+                fprintf(stderr, "unknown op: %s\n", arg);
+                return -1;
+            }
+            break;
         case 'i':
-                if (strcasecmp (arg, "fileio") == 0) {
-                        state->need_iface_fileio = 1;
-                        state->need_iface_xattr = 0;
-                } else if (strcasecmp (arg, "xattr") == 0) {
-                        state->need_iface_fileio = 0;
-                        state->need_iface_xattr = 1;
-                } else if (strcasecmp (arg, "both") == 0) {
-                        state->need_iface_fileio = 1;
-                        state->need_iface_xattr = 1;
-                } else {
-                        fprintf (stderr, "unknown interface: %s\n", arg);
-                        return -1;
-                }
-                break;
-        case 'b':
-        {
-                size_t block_size = atoi (arg);
-                if (!block_size) {
-                        fprintf (stderr, "incorrect size: %s\n", arg);
-                        return -1;
-                }
-                state->block_size = block_size;
-        }
-        break;
+            if (strcasecmp(arg, "fileio") == 0) {
+                state->need_iface_fileio = 1;
+                state->need_iface_xattr = 0;
+            } else if (strcasecmp(arg, "xattr") == 0) {
+                state->need_iface_fileio = 0;
+                state->need_iface_xattr = 1;
+            } else if (strcasecmp(arg, "both") == 0) {
+                state->need_iface_fileio = 1;
+                state->need_iface_xattr = 1;
+            } else {
+                fprintf(stderr, "unknown interface: %s\n", arg);
+                return -1;
+            }
+            break;
+        case 'b': {
+            size_t block_size = atoi(arg);
+            if (!block_size) {
+                fprintf(stderr, "incorrect size: %s\n", arg);
+                return -1;
+            }
+            state->block_size = block_size;
+        } break;
         case 's':
-                state->specfile = strdup (arg);
-                break;
+            state->specfile = strdup(arg);
+            break;
         case 'p':
-                fprintf (stderr, "using prefix: %s\n", arg);
-                strncpy (state->prefix, arg, 512);
-                break;
-        case 'c':
-        {
-                long count = atol (arg);
-                if (!count) {
-                        fprintf (stderr, "incorrect count: %s\n", arg);
-                        return -1;
-                }
-                state->count = count;
-        }
-        break;
+            fprintf(stderr, "using prefix: %s\n", arg);
+            strncpy(state->prefix, arg, 512);
+            break;
+        case 'c': {
+            long count = atol(arg);
+            if (!count) {
+                fprintf(stderr, "incorrect count: %s\n", arg);
+                return -1;
+            }
+            state->count = count;
+        } break;
         case ARGP_KEY_NO_ARGS:
-                break;
+            break;
         case ARGP_KEY_ARG:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-do_mode_posix_iface_fileio_write (struct state *state)
+do_mode_posix_iface_fileio_write(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-
-        for (i=0; i<state->count; i++) {
-                int fd = -1;
-                char filename[512];
-
-                sprintf (filename, "%s.%06ld", state->prefix, i);
-
-                fd = open (filename, O_CREAT|O_WRONLY, 00600);
-                if (fd == -1) {
-                        fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno));
-                        break;
-                }
-                ret = write (fd, block, state->block_size);
-                if (ret != state->block_size) {
-                        fprintf (stderr, "write (%s) => %d/%s\n", filename, ret,
-                                 strerror (errno));
-                        close (fd);
-                        break;
-                }
-                close (fd);
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+
+    for (i = 0; i < state->count; i++) {
+        int fd = -1;
+        char filename[512];
+
+        sprintf(filename, "%s.%06ld", state->prefix, i);
+
+        fd = open(filename, O_CREAT | O_WRONLY, 00600);
+        if (fd == -1) {
+            fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno));
+            break;
         }
+        ret = write(fd, block, state->block_size);
+        if (ret != state->block_size) {
+            fprintf(stderr, "write (%s) => %d/%s\n", filename, ret,
+                    strerror(errno));
+            close(fd);
+            break;
+        }
+        close(fd);
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_fileio_read (struct state *state)
+do_mode_posix_iface_fileio_read(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-
-        for (i=0; i<state->count; i++) {
-                int fd = -1;
-                char filename[512];
-
-                sprintf (filename, "%s.%06ld", state->prefix, i);
-
-                fd = open (filename, O_RDONLY);
-                if (fd == -1) {
-                        fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno));
-                        break;
-                }
-                ret = read (fd, block, state->block_size);
-                if (ret == -1) {
-                        fprintf (stderr, "read(%s) => %d/%s\n", filename, ret, strerror (errno));
-                        close (fd);
-                        break;
-                }
-                close (fd);
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+
+    for (i = 0; i < state->count; i++) {
+        int fd = -1;
+        char filename[512];
+
+        sprintf(filename, "%s.%06ld", state->prefix, i);
+
+        fd = open(filename, O_RDONLY);
+        if (fd == -1) {
+            fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno));
+            break;
+        }
+        ret = read(fd, block, state->block_size);
+        if (ret == -1) {
+            fprintf(stderr, "read(%s) => %d/%s\n", filename, ret,
+                    strerror(errno));
+            close(fd);
+            break;
         }
+        close(fd);
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_fileio (struct state *state)
+do_mode_posix_iface_fileio(struct state *state)
 {
-        if (state->need_op_write)
-                MEASURE (do_mode_posix_iface_fileio_write, state);
+    if (state->need_op_write)
+        MEASURE(do_mode_posix_iface_fileio_write, state);
 
-        if (state->need_op_read)
-                MEASURE (do_mode_posix_iface_fileio_read, state);
+    if (state->need_op_read)
+        MEASURE(do_mode_posix_iface_fileio_read, state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-do_mode_posix_iface_xattr_write (struct state *state)
+do_mode_posix_iface_xattr_write(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-        char *dname = NULL, *dirc = NULL;
-        char *bname = NULL, *basec = NULL;
-
-        dirc = strdup (state->prefix);
-        basec = strdup (state->prefix);
-        dname = dirname (dirc);
-        bname = basename (basec);
-
-        for (i=0; i<state->count; i++) {
-                char key[512];
-
-                sprintf (key, "glusterfs.file.%s.%06ld", bname, i);
-
-                ret = lsetxattr (dname, key, block, state->block_size, 0);
-
-                if (ret != 0) {
-                        fprintf (stderr, "lsetxattr (%s, %s, %p) => %s\n",
-                                 dname, key, block, strerror (errno));
-                        break;
-                }
-                state->io_size += state->block_size;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+    char *dname = NULL, *dirc = NULL;
+    char *bname = NULL, *basec = NULL;
+
+    dirc = strdup(state->prefix);
+    basec = strdup(state->prefix);
+    dname = dirname(dirc);
+    bname = basename(basec);
+
+    for (i = 0; i < state->count; i++) {
+        char key[512];
+
+        sprintf(key, "glusterfs.file.%s.%06ld", bname, i);
+
+        ret = lsetxattr(dname, key, block, state->block_size, 0);
+
+        if (ret != 0) {
+            fprintf(stderr, "lsetxattr (%s, %s, %p) => %s\n", dname, key, block,
+                    strerror(errno));
+            break;
         }
+        state->io_size += state->block_size;
+    }
 
-        free (dirc);
-        free (basec);
+    free(dirc);
+    free(basec);
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_xattr_read (struct state *state)
+do_mode_posix_iface_xattr_read(struct state *state)
 {
-        long int i;
-        int ret = -1;
-        char block[state->block_size];
-        char *dname = NULL, *dirc = NULL;
-        char *bname = NULL, *basec = NULL;
-
-        dirc = strdup (state->prefix);
-        basec = strdup (state->prefix);
-        dname = dirname (dirc);
-        bname = basename (basec);
-
-        for (i=0; i<state->count; i++) {
-                char key[512];
-
-                sprintf (key, "glusterfs.file.%s.%06ld", bname, i);
-
-                ret = lgetxattr (dname, key, block, state->block_size);
-
-                if (ret < 0) {
-                        fprintf (stderr, "lgetxattr (%s, %s, %p) => %s\n",
-                                 dname, key, block, strerror (errno));
-                        break;
-                }
-                state->io_size += ret;
+    long int i;
+    int ret = -1;
+    char block[state->block_size];
+    char *dname = NULL, *dirc = NULL;
+    char *bname = NULL, *basec = NULL;
+
+    dirc = strdup(state->prefix);
+    basec = strdup(state->prefix);
+    dname = dirname(dirc);
+    bname = basename(basec);
+
+    for (i = 0; i < state->count; i++) {
+        char key[512];
+
+        sprintf(key, "glusterfs.file.%s.%06ld", bname, i);
+
+        ret = lgetxattr(dname, key, block, state->block_size);
+
+        if (ret < 0) {
+            fprintf(stderr, "lgetxattr (%s, %s, %p) => %s\n", dname, key, block,
+                    strerror(errno));
+            break;
         }
+        state->io_size += ret;
+    }
 
-        return i;
+    return i;
 }
 
-
 int
-do_mode_posix_iface_xattr (struct state *state)
+do_mode_posix_iface_xattr(struct state *state)
 {
-        if (state->need_op_write)
-                MEASURE (do_mode_posix_iface_xattr_write, state);
+    if (state->need_op_write)
+        MEASURE(do_mode_posix_iface_xattr_write, state);
 
-        if (state->need_op_read)
-                MEASURE (do_mode_posix_iface_xattr_read, state);
+    if (state->need_op_read)
+        MEASURE(do_mode_posix_iface_xattr_read, state);
 
-        return 0;
+    return 0;
 }
 
 int
-do_mode_posix (struct state *state)
+do_mode_posix(struct state *state)
 {
-        if (state->need_iface_fileio)
-                do_mode_posix_iface_fileio (state);
+    if (state->need_iface_fileio)
+        do_mode_posix_iface_fileio(state);
 
-        if (state->need_iface_xattr)
-                do_mode_posix_iface_xattr (state);
+    if (state->need_iface_xattr)
+        do_mode_posix_iface_xattr(state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-do_actions (struct state *state)
+do_actions(struct state *state)
 {
-        if (state->need_mode_posix)
-                do_mode_posix (state);
+    if (state->need_mode_posix)
+        do_mode_posix(state);
 
-        return 0;
+    return 0;
 }
 
 static struct argp_option options[] = {
-        {"op", 'o', "OPERATIONS", 0,
-         "WRITE|READ|BOTH - defaults to BOTH"},
-        {"iface", 'i', "INTERFACE", 0,
-         "FILEIO|XATTR|BOTH - defaults to FILEIO"},
-        {"block", 'b', "BLOCKSIZE", 0,
-         "<NUM> - defaults to 4096"},
-        {"specfile", 's', "SPECFILE", 0,
-         "absolute path to specfile"},
-        {"prefix", 'p', "PREFIX", 0,
-         "filename prefix"},
-        {"count", 'c', "COUNT", 0,
-         "number of files"},
-        {0, 0, 0, 0, 0}
-};
+    {"op", 'o', "OPERATIONS", 0, "WRITE|READ|BOTH - defaults to BOTH"},
+    {"iface", 'i', "INTERFACE", 0, "FILEIO|XATTR|BOTH - defaults to FILEIO"},
+    {"block", 'b', "BLOCKSIZE", 0, "<NUM> - defaults to 4096"},
+    {"specfile", 's', "SPECFILE", 0, "absolute path to specfile"},
+    {"prefix", 'p', "PREFIX", 0, "filename prefix"},
+    {"count", 'c', "COUNT", 0, "number of files"},
+    {0, 0, 0, 0, 0}};
 
-static struct argp argp = {
-        options,
-        parse_opts,
-        "tool",
-        "tool to benchmark small file performance"
-};
+static struct argp argp = {options, parse_opts, "tool",
+                           "tool to benchmark small file performance"};
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        struct state state = {0, };
+    struct state state = {
+        0,
+    };
 
-        state.need_op_write = 1;
-        state.need_op_read = 1;
+    state.need_op_write = 1;
+    state.need_op_read = 1;
 
-        state.need_iface_fileio = 1;
-        state.need_iface_xattr = 0;
+    state.need_iface_fileio = 1;
+    state.need_iface_xattr = 0;
 
-        state.need_mode_posix = 1;
+    state.need_mode_posix = 1;
 
-        state.block_size = 4096;
+    state.block_size = 4096;
 
-        strcpy (state.prefix, "tmpfile");
-        state.count = 1048576;
+    strcpy(state.prefix, "tmpfile");
+    state.count = 1048576;
 
-        if (argp_parse (&argp, argc, argv, 0, 0, &state) != 0) {
-                fprintf (stderr, "argp_parse() failed\n");
-                return 1;
-        }
+    if (argp_parse(&argp, argc, argv, 0, 0, &state) != 0) {
+        fprintf(stderr, "argp_parse() failed\n");
+        return 1;
+    }
 
-        do_actions (&state);
+    do_actions(&state);
 
-        return 0;
+    return 0;
 }
diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c
index dc7aafa76db..efc9d342a37 100644
--- a/extras/benchmarking/rdd.c
+++ b/extras/benchmarking/rdd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -30,463 +20,586 @@
 
 #define TWO_POWER(power) (2UL << (power))
 
-#define RDD_INTEGER_VALUE ((TWO_POWER ((sizeof (int) * 8))) - 1)
+#define RDD_INTEGER_VALUE ((TWO_POWER((sizeof(int) * 8))) - 1)
 
 #ifndef UNIX_PATH_MAX
 #define UNIX_PATH_MAX 108
 #endif
 
+#define UNIT_KB 1024ULL
+#define UNIT_MB UNIT_KB * 1024ULL
+#define UNIT_GB UNIT_MB * 1024ULL
+#define UNIT_TB UNIT_GB * 1024ULL
+#define UNIT_PB UNIT_TB * 1024ULL
+
+#define UNIT_KB_STRING "KB"
+#define UNIT_MB_STRING "MB"
+#define UNIT_GB_STRING "GB"
+#define UNIT_TB_STRING "TB"
+#define UNIT_PB_STRING "PB"
+
 struct rdd_file {
-	char path[UNIX_PATH_MAX];
-	struct stat st;
-	int fd;
+    char path[UNIX_PATH_MAX];
+    struct stat st;
+    int fd;
 };
- 
+
 struct rdd_config {
-	long iters;
-	long max_ops_per_seq;
-	size_t max_bs;
-	size_t min_bs;
-	int thread_count;
-	pthread_t *threads;
-	pthread_barrier_t barrier;
-	pthread_mutex_t lock;
-	struct rdd_file in_file;
-	struct rdd_file out_file;
+    long iters;
+    long max_ops_per_seq;
+    size_t max_bs;
+    size_t min_bs;
+    int thread_count;
+    pthread_t *threads;
+    pthread_barrier_t barrier;
+    pthread_mutex_t lock;
+    struct rdd_file in_file;
+    struct rdd_file out_file;
+    ssize_t file_size;
 };
 static struct rdd_config rdd_config;
 
 enum rdd_keys {
-	RDD_MIN_BS_KEY = 1,
-	RDD_MAX_BS_KEY,
+    RDD_MIN_BS_KEY = 1,
+    RDD_MAX_BS_KEY,
 };
- 
+
 static error_t
-rdd_parse_opts (int key, char *arg,
-		struct argp_state *_state)
+rdd_parse_opts(int key, char *arg, struct argp_state *_state)
 {
-	switch (key) {
-	case 'o':
-	{
-		int len = 0;
-		len = strlen (arg);
-		if (len > UNIX_PATH_MAX) {
-			fprintf (stderr, "output file name too long (%s)\n", 
-                                 arg);
-			return -1;
-		}
-
-		strncpy (rdd_config.out_file.path, arg, len);
-	}
-	break;
-
-	case 'i':
-	{
-		int len = 0;
-		len = strlen (arg);
-		if (len > UNIX_PATH_MAX) {
-			fprintf (stderr, "input file name too long (%s)\n", 
-                                 arg);
-			return -1;
-		}
-
-		strncpy (rdd_config.in_file.path, arg, len);
-	}
-	break;
-
-	case RDD_MIN_BS_KEY:
-	{
-		char *tmp = NULL;
-		long bs = 0;
-		bs = strtol (arg, &tmp, 10);
-		if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
-			fprintf (stderr, "invalid argument for minimum block"
-                                 "size (%s)\n", arg);
-			return -1;
-		}
-
-		rdd_config.min_bs = bs;
-	}
-	break;
-
-	case RDD_MAX_BS_KEY:
-	{
-		char *tmp = NULL;
-		long bs = 0;
-		bs = strtol (arg, &tmp, 10);
-		if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
-			fprintf (stderr, "invalid argument for maximum block"
-                                 "size (%s)\n", arg);
-			return -1;
-		}
-
-		rdd_config.max_bs = bs;
-	}
-	break;
-
-	case 'r':
-	{
-		char *tmp = NULL;
-		long iters = 0;
-		iters = strtol (arg, &tmp, 10);
-		if ((iters == LONG_MAX) || 
-                    (iters == LONG_MIN) || 
-                    (tmp && *tmp)) {
-			fprintf (stderr, "invalid argument for iterations"
-                                 "(%s)\n", arg);
-			return -1;
-		}
-
-		rdd_config.iters = iters;
-	}
-	break;
-
-	case 'm':
-	{
-		char *tmp = NULL;
-		long max_ops = 0;
-		max_ops = strtol (arg, &tmp, 10);
-		if ((max_ops == LONG_MAX) || 
-                    (max_ops == LONG_MIN) || 
-                    (tmp && *tmp)) {
-			fprintf (stderr, "invalid argument for max-ops"
-                                 "(%s)\n", arg);
-			return -1;
-		}
-
-		rdd_config.max_ops_per_seq = max_ops;
-	}
-	break;
-
-	case 't':
-	{
-		char *tmp = NULL;
-		long threads = 0;
-		threads = strtol (arg, &tmp, 10);
-		if ((threads == LONG_MAX) || 
-                    (threads == LONG_MIN) || 
-                    (tmp && *tmp)) {
-			fprintf (stderr, "invalid argument for thread count"
-                                 "(%s)\n", arg);
-			return -1;
-		}
-
-		rdd_config.thread_count = threads;
-	}
-	break;
-
-	case ARGP_KEY_NO_ARGS:
-		break;
-	case ARGP_KEY_ARG:
-		break;
-	case ARGP_KEY_END:
-		if (_state->argc == 1) {
-			argp_usage (_state); 
-		}
-
-	}
-
-	return 0;
+    switch (key) {
+        case 'o': {
+            int len = 0;
+            len = strlen(arg);
+            if (len > UNIX_PATH_MAX) {
+                fprintf(stderr, "output file name too long (%s)\n", arg);
+                return -1;
+            }
+
+            strncpy(rdd_config.out_file.path, arg, len);
+        } break;
+
+        case 'i': {
+            int len = 0;
+            len = strlen(arg);
+            if (len > UNIX_PATH_MAX) {
+                fprintf(stderr, "input file name too long (%s)\n", arg);
+                return -1;
+            }
+
+            strncpy(rdd_config.in_file.path, arg, len);
+            rdd_config.in_file.path[len] = '\0';
+        } break;
+
+        case 'f': {
+            char *tmp = NULL;
+            unsigned long long fs = 0;
+            if (string2bytesize(arg, &fs) == -1) {
+                fprintf(stderr,
+                        "invalid argument for file size "
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.file_size = fs;
+        } break;
+
+        case RDD_MIN_BS_KEY: {
+            char *tmp = NULL;
+            long bs = 0;
+            bs = strtol(arg, &tmp, 10);
+            if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for minimum block"
+                        "size (%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.min_bs = bs;
+        } break;
+
+        case RDD_MAX_BS_KEY: {
+            char *tmp = NULL;
+            long bs = 0;
+            bs = strtol(arg, &tmp, 10);
+            if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for maximum block"
+                        "size (%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.max_bs = bs;
+        } break;
+
+        case 'r': {
+            char *tmp = NULL;
+            long iters = 0;
+            iters = strtol(arg, &tmp, 10);
+            if ((iters == LONG_MAX) || (iters == LONG_MIN) || (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for iterations"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.iters = iters;
+        } break;
+
+        case 'm': {
+            char *tmp = NULL;
+            long max_ops = 0;
+            max_ops = strtol(arg, &tmp, 10);
+            if ((max_ops == LONG_MAX) || (max_ops == LONG_MIN) ||
+                (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for max-ops"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.max_ops_per_seq = max_ops;
+        } break;
+
+        case 't': {
+            char *tmp = NULL;
+            long threads = 0;
+            threads = strtol(arg, &tmp, 10);
+            if ((threads == LONG_MAX) || (threads == LONG_MIN) ||
+                (tmp && *tmp)) {
+                fprintf(stderr,
+                        "invalid argument for thread count"
+                        "(%s)\n",
+                        arg);
+                return -1;
+            }
+
+            rdd_config.thread_count = threads;
+        } break;
+
+        case ARGP_KEY_NO_ARGS:
+            break;
+        case ARGP_KEY_ARG:
+            break;
+        case ARGP_KEY_END:
+            if (_state->argc == 1) {
+                argp_usage(_state);
+            }
+    }
+
+    return 0;
+}
+
+int
+string2bytesize(const char *str, unsigned long long *n)
+{
+    unsigned long long value = 0ULL;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s)) {
+            continue;
+        }
+        if (*s == '-') {
+            return -1;
+        }
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoull(str, &tail, 10);
+
+    if (errno == ERANGE || errno == EINVAL) {
+        return -1;
+    }
+
+    if (errno == 0) {
+        errno = old_errno;
+    }
+
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, UNIT_KB_STRING) == 0) {
+            value *= UNIT_KB;
+        } else if (strcasecmp(tail, UNIT_MB_STRING) == 0) {
+            value *= UNIT_MB;
+        } else if (strcasecmp(tail, UNIT_GB_STRING) == 0) {
+            value *= UNIT_GB;
+        } else if (strcasecmp(tail, UNIT_TB_STRING) == 0) {
+            value *= UNIT_TB;
+        } else if (strcasecmp(tail, UNIT_PB_STRING) == 0) {
+            value *= UNIT_PB;
+        }
+
+        else {
+            return -1;
+        }
+    }
+
+    *n = value;
+
+    return 0;
 }
 
 static struct argp_option rdd_options[] = {
-	{"if", 'i', "INPUT_FILE", 0, "input-file"},
-	{"of", 'o', "OUTPUT_FILE", 0, "output-file"},
-	{"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"},
-	{"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, 
-	 "Minimum block size in bytes (defaults to 1024)"},
-	{"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0,
-	 "Maximum block size in bytes (defaults to 4096)"},
-	{"iters", 'r', "ITERS", 0,
-	 "Number of read-write sequences (defaults to 1000000)"},
-	{"max-ops", 'm', "MAXOPS", 0,
-	 "maximum number of read-writes to be performed in a sequence (defaults to 1)"},
-	{0, 0, 0, 0, 0}
-};
+    {"if", 'i', "INPUT_FILE", 0, "input-file"},
+    {"of", 'o', "OUTPUT_FILE", 0, "output-file"},
+    {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"},
+    {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0,
+     "Minimum block size in bytes (defaults to 1024)"},
+    {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0,
+     "Maximum block size in bytes (defaults to 4096)"},
+    {"iters", 'r', "ITERS", 0,
+     "Number of read-write sequences (defaults to 1000000)"},
+    {"max-ops", 'm', "MAXOPS", 0,
+     "maximum number of read-writes to be performed in a sequence (defaults to "
+     "1)"},
+    {"file-size", 'f', "FILESIZE", 0,
+     "the size of the file which will be created and upon it I/O will be done"
+     " (defaults to 100MB"},
+    {0, 0, 0, 0, 0}};
 
 static struct argp argp = {
-  rdd_options,
-  rdd_parse_opts,
-  "",
-  "random dd - tool to do a sequence of random block-sized continuous"
-  "read writes starting at a random offset"
-};
-
+    rdd_options, rdd_parse_opts, "",
+    "random dd - tool to do a sequence of random block-sized continuous"
+    "read writes starting at a random offset"};
 
 static void
-rdd_default_config (void)
+rdd_default_config(void)
 {
-	rdd_config.thread_count = 2;
-	rdd_config.iters = 1000000;
-	rdd_config.max_bs = 4096;
-	rdd_config.min_bs = 1024;
-	rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-	rdd_config.max_ops_per_seq = 1;
-
-	return;
+    char *tmp_path = "rdd.in";
+
+    rdd_config.thread_count = 2;
+    rdd_config.iters = 1000000;
+    rdd_config.max_bs = 4096;
+    rdd_config.min_bs = 1024;
+    rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
+    rdd_config.max_ops_per_seq = 1;
+    strncpy(rdd_config.in_file.path, tmp_path, strlen(tmp_path));
+    rdd_config.file_size = 104857600;
+
+    return;
 }
 
-
 static char
-rdd_valid_config (void)
+rdd_valid_config(void)
 {
-	char ret = 1;
-	int fd = -1;
-
-	fd = open (rdd_config.in_file.path, O_RDONLY);
-	if (fd == -1) {
-		ret = 0;
-		goto out;
-	}
-	close (fd);
-
-	if (rdd_config.min_bs > rdd_config.max_bs) {
-		ret = 0;
-		goto out;
-	}
-
-	if (strlen (rdd_config.out_file.path) == 0) {
-		sprintf (rdd_config.out_file.path, "%s.rddout", 
-                         rdd_config.in_file.path);
-	}
+    char ret = 1;
+    int fd = -1;
+
+    fd = open(rdd_config.in_file.path, O_RDONLY);
+    if (fd == -1 && (errno != ENOENT)) {
+        fprintf(stderr, "open: (%s)", strerror(errno));
+        ret = 0;
+        goto out;
+    }
+    close(fd);
+
+    if (rdd_config.min_bs > rdd_config.max_bs) {
+        fprintf(stderr,
+                "minimum blocksize %ld is greater than the "
+                "maximum blocksize %ld",
+                rdd_config.min_bs, rdd_config.max_bs);
+        ret = 0;
+        goto out;
+    }
+
+    if (strlen(rdd_config.out_file.path) == 0) {
+        sprintf(rdd_config.out_file.path, "%s.rddout", rdd_config.in_file.path);
+    }
 
 out:
-	return ret;
+    return ret;
 }
 
-
 static void *
-rdd_read_write (void *arg)
+rdd_read_write(void *arg)
 {
-	int i = 0, ret = 0;
-	size_t bs = 0;
-	off_t offset = 0;
-	long rand = 0;
-	long max_ops = 0;
-	char *buf = NULL;
-
-	buf = calloc (1, rdd_config.max_bs);
-	if (!buf) {
-		fprintf (stderr, "calloc failed (%s)\n", strerror (errno));
-		ret = -1;
-		goto out;
-	}
-
-	for (i = 0; i < rdd_config.iters; i++) 
-	{
-		pthread_mutex_lock (&rdd_config.lock);
-		{
-			int bytes = 0;
-			rand = random ();
-			
-			if (rdd_config.min_bs == rdd_config.max_bs) {
-				bs = rdd_config.max_bs;
-			} else {
-				bs = rdd_config.min_bs + 
-                                        (rand % 
-                                         (rdd_config.max_bs - 
-                                          rdd_config.min_bs));
-			}
-			
-			offset = rand % rdd_config.in_file.st.st_size;
-			max_ops = rand % rdd_config.max_ops_per_seq;
-			if (!max_ops) {
-				max_ops ++;
-			}
-
-			ret = lseek (rdd_config.in_file.fd, offset, SEEK_SET);
-			if (ret != offset) {
-				fprintf (stderr, "lseek failed (%s)\n", 
-                                         strerror (errno));
-				ret = -1;
-				goto unlock;
-			}
-
-			ret = lseek (rdd_config.out_file.fd, offset, SEEK_SET);
-			if (ret != offset) {
-				fprintf (stderr, "lseek failed (%s)\n", 
-                                         strerror (errno));
-				ret = -1;
-				goto unlock;
-			}
-
-			while (max_ops--) 
-			{ 
-				bytes = read (rdd_config.in_file.fd, buf, bs);
-				if (!bytes) {
-					break;
-				}
-
-				if (bytes == -1) {
-					fprintf (stderr, "read failed (%s)\n", 
-                                                 strerror (errno));
-					ret = -1;
-					goto unlock;
-				}
-
-				if (write (rdd_config.out_file.fd, buf, bytes) 
-                                    != bytes) {
-					fprintf (stderr, "write failed (%s)\n", 
-                                                 strerror (errno));
-					ret = -1;
-					goto unlock;
-				}
-			}
-		}
-	unlock:
-		pthread_mutex_unlock (&rdd_config.lock);
-		if (ret == -1) {
-			goto out;
-		}
-		ret = 0;
-	}
+    int i = 0, ret = 0;
+    size_t bs = 0;
+    off_t offset = 0;
+    long rand = 0;
+    long max_ops = 0;
+    char *buf = NULL;
+
+    buf = calloc(1, rdd_config.max_bs);
+    if (!buf) {
+        fprintf(stderr, "calloc failed (%s)\n", strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < rdd_config.iters; i++) {
+        pthread_mutex_lock(&rdd_config.lock);
+        {
+            int bytes = 0;
+            rand = random();
+
+            if (rdd_config.min_bs == rdd_config.max_bs) {
+                bs = rdd_config.max_bs;
+            } else {
+                bs = rdd_config.min_bs +
+                     (rand % (rdd_config.max_bs - rdd_config.min_bs));
+            }
+
+            offset = rand % rdd_config.in_file.st.st_size;
+            max_ops = rand % rdd_config.max_ops_per_seq;
+            if (!max_ops) {
+                max_ops++;
+            }
+
+            ret = lseek(rdd_config.in_file.fd, offset, SEEK_SET);
+            if (ret != offset) {
+                fprintf(stderr, "lseek failed (%s)\n", strerror(errno));
+                ret = -1;
+                goto unlock;
+            }
+
+            ret = lseek(rdd_config.out_file.fd, offset, SEEK_SET);
+            if (ret != offset) {
+                fprintf(stderr, "lseek failed (%s)\n", strerror(errno));
+                ret = -1;
+                goto unlock;
+            }
+
+            while (max_ops--) {
+                bytes = read(rdd_config.in_file.fd, buf, bs);
+                if (!bytes) {
+                    break;
+                }
+
+                if (bytes == -1) {
+                    fprintf(stderr, "read failed (%s)\n", strerror(errno));
+                    ret = -1;
+                    goto unlock;
+                }
+
+                if (write(rdd_config.out_file.fd, buf, bytes) != bytes) {
+                    fprintf(stderr, "write failed (%s)\n", strerror(errno));
+                    ret = -1;
+                    goto unlock;
+                }
+            }
+        }
+    unlock:
+        pthread_mutex_unlock(&rdd_config.lock);
+        if (ret == -1) {
+            goto out;
+        }
+        ret = 0;
+    }
 out:
-	free (buf);
-	pthread_barrier_wait (&rdd_config.barrier);
+    free(buf);
+    pthread_barrier_wait(&rdd_config.barrier);
 
-	return NULL;
+    return NULL;
 }
 
+static void
+cleanup(void)
+{
+    close(rdd_config.in_file.fd);
+    close(rdd_config.out_file.fd);
+    rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
+}
 
 static int
-rdd_spawn_threads (void)
+check_and_create(void)
 {
-	int i = 0, ret = -1, fd = -1;
-	char buf[4096]; 
-
-	fd = open (rdd_config.in_file.path, O_RDONLY);
-	if (fd < 0) {
-		fprintf (stderr, "cannot open %s (%s)\n", 
-                         rdd_config.in_file.path, strerror (errno));
-		ret = -1;
-		goto out;
-	}
-	ret = fstat (fd, &rdd_config.in_file.st);
-	if (ret != 0) {
-		close (fd);
-		fprintf (stderr, "cannot stat %s (%s)\n", 
-                         rdd_config.in_file.path, strerror (errno));
-		ret = -1;
-		goto out;
-	}
-	rdd_config.in_file.fd = fd;
-
-	fd = open (rdd_config.out_file.path, O_WRONLY | O_CREAT, 
-                   S_IRWXU | S_IROTH);
-	if (fd < 0) {
-		close (rdd_config.in_file.fd);
-		rdd_config.in_file.fd = -1;
-		fprintf (stderr, "cannot open %s (%s)\n", 
-                         rdd_config.out_file.path, strerror (errno));
-		ret = -1;
-		goto out;
-	}
-	rdd_config.out_file.fd = fd;
-
-	while ((ret = read (rdd_config.in_file.fd, buf, 4096)) > 0) {
-		if (write (rdd_config.out_file.fd, buf, ret) != ret) {
-			fprintf (stderr, "write failed (%s)\n", 
-                                 strerror (errno));
-			close (rdd_config.in_file.fd);
-			close (rdd_config.out_file.fd);
-			rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-			ret = -1;
-			goto out;
-		}
-	}
-
-	rdd_config.threads = calloc (rdd_config.thread_count, 
-                                     sizeof (pthread_t));
-	if (rdd_config.threads == NULL) {
-		fprintf (stderr, "calloc() failed (%s)\n", strerror (errno));
-
-		ret = -1;
-		close (rdd_config.in_file.fd);
-		close (rdd_config.out_file.fd);
-		rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-		goto out;
-	}
-
-	ret = pthread_barrier_init (&rdd_config.barrier, NULL, 
-                                    rdd_config.thread_count + 1);
-	if (ret != 0) {
-		fprintf (stderr, "pthread_barrier_init() failed (%s)\n", 
-                         strerror (ret));
-
-		free (rdd_config.threads);
-		close (rdd_config.in_file.fd);
-		close (rdd_config.out_file.fd);
-		rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-		ret = -1;
-		goto out;
-	}
-
-	ret = pthread_mutex_init (&rdd_config.lock, NULL);
-	if (ret != 0) {
-		fprintf (stderr, "pthread_mutex_init() failed (%s)\n", 
-                         strerror (ret));
-
-		free (rdd_config.threads);
-		pthread_barrier_destroy (&rdd_config.barrier);
-		close (rdd_config.in_file.fd);
-		close (rdd_config.out_file.fd);
-		rdd_config.in_file.fd = rdd_config.out_file.fd = -1;
-		ret = -1;
-		goto out;
-	}
-
-	for (i = 0; i < rdd_config.thread_count; i++)
-	{
-		ret = pthread_create (&rdd_config.threads[i], NULL, 
-                                      rdd_read_write, NULL);
-		if (ret != 0) {
-			fprintf (stderr, "pthread_create failed (%s)\n", 
-                                 strerror (errno));
-			exit (1);
-		}
-	}
+    int ret = -1;
+    char buf[4096] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int fd[2] = {
+        -1,
+    };
+    size_t total_size = -1;
+
+    total_size = rdd_config.file_size;
+
+    ret = stat(rdd_config.in_file.path, &stbuf);
+    if (ret == -1 && (errno != ENOENT))
+        goto out;
+
+    fd[1] = open(rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC);
+    if (fd[1] == -1)
+        goto out;
+
+    fd[0] = open("/dev/urandom", O_RDONLY);
+    if (fd[0] == -1)
+        goto out;
+
+    while (total_size > 0) {
+        if (total_size >= 4096) {
+            ret = read(fd[0], buf, 4096);
+            if (ret == -1)
+                goto out;
+            ret = write(fd[1], buf, 4096);
+            if (ret == -1)
+                goto out;
+            total_size = total_size - 4096;
+        } else {
+            ret = read(fd[0], buf, total_size);
+            if (ret == -1)
+                goto out;
+            ret = write(fd[1], buf, total_size);
+            if (ret == -1)
+                goto out;
+            total_size = total_size - total_size;
+        }
+    }
+
+    ret = 0;
 
 out:
-	return ret;
+    if (fd[0] > 0)
+        close(fd[0]);
+    if (fd[1] > 0)
+        close(fd[1]);
+    return ret;
 }
 
+static int
+rdd_spawn_threads(void)
+{
+    int i = 0, ret = -1, fd = -1;
+    char buf[4096];
+
+    ret = check_and_create();
+    if (ret == -1)
+        goto out;
+
+    fd = open(rdd_config.in_file.path, O_RDONLY);
+    if (fd < 0) {
+        fprintf(stderr, "cannot open %s (%s)\n", rdd_config.in_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    ret = fstat(fd, &rdd_config.in_file.st);
+    if (ret != 0) {
+        close(fd);
+        fprintf(stderr, "cannot stat %s (%s)\n", rdd_config.in_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    rdd_config.in_file.fd = fd;
+
+    fd = open(rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC,
+              S_IRWXU | S_IROTH);
+    if (fd < 0) {
+        close(rdd_config.in_file.fd);
+        rdd_config.in_file.fd = -1;
+        fprintf(stderr, "cannot open %s (%s)\n", rdd_config.out_file.path,
+                strerror(errno));
+        ret = -1;
+        goto out;
+    }
+    rdd_config.out_file.fd = fd;
+
+    while ((ret = read(rdd_config.in_file.fd, buf, 4096)) > 0) {
+        if (write(rdd_config.out_file.fd, buf, ret) != ret) {
+            fprintf(stderr, "write failed (%s)\n", strerror(errno));
+            cleanup();
+            ret = -1;
+            goto out;
+        }
+    }
+
+    rdd_config.threads = calloc(rdd_config.thread_count, sizeof(pthread_t));
+    if (rdd_config.threads == NULL) {
+        fprintf(stderr, "calloc() failed (%s)\n", strerror(errno));
+
+        ret = -1;
+        cleanup();
+        goto out;
+    }
+
+    ret = pthread_barrier_init(&rdd_config.barrier, NULL,
+                               rdd_config.thread_count + 1);
+    if (ret != 0) {
+        fprintf(stderr, "pthread_barrier_init() failed (%s)\n", strerror(ret));
+
+        free(rdd_config.threads);
+        cleanup();
+        ret = -1;
+        goto out;
+    }
+
+    ret = pthread_mutex_init(&rdd_config.lock, NULL);
+    if (ret != 0) {
+        fprintf(stderr, "pthread_mutex_init() failed (%s)\n", strerror(ret));
+
+        free(rdd_config.threads);
+        pthread_barrier_destroy(&rdd_config.barrier);
+        cleanup();
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < rdd_config.thread_count; i++) {
+        ret = pthread_create(&rdd_config.threads[i], NULL, rdd_read_write,
+                             NULL);
+        if (ret != 0) {
+            fprintf(stderr, "pthread_create failed (%s)\n", strerror(errno));
+            exit(1);
+        }
+    }
+
+out:
+    return ret;
+}
 
 static void
-rdd_wait_for_completion (void)
+rdd_wait_for_completion(void)
 {
-	pthread_barrier_wait (&rdd_config.barrier);
+    pthread_barrier_wait(&rdd_config.barrier);
 }
 
-
-int 
-main (int argc, char *argv[])
+int
+main(int argc, char *argv[])
 {
-	int ret = -1;
+    int ret = -1;
 
-	rdd_default_config ();
+    rdd_default_config();
 
-	ret = argp_parse (&argp, argc, argv, 0, 0, NULL);
-	if (ret != 0) {
-		ret = -1;
-		fprintf (stderr, "%s: argp_parse() failed\n", argv[0]);
-		goto err;
-	}
+    ret = argp_parse(&argp, argc, argv, 0, 0, NULL);
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "%s: argp_parse() failed\n", argv[0]);
+        goto err;
+    }
 
-	if (!rdd_valid_config ()) {
-		ret = -1;
-		fprintf (stderr, "%s: configuration validation failed\n", 
-                         argv[0]);
-		goto err;
-	}
+    if (!rdd_valid_config()) {
+        ret = -1;
+        fprintf(stderr, "%s: configuration validation failed\n", argv[0]);
+        goto err;
+    }
 
-	ret = rdd_spawn_threads ();
-	if (ret != 0) {
-		fprintf (stderr, "%s: spawning threads failed\n", argv[0]);
-		goto err;
-	}
+    ret = rdd_spawn_threads();
+    if (ret != 0) {
+        fprintf(stderr, "%s: spawning threads failed\n", argv[0]);
+        goto err;
+    }
 
-	rdd_wait_for_completion ();
+    rdd_wait_for_completion();
 
 err:
-	return ret;
-} 
+    return ret;
+}
diff --git a/extras/check_goto.pl b/extras/check_goto.pl
new file mode 100755
index 00000000000..fa71bfc6683
--- /dev/null
+++ b/extras/check_goto.pl
@@ -0,0 +1,45 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+my @ignore_labels = qw (TODO retry fetch_data again try_again sp_state_read_proghdr redo disabled_loop fd_alloc_try_again);
+my @ignore_files  = qw (y.tab.c lex.c);
+my @c_files;
+my $line;
+my @labels;
+my $in_comments;
+
+{
+        local $" = "|";
+        my $cmd = "find . -type f -name '*.c' | grep -vE '(@ignore_files)'";
+        @c_files = `$cmd`;
+}
+
+foreach my $file (@c_files) {
+        chomp ($file);
+        open FD, $file or die ("Failed to read file $file: $!");
+        @labels = ();
+        $in_comments = 0;
+        while ($line = <FD>) {
+                chomp ($line);
+
+                next if $line =~ /^\s*(#|\/\/)/;
+                $in_comments = 1 if ($line =~ /\/\*/);
+                $in_comments = 0 if ($line =~ /\*\//);
+
+                next if $in_comments;
+                if ($line =~ /^\s*(([a-zA-Z]|_)\w*)\s*:/) {
+                        push (@labels, $1) unless grep (/$1/, @ignore_labels);
+                }
+                @labels = () if $line =~ /^}/;
+
+                next unless @labels;
+                if ($line =~ /^\s*goto\s*(\w+)/) {
+                        print "$file:$.: $line\n" if grep /^$1$/, @labels;
+                }
+        }
+
+        close FD;
+}
+
diff --git a/extras/clang-checker.sh b/extras/clang-checker.sh
new file mode 100755
index 00000000000..4909d3adfcd
--- /dev/null
+++ b/extras/clang-checker.sh
@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+#*******************************************************************************
+#                                                                              *
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>                    *
+#  This file is part of GlusterFS.                                             *
+#                                                                              *
+#  This file is licensed to you under your choice of the GNU Lesser            *
+#  General Public License, version 3 or any later version (LGPLv3 or           *
+#  later), or the GNU General Public License, version 2 (GPLv2), in all        *
+#  cases as published by the Free Software Foundation.                         *
+#------------------------------------------------------------------------------*
+#                                                                              *
+# clang-checker.sh:  This script runs clang static analyzer using 'scan-build' *
+#                    a perl wrapper. After you commit your patch i.e. right    *
+#                    before executing rfc.sh in order to push the patch to     *
+#                    repository, it is recommended that you execute            *
+#                    clang-checker.sh to perform static analysis inorder to    *
+#                    check if there are any possible bugs in the code.         *
+#                                                                              *
+#                    This script performs the static analysis with and      *
+#                    without HEAD commit, it runs the analyzer only in the     *
+#                    directory where changes have been made and finally diff's *
+#                    the number of bugs using both cases (i.e. with and        *
+#                    without your commit) and gives a summary, which explain's *
+#                    about the eligibility of your patch.                      *
+#                                                                              *
+# Usage:             $ cd $PATH_TO_GLUSTERFS                                   *
+#                    $ extras/clang-checker.sh (or) $ make clang-check         *
+#                                                                              *
+# Author:            Prasanna Kumar Kalever <prasanna.kalever@redhat.com>      *
+#                                                                              *
+#*******************************************************************************
+
+REPORTS_DIR=$(pwd)
+BASELINE_DIR=${REPORTS_DIR}/baseline
+BRESULTS_DIR=${BASELINE_DIR}/results
+BBACKUP_DIR=${BASELINE_DIR}/backup
+TARGET_DIR=${REPORTS_DIR}/target
+TRESULTS_DIR=${TARGET_DIR}/results
+TBACKUP_DIR=${TARGET_DIR}/backup
+
+declare -A DICT_B
+declare -A DICT_T
+declare -A ARR
+declare -A FILES
+
+function identify_changes () {
+    MODIFIED_DATA=$(git show --name-status --oneline | tail -n +2)
+    FLAG=0
+    for i in ${MODIFIED_DATA}; do
+        if [ $FLAG -eq 1 ]; then
+            ARR+="$(dirname $i) ";
+            FLAG=0;
+        fi
+        if [ $i = 'M' ] || [ $i = 'A' ]; then
+            FLAG=1;
+        fi
+    done
+
+    MODIFIED_DIR=$(echo "${ARR[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
+    for i in $MODIFIED_DIR; do
+        # run only in directories which has Makefile
+        if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then
+            # skip 'doc' and '.'(top) directory
+            if [ "xx$i" != "xxdoc" ] && [ "xx$i" != "xx." ]; then
+                FILES+="$i "
+            fi
+        fi
+    done
+    if [ -z $FILES ]; then
+        echo "Probably no changes made to 'c' files"
+        exit;
+    fi
+}
+
+function check_prerequisites () {
+    if ! type "clang" 2> /dev/null; then
+        echo -e "\ntry after installing clang and scan-build..."
+        echo    "useful info at http://clang-analyzer.llvm.org/installation.html\n"
+        echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n"
+        exit 1;
+    elif ! type "scan-build" 2> /dev/null; then
+        echo -e "\ntry after installing scan-build..."
+        echo    "useful info at http://clang-analyzer.llvm.org/installation.html"
+        echo -e "hint: 'dnf -y install clang-analyzer.noarch'\n"
+        exit 1;
+    fi
+}
+
+function force_terminate () {
+    echo -e "\nreceived a signal to force terminate ..\n"
+    git am --abort 2> /dev/null
+    git am ${PATCH_NAME}
+    rm -f ${REPORTS_DIR}/${PATCH_NAME}
+    exit 1;
+}
+
+function run_scanbuild () {
+    local CLANG=$(which clang)
+    local SCAN_BUILD=$(which scan-build)
+    local ORIG_COMMIT=$(git rev-parse --verify HEAD^)
+    PATCH_NAME=$(git format-patch $ORIG_COMMIT)
+
+    echo -e "\n| Performing clang analysis on:" \
+            "$(git log --pretty=format:"%h - '%s' by %an" -1) ... |\n"
+    echo -e "Changes are identified in '${FILES[@]}' directorie[s]\n"
+
+    if [ -d "${BRESULTS_DIR}" ]; then
+        mkdir -p ${BBACKUP_DIR} ${TBACKUP_DIR}
+        mv ${BRESULTS_DIR} \
+           ${BBACKUP_DIR}/results_$(ls -l ${BBACKUP_DIR} | wc -l)
+        mv ${TRESULTS_DIR} \
+           ${TBACKUP_DIR}/results_$(ls -l ${TBACKUP_DIR} | wc -l)
+    fi
+    for DIR in ${FILES[@]}; do
+        mkdir -p ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g')
+        mkdir -p ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g')
+    done
+    # get nproc info
+    case $(uname -s) in
+    'Linux')
+        local NPROC=$(getconf _NPROCESSORS_ONLN)
+        ;;
+    'NetBSD')
+        local NPROC=$(getconf NPROCESSORS_ONLN)
+        ;;
+    esac
+
+    trap force_terminate INT TERM QUIT EXIT
+
+    git reset --hard HEAD^
+
+    # build complete source code for sake of dependencies
+    echo -e "\n# make -j${NPROC} ..."
+    make -j${NPROC} 1>/dev/null
+
+    for DIR in ${FILES[@]}; do
+        if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then
+            make clean -C ${DIR} 1>/dev/null
+            echo -e "\n| Analyzing ${DIR} without commit ... |\n"
+            # run only in directory where changes are made
+            ${SCAN_BUILD} -o ${BRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \
+                          --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR}
+        fi
+    done
+
+    echo -e "\n| Analyzing without commit complete ... |\n"
+
+    git am ${PATCH_NAME}
+    trap - INT TERM QUIT EXIT
+
+    # In case commit has changes to configure stuff ?
+    echo -e "\n# make clean ..."
+    make clean 1>/dev/null
+    echo -e "\n# ./autogen.sh && ./configure --with-previous-options ..."
+    ${REPORTS_DIR}/autogen.sh 2>/dev/null
+    ${REPORTS_DIR}/configure  --with-previous-options 1>/dev/null
+    echo -e "\n# make -j${NPROC} ..."
+    make -j${NPROC} 1>/dev/null
+
+    for DIR in ${FILES[@]}; do
+        if [ $(find ./$i -iname "makefile*" | wc -c) -gt 0 ]; then
+            make clean -C ${DIR} 1>/dev/null
+            echo -e "\n| Analyzing ${DIR} with commit ... |\n"
+            # run only in directory where changes are made
+            ${SCAN_BUILD} -o ${TRESULTS_DIR}/$(echo ${DIR} | sed 's/\//_/g') \
+                          --use-analyzer=${CLANG} make -j${NPROC} -C ${DIR}
+        fi
+    done
+
+    echo -e "\n| Analyzing with commit complete ... |\n"
+
+    rm -f ${REPORTS_DIR}/${PATCH_NAME}
+}
+
+function count_for_baseline () {
+    for DIR in ${FILES[@]}; do
+        HTMLS_DIR=${BRESULTS_DIR}/$(echo ${DIR} |
+                    sed 's/\//_/g')/$(ls ${BRESULTS_DIR}/$(echo ${DIR} |
+                    sed 's/\//_/g')/);
+
+        local NAMES_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html |
+                                cut -d"<" -f3 | cut -d">" -f2 |
+                                sed 's/[^a-zA-Z0]/_/g' | tr '\n' ' ')
+        local NO_OF_BUGS_B=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html |
+                             cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ')
+        local count_B=0;
+
+        read -a BUG_NAME_B <<<$NAMES_OF_BUGS_B
+        read -a BUG_COUNT_B <<<$NO_OF_BUGS_B
+        for i in ${BUG_NAME_B[@]};
+        do
+            if [ ! -z ${DICT_B[$i]} ]; then
+                DICT_B[$i]=$(expr ${BUG_COUNT_B[count_B]} + ${DICT_B[$i]});
+            else
+                DICT_B+=([$i]=${BUG_COUNT_B[count_B]});
+            fi
+            count_B=$(expr $count_B + 1)
+        done
+    done
+
+    echo -e "\nBASELINE BUGS LIST (before applying patch):"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    for key_B in ${!DICT_B[@]}; do
+        echo "${key_B} --> ${DICT_B[${key_B}]}" | sed 's/_/ /g' | tr -s ' '
+    done
+}
+
+function count_for_target () {
+    for DIR in ${FILES[@]}; do
+        HTMLS_DIR=${TRESULTS_DIR}/$(echo ${DIR} |
+                    sed 's/\//_/g')/$(ls ${TRESULTS_DIR}/$(echo ${DIR} |
+                    sed 's/\//_/g')/);
+
+        local NAME_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html |
+                              cut -d"<" -f3 | cut -d">" -f2 |
+                              sed 's/[^a-zA-Z0]/_/g'| tr '\n' ' ')
+        local NO_OF_BUGS_T=$(grep -n "SUMM_DESC" ${HTMLS_DIR}/index.html |
+                             cut -d"<" -f5 | cut -d">" -f2 | tr '\n' ' ')
+        local count_T=0;
+
+        read -a BUG_NAME_T <<<$NAME_OF_BUGS_T
+        read -a BUG_COUNT_T <<<$NO_OF_BUGS_T
+
+        for i in ${BUG_NAME_T[@]};
+        do
+            if [ ! -z ${DICT_T[$i]} ]; then
+                DICT_T[$i]=$(expr ${BUG_COUNT_T[count_T]} + ${DICT_T[$i]});
+            else
+                DICT_T+=([$i]=${BUG_COUNT_T[count_T]});
+            fi
+            count_T=$(expr $count_T + 1)
+        done
+    done
+
+    echo -e "\nTARGET BUGS LIST (after applying patch):"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    for key_T in ${!DICT_T[@]}; do
+        echo "${key_T} --> ${DICT_T[${key_T}]}" | sed 's/_/ /g' | tr -s ' '
+    done
+}
+
+function array_contains () {
+    local SEEKING=$1; shift
+    local IN=1
+    for ELEMENT; do
+        if [[ $ELEMENT == $SEEKING ]]; then
+            IN=0
+            break
+        fi
+    done
+    return $IN
+}
+
+function main () {
+    echo -e "\n================ Clang analyzer in progress ================\n"
+    check_prerequisites
+    identify_changes
+    run_scanbuild
+    clear
+    count_for_baseline
+    count_for_target
+    echo -e "\nSUMMARY OF CLANG-ANALYZER:"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+
+    FLAG=0
+    for BUG in ${!DICT_T[@]}; do
+        array_contains $BUG "${!DICT_B[@]}"
+        if [ $? -eq 1 ]; then
+            echo "New ${DICT_T[${BUG}]} Bug[s] introduced: $(echo $BUG |
+                                                            sed 's/_/ /g' |
+                                                            tr -s ' ')"
+            FLAG=1
+        else
+            if [ ${BUG} != "All_Bugs" ]; then
+                if [ ${DICT_B[${BUG}]} -lt            \
+                    ${DICT_T[${BUG}]} ]; then
+                    echo "Extra $(expr ${DICT_T[${BUG}]} - \
+                          ${DICT_B[${BUG}]}) Bug[s] Introduced in: $(echo $BUG |
+                          sed 's/_/ /g' | tr -s ' ')"
+                    FLAG=1
+                fi
+            fi
+        fi
+    done
+
+    echo
+    if [ $FLAG -eq 0 ]; then
+        echo -e "Patch Value given by Clang analyzer '+1'\n"
+    else
+        echo -e "Patch Value given by Clang analyzer '-1'\n"
+    fi
+    echo -e "\nExplore complete results at:"
+    find ${BRESULTS_DIR}/ -iname "index.html"
+    find ${TRESULTS_DIR}/ -iname "index.html"
+    echo -e "\n================= Done with Clang Analysis =================\n"
+
+    exit ${FLAG}
+}
+
+main
diff --git a/extras/clear_xattrs.sh b/extras/clear_xattrs.sh
new file mode 100755
index 00000000000..dd04731e8bd
--- /dev/null
+++ b/extras/clear_xattrs.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Clear the trusted.gfid xattr in the brick tree
+
+# This script must be run only on a stopped brick/volume
+# Stop the volume to make sure no rebalance/replace-brick
+# operations are on-going
+
+# Not much error checking
+remove_xattrs ()
+{
+    find "$1" -exec setfattr -h -x "trusted.gfid" '{}' \;  > /dev/null 2>&1;
+    find "$1" -exec setfattr -h -x "trusted.glusterfs.volume-id" '{}' \;  > /dev/null 2>&1;
+}
+
+main ()
+{
+    if [ -z "$1" ]; then
+        echo "Usage: $0 <brick_path(s)>";
+        exit 1;
+    fi
+
+    export PATH;
+    which getfattr > /dev/null 2>&1;
+    if [ $? -ne 0 ]; then
+        echo "attr package missing";
+        exit 2;
+    fi
+
+    which setfattr > /dev/null 2>&1;
+    if [ $? -ne 0 ]; then
+        echo "attr package missing";
+        exit 2;
+    fi
+
+    for brick in "$@";
+    do
+        stat "$brick" > /dev/null 2>&1;
+        if [ $? -ne 0 ]; then
+            echo "brick: $brick does not exist";
+            exit 3;
+        fi
+        if [ ! -d "$brick" ]; then
+            echo "$brick: not a directory";
+            exit 4;
+        fi
+        echo "xattr clean-up in progress: $brick";
+        remove_xattrs "$brick";
+        echo "$brick ready to be used as a glusterfs brick";
+    done;
+}
+
+main "$@";
+\ No newline at end of file
diff --git a/extras/cliutils/Makefile.am b/extras/cliutils/Makefile.am
new file mode 100644
index 00000000000..7039703e275
--- /dev/null
+++ b/extras/cliutils/Makefile.am
@@ -0,0 +1,4 @@
+EXTRA_DIST= cliutils.py __init__.py
+
+cliutilsdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster/cliutils
+cliutils_PYTHON = cliutils.py __init__.py
diff --git a/extras/cliutils/README.md b/extras/cliutils/README.md
new file mode 100644
index 00000000000..309beb1ca25
--- /dev/null
+++ b/extras/cliutils/README.md
@@ -0,0 +1,233 @@
+# CLI utility for creating Cluster aware CLI tools for Gluster
+cliutils is a Python library which provides wrapper around `gluster system::
+execute` command to extend the functionalities of Gluster.
+
+Example use cases:
+- Start a service in all peer nodes of Cluster
+- Collect the status of a service from all peer nodes
+- Collect the config values from each peer nodes and display latest
+  config based on version.
+- Copy a file present in GLUSTERD_WORKDIR from one peer node to all
+  other peer nodes.(Geo-replication create push-pem is using this to
+  distribute the SSH public keys from all master nodes to all slave
+  nodes)
+- Generate pem keys in all peer nodes and collect all the public keys
+  to one place(Geo-replication gsec_create is doing this)
+- Provide Config sync CLIs for new features like `gluster-eventsapi`,
+  `gluster-restapi`, `gluster-mountbroker` etc.
+
+## Introduction
+
+If a executable file present in `$GLUSTER_LIBEXEC` directory in all
+peer nodes(Filename startswith `peer_`) then it can be executed by
+running `gluster system:: execute` command from any one peer node.
+
+- This command will not copy any executables to peer nodes, Script
+  should exist in all peer nodes to use this infrastructure. Raises
+  error in case script not exists in any one of the peer node.
+- Filename should start with `peer_` and should exist in
+  `$GLUSTER_LIBEXEC` directory.
+- This command can not be called from outside the cluster.
+
+To understand the functionality, create a executable file `peer_hello`
+under $GLUSTER_LIBEXEC directory and copy to all peer nodes.
+
+    #!/usr/bin/env bash
+    echo "Hello from $(gluster system:: uuid get)"
+
+Now run the following command from any one gluster node,
+
+    gluster system:: execute hello
+
+**Note:** Gluster will not copy the executable script to all nodes,
+  copy `peer_hello` script to all peer nodes to use `gluster system::
+  execute` infrastructure.
+
+It will run `peer_hello` executable in all peer nodes and shows the
+output from each node(Below example shows output from my two nodes
+cluster)
+
+    Hello from UUID: e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84
+    Hello from UUID: c680fc0a-01f9-4c93-a062-df91cc02e40f
+
+## cliutils
+A Python wrapper around `gluster system:: execute` command is created
+to address the following issues
+
+- If a node is down in the cluster, `system:: execute` just skips it
+  and runs only in up nodes.
+- `system:: execute` commands are not user friendly
+- It captures only stdout, so handling errors is tricky.
+
+**Advantages of cliutils:**
+
+- Single executable file will act as node component as well as User CLI.
+- `execute_in_peers` utility function will merge the `gluster system::
+  execute` output with `gluster peer status` to identify offline nodes.
+- Easy CLI Arguments handling.
+- If node component returns non zero return value then, `gluster
+  system:: execute` will fail to aggregate the output from other
+  nodes. `node_output_ok` or `node_output_notok` utility functions
+  returns zero both in case of success or error, but returns json
+  with ok: true or ok:false respectively.
+- Easy to iterate on the node outputs.
+- Better error handling - Geo-rep CLIs `gluster system:: execute
+  mountbroker`, `gluster system:: execute gsec_create` and `gluster
+  system:: add_secret_pub` are suffering from error handling. These
+  tools are not notifying user if any failures during execute or if a node
+  is down during execute.
+
+### Hello World
+Create a file in `$LIBEXEC/glusterfs/peer_message.py` with following
+content.
+
+    #!/usr/bin/python3
+    from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok
+
+    class NodeHello(Cmd):
+        name = "node-hello"
+
+        def run(self, args):
+            node_output_ok("Hello")
+
+    class Hello(Cmd):
+        name = "hello"
+
+        def run(self, args):
+            out = execute_in_peers("node-hello")
+            for row in out:
+                print ("{0} from {1}".format(row.output, row.hostname))
+
+    if __name__ == "__main__":
+        runcli()
+
+When we run `python peer_message.py`, it will have two subcommands,
+"node-hello" and "hello". This file should be copied to
+`$LIBEXEC/glusterfs` directory in all peer nodes. User will call
+subcommand "hello" from any one peer node, which internally call
+`gluster system:: execute message.py node-hello`(This runs in all peer
+nodes and collect the outputs)
+
+For node component do not print the output directly, use
+`node_output_ok` or `node_output_notok` functions. `node_output_ok`
+additionally collects the node UUID and prints in JSON
+format. `execute_in_peers` function will collect this output and
+merges with `peers list` so that we don't miss the node information if
+that node is offline.
+
+If you observed already, function `args` is optional, if you don't
+have arguments then no need to create a function. When we run the
+file, we will have two subcommands. For example,
+
+    python peer_message.py hello
+    python peer_message.py node-hello
+
+First subcommand calls second subcommand in all peer nodes. Basically
+`execute_in_peers(NAME, ARGS)` will be converted into
+
+    CMD_NAME = FILENAME without "peers_"
+    gluster system:: execute <CMD_NAME> <SUBCOMMAND> <ARGS>
+
+In our example,
+
+    filename = "peer_message.py"
+    cmd_name = "message.py"
+    gluster system:: execute ${cmd_name} node-hello
+
+Now create symlink in `/usr/bin` or `/usr/sbin` directory depending on
+the usecase.(Optional step for usability)
+
+    ln -s /usr/libexec/glusterfs/peer_message.py /usr/bin/gluster-message
+
+Now users can use `gluster-message` instead of calling
+`/usr/libexec/glusterfs/peer_message.py`
+
+    gluster-message hello
+
+### Showing CLI output as Table
+
+Following example uses prettytable library, which can be installed
+using `pip install prettytable` or `dnf install python-prettytable`
+
+    #!/usr/bin/python3
+    from prettytable import PrettyTable
+    from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok
+
+    class NodeHello(Cmd):
+        name = "node-hello"
+
+        def run(self, args):
+            node_output_ok("Hello")
+
+    class Hello(Cmd):
+        name = "hello"
+
+        def run(self, args):
+            out = execute_in_peers("node-hello")
+            # Initialize the CLI table
+            table = PrettyTable(["ID", "NODE", "NODE STATUS", "MESSAGE"])
+            table.align["NODE STATUS"] = "r"
+            for row in out:
+                table.add_row([row.nodeid,
+                               row.hostname,
+                               "UP" if row.node_up else "DOWN",
+                               row.output if row.ok else row.error])
+
+            print table
+
+    if __name__ == "__main__":
+        runcli()
+
+
+Example output,
+
+    +--------------------------------------+-----------+-------------+---------+
+    |                  ID                  |    NODE   | NODE STATUS | MESSAGE |
+    +--------------------------------------+-----------+-------------+---------+
+    | e7a3c5c8-e7ad-47ad-aa9c-c13907c4da84 | localhost |          UP |  Hello  |
+    | bb57a4c4-86eb-4af5-865d-932148c2759b | vm2       |          UP |  Hello  |
+    | f69b918f-1ffa-4fe5-b554-ee10f051294e | vm3       |        DOWN |  N/A    |
+    +--------------------------------------+-----------+-------------+---------+
+
+## How to package in Gluster
+If the project is created in `$GLUSTER_SRC/tools/message`
+
+Add "message" to SUBDIRS list in `$GLUSTER_SRC/tools/Makefile.am`
+
+and then create a `Makefile.am` in `$GLUSTER_SRC/tools/message`
+directory with following content.
+
+    EXTRA_DIST = peer_message.py
+
+    peertoolsdir = $(libexecdir)/glusterfs/
+    peertools_SCRIPTS = peer_message.py
+
+    install-exec-hook:
+        $(mkdir_p) $(DESTDIR)$(bindir)
+        rm -f $(DESTDIR)$(bindir)/gluster-message
+        ln -s $(libexecdir)/glusterfs/peer_message.py \
+            $(DESTDIR)$(bindir)/gluster-message
+
+    uninstall-hook:
+        rm -f $(DESTDIR)$(bindir)/gluster-message
+
+Thats all. Add following files in `glusterfs.spec.in` if packaging is
+required.(Under `%files` section)
+
+    %{_libexecdir}/glusterfs/peer_message.py*
+    %{_bindir}/gluster-message
+
+## Who is using cliutils
+- gluster-mountbroker   http://review.gluster.org/14544
+- gluster-eventsapi     http://review.gluster.org/14248
+- gluster-georep-sshkey http://review.gluster.org/14732
+- gluster-restapi       https://github.com/gluster/restapi
+
+## Limitations/TODOs
+- Not yet possible to create CLI without any subcommand, For example
+  `gluster-message` without any arguments
+- Hiding node subcommands in `--help`(`gluster-message --help` will
+  show all subcommands including node subcommands)
+- Only positional arguments supported for node arguments, Optional
+  arguments can be used for other commands.
+- API documentation
diff --git a/extras/cliutils/__init__.py b/extras/cliutils/__init__.py
new file mode 100644
index 00000000000..8765cc85099
--- /dev/null
+++ b/extras/cliutils/__init__.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Reexporting the utility funcs and classes
+from .cliutils import (runcli,
+                       sync_file_to_peers,
+                       execute_in_peers,
+                       execute,
+                       node_output_ok,
+                       node_output_notok,
+                       output_error,
+                       oknotok,
+                       yesno,
+                       get_node_uuid,
+                       Cmd,
+                       GlusterCmdException,
+                       set_common_args_func)
+
+
+# This will be useful when `from cliutils import *`
+__all__ = ["runcli",
+           "sync_file_to_peers",
+           "execute_in_peers",
+           "execute",
+           "node_output_ok",
+           "node_output_notok",
+           "output_error",
+           "oknotok",
+           "yesno",
+           "get_node_uuid",
+           "Cmd",
+           "GlusterCmdException",
+           "set_common_args_func"]
diff --git a/extras/cliutils/cliutils.py b/extras/cliutils/cliutils.py
new file mode 100644
index 00000000000..55fbaf56704
--- /dev/null
+++ b/extras/cliutils/cliutils.py
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+import inspect
+import subprocess
+import os
+import xml.etree.cElementTree as etree
+import json
+import sys
+
+MY_UUID = None
+parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
+                        description=__doc__)
+subparsers = parser.add_subparsers(dest="mode")
+
+subcommands = {}
+cache_data = {}
+ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError
+_common_args_func = lambda p: True
+
+
+class GlusterCmdException(Exception):
+    def __init__(self, message):
+        self.message = message
+        try:
+            # Python 3
+            super().__init__(message)
+        except TypeError:
+            # Python 2
+            super(GlusterCmdException, self).__init__(message)
+
+
+def get_node_uuid():
+    # Caches the Node UUID in global variable,
+    # Executes gluster system:: uuid get command only if
+    # calling this function for first time
+    global MY_UUID
+    if MY_UUID is not None:
+        return MY_UUID
+
+    cmd = ["gluster", "system::", "uuid", "get", "--xml"]
+    rc, out, err = execute(cmd)
+
+    if rc != 0:
+        return None
+
+    tree = etree.fromstring(out)
+    uuid_el = tree.find("uuidGenerate/uuid")
+    MY_UUID = uuid_el.text
+    return MY_UUID
+
+
+def yesno(flag):
+    return "Yes" if flag else "No"
+
+
+def oknotok(flag):
+    return "OK" if flag else "NOT OK"
+
+
+def output_error(message, errcode=1):
+    print (message, file=sys.stderr)
+    sys.exit(errcode)
+
+
+def node_output_ok(message=""):
+    # Prints Success JSON output and exits with returncode zero
+    out = {"ok": True, "nodeid": get_node_uuid(), "output": message}
+    print (json.dumps(out))
+    sys.exit(0)
+
+
+def node_output_notok(message):
+    # Prints Error JSON output and exits with returncode zero
+    out = {"ok": False, "nodeid": get_node_uuid(), "error": message}
+    print (json.dumps(out))
+    sys.exit(0)
+
+
+def execute(cmd):
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         universal_newlines=True)
+    out, err = p.communicate()
+    return p.returncode, out, err
+
+
+def get_pool_list():
+    cmd = ["gluster", "--mode=script", "pool", "list", "--xml"]
+    rc, out, err = execute(cmd)
+    if rc != 0:
+        output_error("Failed to get Pool Info: {0}".format(err))
+
+    tree = etree.fromstring(out)
+
+    pool = []
+    try:
+        for p in tree.findall('peerStatus/peer'):
+            pool.append({"nodeid": p.find("uuid").text,
+                         "hostname": p.find("hostname").text,
+                         "connected": (True if p.find("connected").text == "1"
+                                       else False)})
+    except (ParseError, AttributeError, ValueError) as e:
+        output_error("Failed to parse Pool Info: {0}".format(e))
+
+    return pool
+
+
+class NodeOutput(object):
+    def __init__(self, **kwargs):
+        self.nodeid = kwargs.get("nodeid", "")
+        self.hostname = kwargs.get("hostname", "")
+        self.node_up = kwargs.get("node_up", False)
+        self.ok = kwargs.get("ok", False)
+        self.output = kwargs.get("output", "N/A")
+        self.error = kwargs.get("error", "N/A")
+
+
+def execute_in_peers(name, args=[]):
+    # Get the file name of Caller function, If the file name is peer_example.py
+    # then Gluster peer command will be gluster system:: execute example.py
+    # Command name is without peer_
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    actual_file = module.__file__
+    # If file is symlink then find actual file
+    if os.path.islink(actual_file):
+        actual_file = os.readlink(actual_file)
+
+    # Get the name of file without peer_
+    cmd_name = os.path.basename(actual_file).replace("peer_", "")
+    cmd = ["gluster", "system::", "execute", cmd_name, name] + args
+    rc, out, err = execute(cmd)
+    if rc != 0:
+        raise GlusterCmdException((rc, out, err, " ".join(cmd)))
+
+    out = out.strip().splitlines()
+
+    # JSON decode each line and construct one object with node id as key
+    all_nodes_data = {}
+    for node_data in out:
+        data = json.loads(node_data)
+        all_nodes_data[data["nodeid"]] = {
+            "nodeid": data.get("nodeid"),
+            "ok": data.get("ok"),
+            "output": data.get("output", ""),
+            "error": data.get("error", "")}
+
+    # gluster pool list
+    pool_list = get_pool_list()
+
+    data_out = []
+    # Iterate pool_list and merge all_nodes_data collected above
+    # If a peer node is down then set node_up = False
+    for p in pool_list:
+        p_data = all_nodes_data.get(p.get("nodeid"), None)
+        row_data = NodeOutput(node_up=False,
+                              hostname=p.get("hostname"),
+                              nodeid=p.get("nodeid"),
+                              ok=False)
+
+        if p_data is not None:
+            # Node is UP
+            row_data.node_up = True
+            row_data.ok = p_data.get("ok")
+            row_data.output = p_data.get("output")
+            row_data.error = p_data.get("error")
+
+        data_out.append(row_data)
+
+    return data_out
+
+
+def sync_file_to_peers(fname):
+    # Copy file from current node to all peer nodes, fname
+    # is path after GLUSTERD_WORKDIR
+    cmd = ["gluster", "system::", "copy", "file", fname]
+    rc, out, err = execute(cmd)
+    if rc != 0:
+        raise GlusterCmdException((rc, out, err))
+
+
+class Cmd(object):
+    name = ""
+
+    def run(self, args):
+        # Must required method. Raise NotImplementedError if derived class
+        # not implemented this method
+        raise NotImplementedError("\"run(self, args)\" method is "
+                                  "not implemented by \"{0}\"".format(
+                                      self.__class__.__name__))
+
+
+def runcli():
+    # Get list of Classes derived from class "Cmd" and create
+    # a subcommand as specified in the Class name. Call the args
+    # method by passing subcommand parser, Derived class can add
+    # arguments to the subcommand parser.
+    metavar_data = []
+    for c in Cmd.__subclasses__():
+        cls = c()
+        if getattr(cls, "name", "") == "":
+            raise NotImplementedError("\"name\" is not added "
+                                      "to \"{0}\"".format(
+                                          cls.__class__.__name__))
+
+        # Do not show in help message if subcommand starts with node-
+        if not cls.name.startswith("node-"):
+            metavar_data.append(cls.name)
+
+        p = subparsers.add_parser(cls.name)
+        args_func = getattr(cls, "args", None)
+        if args_func is not None:
+            args_func(p)
+
+        # Apply common args if any
+        _common_args_func(p)
+
+        # A dict to save subcommands, key is name of the subcommand
+        subcommands[cls.name] = cls
+
+    # Hide node commands in Help message
+    subparsers.metavar = "{" + ",".join(metavar_data) + "}"
+
+    # Get all parsed arguments
+    args = parser.parse_args()
+
+    # Get the subcommand to execute
+    cls = subcommands.get(args.mode, None)
+
+    # Run
+    if cls is not None:
+        cls.run(args)
+
+
+def set_common_args_func(func):
+    global _common_args_func
+    _common_args_func = func
diff --git a/extras/collect-system-stats.sh b/extras/collect-system-stats.sh
new file mode 100755
index 00000000000..865e70bbc11
--- /dev/null
+++ b/extras/collect-system-stats.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+################################################################################
+# Usage: collect-system-stats.sh <delay-in-seconds>
+# This script starts sar/top/iostat/vmstat processes which collect system stats
+# with the interval <delay-in-seconds> given as argument to the script. When
+# the script is stopped either by entering any input or Ctrl+C the list of
+# files where output is captured will be printed on the screen which can be
+# observed to find any problems/bottlenecks.
+###############################################################################
+
+function stop_processes {
+        echo "Stopping the monitoring processes"
+        echo "sar pid:$sar_pid", "top pid: $top_pid", "iostat pid: $iostat_pid", "vmstat pid: $vmstat_pid"
+        kill "$sar_pid" "$top_pid" "$iostat_pid" "$vmstat_pid"
+        echo "Files created: ${timestamp}-network.out, ${timestamp}-top.out, ${timestamp}-iostat.out, ${timestamp}-vmstat.out"
+}
+
+function check_dependent_commands_exist()
+{
+        declare -a arr=("sar" "top" "iostat" "vmstat")
+        for i in "${arr[@]}"
+        do
+                if ! command -v "$i" > /dev/null 2>&1
+                then
+                        echo "ERROR: '$i' command is not found"
+                        exit 1
+                fi
+        done
+
+}
+
+case "$1" in
+    ''|*[!0-9]*) echo "Usage: $0 <delay-between-successive-metrics-collection-in-seconds>"; exit 1 ;;
+    *) interval="$1" ;;
+esac
+
+timestamp=$(date +"%s")
+
+check_dependent_commands_exist
+sar -n DEV "$interval" > "${timestamp}"-network.out &
+sar_pid="$!"
+top -bHd "$interval" > "${timestamp}"-top.out &
+top_pid="$!"
+iostat -Ntkdx "$interval" > "${timestamp}"-iostat.out &
+iostat_pid="$!"
+vmstat -t "$interval" > "${timestamp}"-vmstat.out &
+vmstat_pid="$!"
+echo "Started sar, vmstat, iostat, top for collecting stats"
+
+
+trap stop_processes EXIT
+read -r -p "Press anything and ENTER to exit";
diff --git a/extras/command-completion/Makefile b/extras/command-completion/Makefile
new file mode 100644
index 00000000000..06cbfe0672f
--- /dev/null
+++ b/extras/command-completion/Makefile
@@ -0,0 +1,6 @@
+install:
+	mkdir -p /etc/bash_completion.d
+	cp gluster.bash /etc/bash_completion.d/gluster
+
+uninstall:
+	rm -f /etc/bash_completion.d/gluster
diff --git a/extras/command-completion/README b/extras/command-completion/README
new file mode 100644
index 00000000000..0acba38168f
--- /dev/null
+++ b/extras/command-completion/README
@@ -0,0 +1,5 @@
+This file is not moved to /etc/bash_completion.d/ with the source installation.
+Please execute make install explicity from here to move this file to
+/etc/bash_completion.d
+
+Similarly, use make uninstall to remove it from /etc/bash_completion.d
diff --git a/extras/command-completion/gluster.bash b/extras/command-completion/gluster.bash
new file mode 100644
index 00000000000..73d16098875
--- /dev/null
+++ b/extras/command-completion/gluster.bash
@@ -0,0 +1,492 @@
+#!/bin/bash
+
+if pidof glusterd > /dev/null 2>&1; then
+        GLUSTER_SET_OPTIONS="
+        $(for token in `gluster volume set help 2>/dev/null | grep "^Option:" | cut -d ' ' -f 2`
+        do
+                echo "{$token},"
+        done)
+        "
+        GLUSTER_RESET_OPTIONS="$GLUSTER_SET_OPTIONS"
+fi
+
+GLUSTER_TOP_SUBOPTIONS1="
+        {nfs},
+        {brick},
+        {list-cnt}
+"
+GLUSTER_TOP_SUBOPTIONS2="
+        {bs
+                {__SIZE
+                        {count}
+                }
+        },
+        {brick},
+        {list-cnt}
+"
+GLUSTER_TOP_OPTIONS="
+        {open
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {read
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {write
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {opendir
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {readdir
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {clear
+                [ $GLUSTER_TOP_SUBOPTIONS1 ]
+        },
+        {read-perf
+                [ $GLUSTER_TOP_SUBOPTIONS2 ]
+        },
+        {write-perf
+                [ $GLUSTER_TOP_SUBOPTIONS2 ]
+        }
+"
+
+GLUSTER_QUOTA_OPTIONS="
+        {enable},
+        {disable},
+        {list},
+        {remove},
+        {default-soft-limit},
+        {limit-usage},
+        {alert-time},
+        {soft-timeout},
+        {hard-timeout}
+"
+
+GLUSTER_PROFILE_OPTIONS="
+        {start},
+        {info [
+                {peek},
+                {incremental
+                        {peek}
+                },
+                {cumulative},
+                {clear},
+              ]
+        },
+        {stop}
+"
+
+GLUSTER_BARRIER_OPTIONS="
+        {enable},
+        {disable}
+"
+
+GLUSTER_GEO_REPLICATION_SUBOPTIONS="
+"
+GLUSTER_GEO_REPLICATION_OPTIONS="
+        {__VOLNAME [
+                {__SLAVEURL [
+                        {create [
+                                {push-pem
+                                        {force}
+                                },
+                                {force}
+                                ]
+                        },
+                        {start {force} },
+                        {status {detail} },
+                        {config},
+                        {pause {force} },
+                        {resume {force} },
+                        {stop {force} },
+                        {delete {force} }
+                            ]
+                },
+                {status}
+                   ]
+        },
+        {status}
+"
+
+GLUSTER_VOLUME_OPTIONS="
+        {volume [
+                {add-brick
+                        {__VOLNAME}
+                },
+                {barrier
+                        {__VOLNAME
+                                [ $GLUSTER_BARRIER_OPTIONS ]
+                        }
+                },
+                {clear-locks
+                        {__VOLNAME}
+                },
+                {create},
+                {delete
+                        {__VOLNAME}
+                },
+                {geo-replication
+                        [ $GLUSTER_GEO_REPLICATION_OPTIONS ]
+                },
+                {heal
+                        {__VOLNAME}
+                },
+                {help},
+                {info
+                        {__VOLNAME}
+                },
+                {list},
+                {log
+                        {__VOLNAME}
+                },
+                {profile
+                        {__VOLNAME
+                                [ $GLUSTER_PROFILE_OPTIONS ]
+                        }
+                },
+                {quota
+                        {__VOLNAME
+                                [ $GLUSTER_QUOTA_OPTIONS ]
+                        }
+                },
+                {rebalance
+                        {__VOLNAME}
+                },
+                {remove-brick
+                        {__VOLNAME}
+                },
+                {replace-brick
+                        {__VOLNAME}
+                },
+                {reset
+                        {__VOLNAME
+                                [ $GLUSTER_RESET_OPTIONS ]
+                        }
+                },
+                {set
+                        {__VOLNAME
+                                [ $GLUSTER_SET_OPTIONS ]
+                        }
+                },
+                {start
+                        {__VOLNAME
+                                {force}
+                        }
+                },
+                {statedump
+                        {__VOLNAME}
+                },
+                {status
+                        {__VOLNAME}
+                },
+                {stop
+                        {__VOLNAME
+                                {force}
+                        }
+                },
+                {sync
+                        {__HOSTNAME}
+                },
+                {top
+                        {__VOLNAME
+                                [ $GLUSTER_TOP_OPTIONS ]
+                        }
+                }
+                ]
+        }
+"
+
+GLUSTER_COMMAND_TREE="
+{gluster [
+        $GLUSTER_VOLUME_OPTIONS ,
+        {peer [
+              {probe
+                      {__HOSTNAME}
+              },
+              {detach
+                      {__HOSTNAME
+                                {force}
+                      }
+              },
+              {status}
+              ]
+        },
+        {pool
+                {list}
+        },
+        {help}
+        ]
+}"
+
+__SIZE ()
+{
+        return 0
+}
+
+__SLAVEURL ()
+{
+        return 0
+}
+
+__HOSTNAME ()
+{
+        local zero=0
+        local ret=0
+        local cur_word="$2"
+
+        if [ "$1" == "X" ]; then
+                return
+
+        elif [ "$1" == "match" ]; then
+                return 0
+
+        elif [ "$1" == "complete" ]; then
+                COMPREPLY=($(compgen -A hostname -- $cur_word))
+        fi
+        return 0
+}
+
+__VOLNAME ()
+{
+        local zero=0
+        local ret=0
+        local cur_word="$2"
+        local list=""
+
+        if [ "X$1" == "X" ]; then
+                return
+
+        elif [ "$1" == "match" ]; then
+                return 0
+
+        elif [ "$1" == "complete" ]; then
+                if ! pidof glusterd > /dev/null 2>&1; then
+                        list='';
+
+                else
+                        list=`gluster volume list 2> /dev/null`
+                fi
+
+        else
+                return 0
+        fi
+
+        COMPREPLY=($(compgen -W "$list" -- $cur_word))
+        return 0
+}
+
+_gluster_throw () {
+#echo $1 >&2
+        COMPREPLY=''
+        exit
+}
+
+declare GLUSTER_FINAL_LIST=''
+declare GLUSTER_LIST=''
+declare -i GLUSTER_TOP=0
+_gluster_push () {
+        GLUSTER_TOP=$((GLUSTER_TOP + 1))
+        return $GLUSTER_TOP
+}
+_gluster_pop () {
+        GLUSTER_TOP=$((GLUSTER_TOP - 1))
+        return $GLUSTER_TOP
+}
+
+_gluster_goto_end ()
+{
+        local prev_top=$1
+        local top=$1
+        local token=''
+
+        while [ $top -ge $prev_top ]; do
+                read -r token
+                case $token in
+                '{' | '[')
+                        _gluster_push
+                        top=$?
+                        ;;
+                '}' | ']')
+                        _gluster_pop
+                        top=$?
+                        ;;
+                esac
+        done
+
+        return
+}
+
+_gluster_form_list ()
+{
+        local token=''
+        local top=0
+        local comma=''
+        local cur_word="$1"
+
+        read -r token
+        case $token in
+        ']')
+                ;;
+        '{')
+                _gluster_push
+                top=$?
+                read -r key
+                if [ "X$cur_word" == "X" -o "${cur_word:0:1}" == "${key:0:1}" -o "${key:0:1}" == "_" ]; then
+                        GLUSTER_LIST="$GLUSTER_LIST $key"
+                fi
+
+                _gluster_goto_end $top
+                read -r comma
+                if [ "$comma" == "," ]; then
+                        _gluster_form_list $cur_word
+                fi
+                ;;
+        *)
+                _gluster_throw "Expected '{' but received $token"
+                ;;
+        esac
+
+        return
+}
+
+_gluster_goto_child ()
+{
+        local match_string="$1"
+        local token=''
+        local top=0
+        local comma=''
+
+        read -r token
+        case $token in
+        '{')
+                _gluster_push
+                top=$?
+                ;;
+        *)
+                _gluster_throw "Expected '{' but received $token"
+                ;;
+        esac
+
+        read -r token
+        case `echo $token` in
+        '[' | ']' | '{' | '}')
+                _gluster_throw "Expected string but received $token"
+                ;;
+        _*)
+                $token "match" $match_string
+                ret=$?
+                if [ $ret -eq 0 ]; then
+                        return
+                else
+                        _gluster_goto_end $top
+
+                        read -r comma
+                        if [ "$comma" == "," ]; then
+                                _gluster_goto_child $match_string
+                        fi
+                fi
+                ;;
+
+        "$match_string")
+                return
+                ;;
+        *)
+                _gluster_goto_end $top
+
+                read -r comma
+                if [ "$comma" == "," ]; then
+                        _gluster_goto_child $match_string
+                fi
+                ;;
+        esac
+
+        return
+}
+
+_gluster_does_match ()
+{
+        local token="$1"
+        local key="$2"
+
+        if [ "${token:0:1}" == "_" ]; then
+                $token $2
+                return $?
+        fi
+
+        [ "$token" == "$key" ] && return 0
+
+        return 1
+}
+
+_gluster_parse ()
+{
+        local i=0
+        local token=''
+        local tmp_token=''
+        local word=''
+
+        while [ $i -lt $COMP_CWORD ]; do
+                read -r token
+                case $token in
+                '[')
+                        _gluster_push
+                        _gluster_goto_child ${COMP_WORDS[$i]}
+                        ;;
+                '{')
+                        _gluster_push
+                        read -r tmp_token
+                        _gluster_does_match $tmp_token ${COMP_WORDS[$i]}
+                        if [ $? -ne 0 ]; then
+                                _gluster_throw "No match"
+                        fi
+                        ;;
+                esac
+                i=$((i+1))
+        done
+
+        read -r token
+        if [ "$token" == '[' ]; then
+                _gluster_push
+                _gluster_form_list ${COMP_WORDS[COMP_CWORD]}
+
+        elif [ "$token" == '{' ]; then
+                read -r tmp_token
+                GLUSTER_LIST="$tmp_token"
+        fi
+
+        echo $GLUSTER_LIST
+}
+
+_gluster_handle_list ()
+{
+        local list="${!1}"
+        local cur_word=$2
+        local count=0
+        local i=0
+
+        for i in `echo $list`; do
+                count=$((count + 1))
+        done
+
+        if [ $count -eq 1 ] && [ "${i:0:1}" == "_" ]; then
+                $i "complete" $cur_word
+        else
+                COMPREPLY=($(compgen -W "$list" -- $cur_word))
+        fi
+        return
+}
+
+_gluster_completion ()
+{
+        GLUSTER_FINAL_LIST=`echo $GLUSTER_COMMAND_TREE |                      \
+                egrep -ao --color=never "([A-Za-z0-9_.-]+)|[[:space:]]+|." |  \
+                        egrep -v --color=never "^[[:space:]]*$" |             \
+                                _gluster_parse`
+
+        ARG="GLUSTER_FINAL_LIST"
+        _gluster_handle_list $ARG ${COMP_WORDS[COMP_CWORD]}
+        return
+}
+
+complete -F _gluster_completion gluster
diff --git a/extras/control-cpu-load.sh b/extras/control-cpu-load.sh
new file mode 100755
index 00000000000..52dcf62fd9f
--- /dev/null
+++ b/extras/control-cpu-load.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+USAGE="This script provides a utility to control CPU utilization for any
+gluster daemon.In this, we use cgroup framework to configure CPU quota
+for a process(like selfheal daemon). Before running this script, make
+sure that daemon is running.Every time daemon restarts, it is required
+to rerun this command to set CPU quota on new daemon process id.
+User can enter any value between 10 to 100 for CPU quota.
+Recommended value of quota period is 25. 25 means, kernel will allocate
+25 ms period to this group of tasks in every 100 ms period. This 25ms
+could be considered as the maximum percentage of CPU quota daemon can take.
+This value will be reflected on CPU usage of "top" command.If provided pid
+is the only process and no other process is in competition to get CPU, more
+ than 25% could be allocated to daemon to speed up the process."
+
+if [  $# -ge 1 ]; then
+  case $1 in
+    -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g'
+               exit 0;
+               ;;
+  *) echo "Please Provide correct input for script."
+     echo "For help correct options are -h or --help."
+     exit 1;
+               ;;
+  esac
+fi
+
+DIR_EXIST=0
+LOC="/sys/fs/cgroup/cpu,cpuacct/system.slice/glusterd.service"
+echo "Enter gluster daemon pid for which you want to control CPU."
+read daemon_pid
+
+if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then
+  CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid})
+  if [ -z "${CHECK_PID}" ]; then
+    echo "No daemon is running or pid ${daemon_pid} does not match."
+    echo "with running gluster processes."
+    exit 1
+  fi
+else
+  echo "Entered daemon_pid is not numeric so Rerun the script."
+  exit 1
+fi
+
+
+if [ -f ${LOC}/tasks ];then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    echo "pid ${daemon_pid} is attached with glusterd.service cgroup."
+  fi
+fi
+
+cgroup_name=cgroup_gluster_${daemon_pid}
+if [ -f ${LOC}/${cgroup_name}/tasks ]; then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    val=`cat ${LOC}/${cgroup_name}/cpu.cfs_quota_us`
+    qval=$((val / 1000))
+    echo "pid ${daemon_pid} is already attached ${cgroup_name} with quota value ${qval}."
+    echo "Press n if you don't want to reassign ${daemon_pid} with new quota value."
+    DIR_EXIST=1
+  else
+    echo "pid ${daemon_pid} is not attached with ${cgroup_name}."
+  fi
+fi
+
+read -p "If you want to continue the script to attach ${daemon_pid} with new ${cgroup_name} cgroup Press (y/n)?" choice
+case "$choice" in
+  y|Y ) echo "yes";;
+  n|N ) echo "no";exit;;
+  * ) echo "invalid";exit;;
+esac
+
+systemctl set-property glusterd.service CPUShares=1024
+
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service."
+  mkdir -p ${LOC}/${cgroup_name}
+  if [ ! -f ${LOC}/${cgroup_name}/tasks ];then
+    echo "Not able to create ${cgroup_name} directory so exit."
+    exit 1
+  fi
+fi
+
+echo "Enter quota value in range [10,100]:  "
+
+read quota_value
+if expr ${quota_value} + 0 > /dev/null 2>&1 ;then
+  if [ ${quota_value} -lt 10 ] || [ ${quota_value} -gt 100 ]; then
+    echo "Entered quota value is not correct,it should be in the range ."
+    echo "10-100. Ideal value is 25."
+    echo "Rerun the sript with correct value."
+    exit 1
+  else
+    echo "Entered quota value is $quota_value"
+  fi
+else
+  echo "Entered quota value is not numeric so Rerun the script."
+  exit 1
+fi
+
+quota_value=$((quota_value * 1000))
+echo "Setting $quota_value to cpu.cfs_quota_us for gluster_cgroup."
+echo ${quota_value} > ${LOC}/${cgroup_name}/cpu.cfs_quota_us
+
+if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then
+  for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`;
+    do
+      echo ${thid} > ${LOC}/${cgroup_name}/tasks ;
+    done
+  if cat /proc/${daemon_pid}/cgroup | grep -w ${cgroup_name} > /dev/null; then
+    echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}."
+  else
+    echo "Tasks are not attached successfully."
+  fi
+fi
diff --git a/extras/control-mem.sh b/extras/control-mem.sh
new file mode 100755
index 00000000000..91b36f8107a
--- /dev/null
+++ b/extras/control-mem.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+USAGE="This commands provides a utility to control MEMORY utilization for any
+gluster daemon.In this, we use cgroup framework to configure MEMORY limit for
+a process. Before running this script, make sure that daemon is running.Every
+time daemon restarts, it is required to rerun this command to set memory limit
+(in bytes) on new daemon process id.User can enter any value between 100
+(in Mega bytes) to 8000000000000 for Memory limit in Mega bytes.
+Memory limit value is depends on how much maximum memory user wants to restrict
+for specific daemon process.If a process will try to consume memore more than
+configured value then cgroup will hang/sleep this task and to resume the task
+rerun the script with new increase memory limit value ."
+
+if [  $# -ge 1 ]; then
+  case $1 in
+    -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g'
+               exit 0;
+               ;;
+    *) echo "Please Provide correct input for script."
+       echo "For help correct options are -h of --help."
+       exit 1;
+               ;;
+  esac
+fi
+
+DIR_EXIST=0
+LOC="/sys/fs/cgroup/memory/system.slice/glusterd.service"
+echo "Enter Any gluster daemon pid for that you want to control MEMORY."
+read daemon_pid
+
+if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then
+  CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid})
+  if [ -z "${CHECK_PID}" ]; then
+    echo "No daemon is running or pid ${daemon_pid} does not match."
+    echo "with running gluster processes."
+    exit 1
+  fi
+else
+  echo "Entered daemon_pid is not numeric so Rerun the script."
+  exit 1
+fi
+
+
+if [ -f ${LOC}/tasks ]; then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks)
+  if [ ${CHECK_CGROUP} ] ;then
+    echo "pid ${daemon_pid} is attached with default glusterd.service cgroup."
+  fi
+fi
+
+cgroup_name=cgroup_gluster_${daemon_pid}
+if [ -f ${LOC}/${cgroup_name}/tasks ];then
+  CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks)
+  if [ ${CHECK_CGROUP} ]; then
+    val=`cat ${LOC}/${cgroup_name}/memory.limit_in_bytes`
+    mval=$((val / 1024 / 1024))
+    echo "pid ${daemon_pid} is already attached ${cgroup_name} with mem value ${mval}."
+    echo "Press n if you don't want to reassign ${daemon_pid} with new mem value."
+    DIR_EXIST=1
+  else
+    echo "pid ${daemon_pid} is not attached with ${cgroup_name}."
+  fi
+fi
+
+read -p "If you want to continue the script to attach daeomon with new cgroup. Press (y/n)?" choice
+case "$choice" in
+  y|Y ) echo "yes";;
+  n|N ) echo "no";exit;;
+  * ) echo "invalid";exit;;
+esac
+
+systemctl set-property glusterd.service CPUShares=1024
+
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service."
+  mkdir -p ${LOC}/${cgroup_name}
+  if [ ! -f ${LOC}/${cgroup_name}/tasks ];then
+    echo "Not able to create ${LOC}/${cgroup_name} directory so exit."
+    exit 1
+  fi
+fi
+
+echo "Enter Memory value in Mega bytes [100,8000000000000]:  "
+
+read mem_value
+if expr ${mem_value} + 0 > /dev/null 2>&1 ;then
+  if [ ${mem_value} -lt 100 ] || [ ${mem_value} -gt 8000000000000 ]; then
+    echo "Entered memory value is not correct,it should be in the range ."
+    echo "100-8000000000000, Rerun the script with correct value ."
+    exit 1
+  else
+    echo "Entered memory limit value is ${mem_value}."
+  fi
+else
+  echo "Entered memory value is not numeric so Rerun the script."
+  exit 1
+fi
+
+mem_value=$(($mem_value * 1024 * 1024))
+if [ ${DIR_EXIST} -eq 0 ];then
+  echo "Setting ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}."
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes
+  #Set memory value to memory.memsw.limit_in_bytes
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes
+  # disable oom_control so that kernel will not send kill signal to the
+  # task once limit has reached
+  echo 1 > ${LOC}/${cgroup_name}/memory.oom_control
+else
+  #Increase mem_value to memory.memsw.limit_in_bytes
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes
+  echo "Increase ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}."
+  echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes
+  # disable oom_control so that kernel will not send kill signal to the
+  # task once limit has reached
+  echo 1 > ${LOC}/${cgroup_name}/memory.oom_control
+fi
+
+if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then
+  for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`;
+    do
+      echo ${thid} > ${LOC}/${cgroup_name}/tasks ;
+    done
+  if cat /proc/${daemon_pid}/cgroup | grep -iw ${cgroup_name} > /dev/null; then
+    echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}."
+  else
+    echo "Tasks are not attached successfully."
+  fi
+fi
diff --git a/extras/create_new_xlator/README.md b/extras/create_new_xlator/README.md
new file mode 100644
index 00000000000..fdc1ba00812
--- /dev/null
+++ b/extras/create_new_xlator/README.md
@@ -0,0 +1,24 @@
+####This document explains how to create a template for your new xlator.
+
+`$ python ./generate_xlator.py <XLATOR_DIRECTORY> <XLATOR_NAME> <FOP_PREFIX>`
+        * XLATOR_DIRECTORY: Directory path where the new xlator folder will reside
+        * XLATOR_NAME: Name of the xlator you wish to create
+        * FOP_PREFIX: This is the fop prefix that you wish to prefix every fop definition in your xlator, fop prefix is generally different than xlator name, if the xlator name is too long.
+
+Eg: `python ./generate_xlator.py /home/u1/glusterfs/xlators/features compression cmpr`
+This command will create the following files with some initial contents like copyright, fops definition etc.
+Note that there shouldn't be a "/" specified at the end of the <XLATOR_DIRECTORY>
+        `* /home/u1/glusterfs/xlators/features/compression/Makefile.am
+        * /home/u1/glusterfs/xlators/features/compression/src/Makefile.am
+        * /home/u1/glusterfs/xlators/features/compression/src/compression.c
+        * /home/u1/glusterfs/xlators/features/compression/src/compression.h
+        * /home/u1/glusterfs/xlators/features/compression/src/compression-mem-types.h
+        * /home/u1/glusterfs/xlators/features/compression/src/compression-messages.h`
+
+By default all the fops and functions are generated, if you wish to not implement certain fops and functions, comment those lines (by adding '#' at the start of the line) in libglusterfs/src/generate_xlator.py
+
+Few other manual steps required to get the new xlator completely functional:
+* Change configure.ac
+* Change `<XLATOR_DIRECTORY>/Makefile.am` to include the new xlator directory.
+  Eg:  `/home/u1/glusterfs/xlators/features/Makefile.am`
+* Change vol file or glusterd volgen to include the new xlator in volfile
diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py
new file mode 100755
index 00000000000..983868c04db
--- /dev/null
+++ b/extras/create_new_xlator/generate_xlator.py
@@ -0,0 +1,208 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import re
+import sys
+import string
+import time
+path = os.path.abspath(os.path.dirname(__file__)) + '/../../libglusterfs/src'
+sys.path.append(path)
+from generator import ops, xlator_cbks, xlator_dumpops
+
+MAKEFILE_FMT = """
+xlator_LTLIBRARIES = @XL_NAME@.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/@XL_TYPE@
+@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+@XL_NAME_NO_HYPHEN@_la_SOURCES = @XL_NAME@.c
+@XL_NAME_NO_HYPHEN@_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = @XL_NAME@.h @XL_NAME@-mem-types.h @XL_NAME@-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+           -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
+"""
+
+fop_subs = {}
+cbk_subs = {}
+fn_subs = {}
+
+
+def get_error_arg(type_str):
+        if type_str.find(" *") != -1:
+                return "NULL"
+        return "-1"
+
+
+def get_param(names, types):
+        # Convert two separate tuples to one of (name, type) sub-tuples.
+        as_tuples = list(zip(types, names))
+        # Convert each sub-tuple into a "type name" string.
+        as_strings = [' '.join(item) for item in as_tuples]
+        # Join all of those into one big string.
+        return ',\n\t'.join(as_strings)
+
+
+def generate(tmpl, name, table):
+        w_arg_names = [a[1] for a in table[name] if a[0] == 'fop-arg']
+        w_arg_types = [a[2] for a in table[name] if a[0] == 'fop-arg']
+        u_arg_names = [a[1] for a in table[name] if a[0] == 'cbk-arg']
+        u_arg_types = [a[2] for a in table[name] if a[0] == 'cbk-arg']
+        fn_arg_names = [a[1] for a in table[name] if a[0] == 'fn-arg']
+        fn_arg_types = [a[2] for a in table[name] if a[0] == 'fn-arg']
+        ret_type = [a[1] for a in table[name] if a[0] == 'ret-val']
+        ret_var = [a[2] for a in table[name] if a[0] == 'ret-val']
+
+        sdict = {}
+        #Parameters are (t1, var1), (t2, var2)...
+        #Args are (var1, var2,...)
+        sdict["@WIND_ARGS@"] = ', '.join(w_arg_names)
+        sdict["@UNWIND_ARGS@"] = ', '.join(u_arg_names)
+        sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, u_arg_types)))
+        sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types)
+        sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types)
+        sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types)
+        sdict["@NAME@"] = name
+        sdict["@FOP_PREFIX@"] = fop_prefix
+        sdict["@RET_TYPE@"] = ''.join(ret_type)
+        sdict["@RET_VAR@"] = ''.join(ret_var)
+
+        for old, new in sdict.items():
+                tmpl = tmpl.replace(old, new)
+        # TBD: reindent/reformat the result for maximum readability.
+        return tmpl
+
+
+def gen_xlator():
+        xl = open(src_dir_path+"/"+xl_name+".c", 'w+')
+
+        print(COPYRIGHT, file=xl)
+        print(fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@",
+                                                              xl_name), file=xl)
+
+        #Generate cbks and fops
+        for fop in ops:
+                print(generate(fragments["CBK_TEMPLATE"], fop, ops), file=xl)
+                print(generate(fragments["FOP_TEMPLATE"], fop, ops), file=xl)
+
+        for cbk in xlator_cbks:
+                print(generate(fragments["FUNC_TEMPLATE"], cbk,
+                                      xlator_cbks), file=xl)
+
+        for dops in xlator_dumpops:
+                print(generate(fragments["FUNC_TEMPLATE"], dops,
+                                      xlator_dumpops), file=xl)
+
+        #Generate fop table
+        print("struct xlator_fops fops = {", file=xl)
+        for fop in ops:
+                print("        .{0:20} = {1}_{2},".format(fop, fop_prefix, fop), file=xl)
+        print("};", file=xl)
+
+        #Generate xlator_cbks table
+        print("struct xlator_cbks cbks = {", file=xl)
+        for cbk in xlator_cbks:
+                print("        .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk), file=xl)
+        print("};", file=xl)
+
+        #Generate xlator_dumpops table
+        print("struct xlator_dumpops dumpops = {", file=xl)
+        for dops in xlator_dumpops:
+                print("        .{0:20} = {1}_{2},".format(dops, fop_prefix, dops), file=xl)
+        print("};", file=xl)
+
+        xlator_methods = fragments["XLATOR_METHODS"].replace("@XL_NAME@", xl_name)
+        xlator_methods = xlator_methods.replace("@FOP_PREFIX@", fop_prefix)
+        print(xlator_methods, file=xl)
+
+        xl.close()
+
+
+def create_dir_struct():
+        if not os.path.exists(dir_path+"/src"):
+                os.makedirs(dir_path+"/src")
+
+
+def gen_header_files():
+        upname = xl_name_no_hyphen.upper()
+        h = open(src_dir_path+"/"+xl_name+".h", 'w+')
+        print(COPYRIGHT, file=h)
+        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname)
+        txt = txt.replace("@XL_NAME@", xl_name)
+        print(txt, file=h)
+        h.close()
+
+        h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+')
+        print(COPYRIGHT, file=h)
+        txt = fragments["MEM_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES")
+        txt = txt.replace("@FOP_PREFIX@", fop_prefix)
+        print(txt, file=h)
+        h.close()
+
+        h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+')
+        print(COPYRIGHT, file=h)
+        txt = fragments["MSG_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES")
+        txt = txt.replace("@FOP_PREFIX@", fop_prefix.upper())
+        print(txt, file=h)
+        h.close()
+
+
+def gen_makefiles():
+        m = open(dir_path+"/Makefile.am", 'w+')
+        print("SUBDIRS = src\n\nCLEANFILES =", file=m)
+        m.close()
+
+        m = open(src_dir_path+"/Makefile.am", 'w+')
+        txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name)
+        txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen)
+        txt = txt.replace("@XL_TYPE@", xlator_type)
+        print(txt, file=m)
+        m.close()
+
+def get_copyright ():
+        return fragments["CP"].replace("@CURRENT_YEAR@",
+                                       time.strftime("%Y"))
+
+def load_fragments ():
+        pragma_re = re.compile('pragma fragment (.*)')
+        cur_symbol = None
+        cur_value = ""
+        result = {}
+        basepath = os.path.abspath(os.path.dirname(__file__))
+        fragpath = basepath + "/new-xlator.c.tmpl"
+        for line in open(fragpath, "r").readlines():
+                m = pragma_re.search(line)
+                if m:
+                        if cur_symbol:
+                                result[cur_symbol] = cur_value
+                        cur_symbol = m.group(1)
+                        cur_value = ""
+                else:
+                        cur_value += line
+        if cur_symbol:
+                result[cur_symbol] = cur_value
+        return result
+
+if __name__ == '__main__':
+
+        if len(sys.argv) < 3:
+                print("USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>")
+                sys.exit(0)
+
+        xl_name = sys.argv[2]
+        xl_name_no_hyphen = xl_name.replace("-", "_")
+        if sys.argv[1].endswith('/'):
+                dir_path = sys.argv[1] + xl_name
+        else:
+                dir_path = sys.argv[1] + "/" + xl_name
+        xlator_type = os.path.basename(sys.argv[1])
+        fop_prefix = sys.argv[3]
+        src_dir_path = dir_path + "/src"
+
+        fragments = load_fragments()
+
+        COPYRIGHT = get_copyright()
+        create_dir_struct()
+        gen_xlator()
+        gen_header_files()
+        gen_makefiles()
diff --git a/extras/create_new_xlator/new-xlator.c.tmpl b/extras/create_new_xlator/new-xlator.c.tmpl
new file mode 100644
index 00000000000..fe9735bfcf1
--- /dev/null
+++ b/extras/create_new_xlator/new-xlator.c.tmpl
@@ -0,0 +1,151 @@
+#pragma fragment CBK_TEMPLATE
+int32_t @FOP_PREFIX@_@NAME@_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, @UNWIND_PARAMS@)
+{
+    STACK_UNWIND_STRICT(@NAME@, frame, op_ret, op_errno, @UNWIND_ARGS@);
+    return 0;
+}
+
+#pragma fragment COMMENT
+If you are generating the leaf xlators, remove the STACK_WIND and replace the
+    @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary
+
+#pragma fragment FOP_TEMPLATE
+        int32_t @FOP_PREFIX@_@NAME@(call_frame_t *frame, xlator_t *this, @WIND_PARAMS@)
+{
+    STACK_WIND(frame, @FOP_PREFIX@_@NAME@_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->@NAME@, @WIND_ARGS@);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(@NAME@, frame, -1, errno, @ERROR_ARGS@);
+    return 0;
+}
+
+#pragma fragment FUNC_TEMPLATE
+@RET_TYPE@ @FOP_PREFIX@_@NAME@(@FUNC_PARAMS@)
+{
+    return @RET_VAR@;
+}
+
+#pragma fragment CP
+/*
+ *   Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#pragma fragment INCLUDE_IN_SRC_FILE
+#include "@XL_NAME@.h"
+
+#pragma fragment XLATOR_METHODS
+
+static int32_t @FOP_PREFIX@_init(xlator_t *this)
+{
+    return 0;
+}
+
+static void @FOP_PREFIX@_fini(xlator_t *this)
+{
+    return;
+}
+
+static int32_t @FOP_PREFIX@_reconfigure(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+static int @FOP_PREFIX@_notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+static int32_t @FOP_PREFIX@_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_@FOP_PREFIX@_mt_end + 1);
+    return ret;
+}
+
+static int32_t @FOP_PREFIX@_dump_metrics(xlator_t *this, int fd)
+{
+    return 0;
+}
+
+struct volume_options @FOP_PREFIX@_options[] = {
+    /*{ .key  = {""},
+      .type = GF_OPTION_TYPE_BOOL,
+      .default_value = "",
+      .op_version = {GD_OP_VERSION_},
+      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+      .tags = {""},
+      .description = "",
+      .category = GF_EXPERIMENTAL,
+    },
+    { .key = {NULL} },
+    */
+};
+
+xlator_api_t xlator_api = {
+    .init = @FOP_PREFIX@_init,
+    .fini = @FOP_PREFIX@_fini,
+    .notify = @FOP_PREFIX@_notify,
+    .reconfigure = @FOP_PREFIX@_reconfigure,
+    .mem_acct_init = @FOP_PREFIX@_mem_acct_init,
+    .dump_metrics = @FOP_PREFIX@_dump_metrics,
+    .op_version = {GD_OP_VERSION_},
+    .dumpops = &@FOP_PREFIX@_dumpops,
+    .fops = &@FOP_PREFIX@_fops,
+    .cbks = &@FOP_PREFIX @_cbks,
+    .options = @FOP_PREFIX@_options,
+    .identifier = "@XL_NAME@",
+    .category = GF_EXPERIMENTAL,
+};
+#pragma fragment HEADER_FMT
+#ifndef __ @HFL_NAME@_H__
+#define __ @HFL_NAME@_H__
+
+#include "@XL_NAME@-mem-types.h"
+#include "@XL_NAME@-messages.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+#endif /* __@HFL_NAME@_H__ */
+
+#pragma fragment MEM_HEADER_FMT
+#ifndef __ @HFL_NAME@_H__
+#define __ @HFL_NAME@_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_mdc_mem_types_ {
+    gf_@FOP_PREFIX@_mt_ = gf_common_mt_end + 1,
+    gf_@FOP_PREFIX@_mt_end
+};
+
+#endif /* __@HFL_NAME@_H__ */
+
+#pragma fragment MSG_HEADER_FMT
+#ifndef __@HFL_NAME@_H__
+#define __@HFL_NAME@_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(@FOP_PREFIX@, @FOP_PREFIX@_MSG_NO_MEMORY);
+
+#endif /* __@HFL_NAME@_H__ */
diff --git a/extras/devel-tools/devel-vagrant/Vagrantfile b/extras/devel-tools/devel-vagrant/Vagrantfile
new file mode 100644
index 00000000000..78dc29bdc68
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/Vagrantfile
@@ -0,0 +1,165 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+#
+# Author: Rajesh Joseph (rjoseph@redhat.com)
+# Author: Christopher Blum (cblum@redhat.com)
+#
+
+
+#Variables
+box_name = "gluster-dev-fedora"
+box_url = "http://download.gluster.org/pub/gluster/glusterfs/vagrant/gluster-dev-fedora/boxes/gluster-dev-fedora.json"
+node_count = 0
+disk_count = -1
+node_name = "Node"
+ipbase="192.168.99."
+source_path = "/source/glusterfs"
+target_path = "/mnt/src"
+
+if ARGV[0] == "up"
+        environment = open('vagrant_env.conf', 'w')
+
+        print "\n\e[1;37mEnter Node (or VM) Count? Default: 1 \e[32m"
+        while node_count < 1 or node_count > 99
+                node_count = $stdin.gets.strip.to_i
+                if node_count == 0 # The user pressed enter without input or we cannot parse the input to a number
+                        node_count = 1
+                elsif node_count < 1
+                        print "\e[31mWe need at least 1 VM ;) Try again \e[32m"
+                elsif node_count > 99
+                        print "\e[31mWe don't support more than 99 VMs - Try again \e[32m"
+                end
+        end
+
+        print "\e[1;37mEnter per Node Disc (Brick) Count? Default: 2 \e[32m"
+
+        while disk_count < 1
+                disk_count = $stdin.gets.strip.to_i
+                if disk_count == 0 # The user pressed enter without input or we cannot parse the input to a number
+                        disk_count = 2
+                elsif disk_count < 1
+                        print "\e[31mWe need at least 1 disk ;) Try again \e[32m"
+                end
+        end
+
+        print "\e[1;37mEnter GlusterFS source location? Default: \"#{source_path}\" : \e[32m"
+        tmploc = $stdin.gets.strip.to_s
+        if tmploc != ""
+                source_path = "#{tmploc}"
+        end
+
+        environment.puts("# BEWARE: Do NOT modify ANY settings in here or your vagrant environment will be messed up")
+        environment.puts(node_count.to_s)
+        environment.puts(disk_count.to_s)
+        environment.puts(source_path)
+
+        print "\e[32m\nOK I will provision #{node_count} VMs for you and each one will have #{disk_count} disks for bricks\e[37m\n\n"
+        system "sleep 1"
+else # So that we destroy and can connect to all VMs...
+        environment = open('vagrant_env.conf', 'r')
+
+        environment.readline # Skip the comment on top
+        node_count = environment.readline.to_i
+        disk_count = environment.readline.to_i
+        source_path = environment.readline.gsub(/\s+/, "")
+
+        if ARGV[0] != "ssh-config"
+                puts "Detected settings from previous vagrant up:"
+                puts "  We deployed #{node_count} VMs with each #{disk_count} disks"
+                puts ""
+        end
+end
+
+environment.close
+
+$ansivar = Hash.new{ |hash,key| hash[key] = [] }
+$devnamecreated = false
+
+#
+# Function to create and attach disks
+#
+def attachDisks(numDisk, provider)
+        suffix = "bcdefghijklmn".split("")
+        for i in 1..numDisk.to_i
+                devname = "vd" + (suffix[i-1]).to_s
+                if $devnamecreated == false
+                        $ansivar["device"].push "#{devname}"
+                end
+                provider.storage :file,
+                        :size           => '1G',
+                        :device         => "vd" + (suffix[i-1]).to_s,
+                        :type           => "qcow2",
+                        :bus            => "virtio",
+                        :cache          => "default"
+        end
+        $devnamecreated = true
+end
+
+
+$ansivar["src_path"].push "#{source_path}"
+$ansivar["trg_path"].push "#{target_path}"
+
+groups = Hash.new{ |hash,key| hash[key] = [] }
+
+groups["origin"].push "#{node_name}1"
+groups["all"].push "#{node_name}1"
+
+(2..node_count).each do |num|
+        $ansivar["peer_nodes"].push "#{node_name}#{num}"
+        groups["all"].push "#{node_name}#{num}"
+end
+
+hostsFile = "\n"
+(1..node_count).each do |num|
+        hostsFile += "#{ipbase}#{( 100 + num).to_s} #{node_name}#{num.to_s}\n"
+end
+
+Vagrant.configure("2") do |config|
+    (1..node_count).each do |num|
+        config.vm.define "#{node_name}#{num}" do |node|
+            ip_addr = "#{ipbase}#{(100 + num).to_s}"
+            node.vm.network "private_network", ip: "#{ip_addr}"
+            node.vm.box = box_name
+            node.vm.box_url = box_url
+            node.vm.hostname = "#{node_name}#{num}"
+            node.ssh.insert_key = false
+            node.vm.synced_folder "#{source_path}", "#{target_path}", type: "nfs"
+
+            # Define basic config for VM, memory, cpu, storage pool
+            node.vm.provider "libvirt" do |virt|
+                virt.storage_pool_name = "default"
+                virt.memory = 1024
+                virt.cpus = 1
+
+                attachDisks( disk_count, virt )
+            end
+
+            node.vm.post_up_message = "\e[37mBuilding of this VM is finished \n"
+                                      "You can access it now with: \n"
+                                      "vagrant ssh #{node_name}#{num.to_s}\n\n"
+                                      "#{target_path} directory in VM #{node_name}#{num.to_s}"
+                                      "is synced with Host machine. \nSo any changes done in this"
+                                      "directory will be reflected in the host machine as well\n"
+                                      "Beware of this when you delete content from this directory\e[32m"
+
+            node.vm.provision :shell, path: "bootstrap.sh"
+
+            node.vm.provision "shell", inline: <<-SHELL
+                echo '#{hostsFile}' | sudo tee -a /etc/hosts
+            SHELL
+
+            if num == node_count
+                # Let's provision
+                node.vm.provision "ansible" do |setup|
+                    setup.verbose = "v"
+                    setup.playbook = "ansible/setup.yml"
+                    setup.limit = "all"
+                    setup.sudo = "true"
+                    setup.groups = groups
+                    setup.extra_vars = $ansivar
+                end
+            end
+
+        end
+    end
+end
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml
new file mode 100644
index 00000000000..3306c7a3dc2
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/cluster/tasks/main.yml
@@ -0,0 +1,5 @@
+---
+- name: gluster peer probe
+  shell: gluster peer probe {{ item }}
+  with_items: "{{ peer_nodes | default([]) }}"
+
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml
new file mode 100644
index 00000000000..6ee258c7780
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/compile-gluster/tasks/main.yml
@@ -0,0 +1,29 @@
+---
+- name: autogen.sh
+  shell: chdir={{ item }} ./autogen.sh
+  with_items: "{{ trg_path }}"
+
+- name: configure
+  shell: chdir={{ item }} CFLAGS="-g -O0 -Werror -Wall -Wno-error=cpp -Wno-error=maybe-uninitialized" \
+        ./configure \
+        --prefix=/usr \
+        --exec-prefix=/usr \
+        --bindir=/usr/bin \
+        --sbindir=/usr/sbin \
+        --sysconfdir=/etc \
+        --datadir=/usr/share \
+        --includedir=/usr/include \
+        --libdir=/usr/lib64 \
+        --libexecdir=/usr/libexec \
+        --localstatedir=/var \
+        --sharedstatedir=/var/lib \
+        --mandir=/usr/share/man \
+        --infodir=/usr/share/info \
+        --libdir=/usr/lib64 \
+        --enable-debug
+  with_items: "{{ trg_path }}"
+
+- name: make install
+  shell: chdir={{ item }} make install
+  with_items: "{{ trg_path }}"
+
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml
new file mode 100644
index 00000000000..3944054dd25
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/install-pkgs/tasks/main.yml
@@ -0,0 +1,72 @@
+---
+- name: install deltarpm
+  dnf:  name=deltarpm state=present
+
+- name: update system
+  shell: dnf  update -y
+
+- name: install other packages
+  dnf:  name={{ item }} state=present
+  with_items:
+    - attr
+    - autoconf
+    - automake
+    - bison
+    - cifs-utils
+    - cscope
+    - ctags
+    - dbench
+    - dos2unix
+    - e2fsprogs
+    - findutils
+    - flex
+    - fuse-devel
+    - fuse-libs
+    - gcc
+    - gdb
+    - git
+    - glib2-devel
+    - hostname
+    - libacl-devel
+    - libaio-devel
+    - libattr-devel
+    - libibverbs-devel
+    - librdmacm-devel
+    - libtool
+    - libxml2-devel
+    - lvm2-devel
+    - make
+    - man-db
+    - mock
+    - net-tools
+    - nfs-utils
+    - openssh-server
+    - openssl-devel
+    - perl-Test-Harness
+    - pkgconfig
+    - procps-ng
+    - psmisc
+    - python-devel
+    - python-eventlet
+    - python-netifaces
+    - python-paste-deploy
+    - python-setuptools
+    - python-simplejson
+    - python-sphinx
+    - python-webob
+    - pyxattr
+    - readline-devel
+    - rpm-build
+    - screen
+    - strace
+    - supervisor
+    - systemtap-sdt-devel
+    - sqlite-devel
+    - samba*
+    - userspace-rcu-devel
+    - vim
+    - wget
+    - which
+    - xfsprogs
+    - yajl-devel
+
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml
new file mode 100644
index 00000000000..768cb0e8668
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/iptables/tasks/main.yml
@@ -0,0 +1,3 @@
+---
+- name: disable iptables, need to add specific rules later
+  shell: iptables -F
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml
new file mode 100644
index 00000000000..a3a6c463468
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/prepare-brick/tasks/main.yml
@@ -0,0 +1,30 @@
+---
+- name: Create physical device
+  shell: pvcreate /dev/{{ item }}
+  with_items: "{{ device }}"
+
+- name: Create volume group
+  shell: vgcreate vg{{ item }} /dev/{{ item }}
+  with_items: "{{ device }}"
+
+- name: Create thin pool
+  shell: lvcreate -L 950M -T vg{{ item }}/thinpool
+  with_items: "{{ device }}"
+
+- name: Create thin volume
+  shell: lvcreate -V900M -T vg{{ item }}/thinpool -n thinp1
+  with_items: "{{ device }}"
+
+- name: Format backend
+  filesystem: fstype=xfs dev=/dev/vg{{ item }}/thinp1
+  with_items: "{{ device }}"
+
+- name: Create mount directory
+  file: path=/bricks/br{{ item }} state=directory recurse=yes
+  with_items: "{{ device }}"
+
+- name: Add entry to fstab and mount
+  mount: name=/bricks/br{{ item }} src=/dev/vg{{ item }}/thinp1 fstype=xfs state=mounted
+  with_items: "{{ device }}"
+
+
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml
new file mode 100644
index 00000000000..c9ba9618428
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/selinux/tasks/main.yml
@@ -0,0 +1,3 @@
+---
+- name: Allow gfapi in Samba to bind to other ports than well known smb ports
+  seboolean: name=samba_load_libgfapi state=yes persistent=yes
diff --git a/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml
new file mode 100644
index 00000000000..ef78a125ae8
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/roles/service/tasks/main.yml
@@ -0,0 +1,21 @@
+---
+- name: disable kernel nfs
+  service: name=nfs-server enabled=no
+
+- name: stop kernel nfs
+  service: name=nfs-server state=stopped
+
+- name: enable rpcbind
+  service: name=rpcbind enabled=yes
+
+- name: start rpcbind
+  service: name=rpcbind state=started
+
+- name: enable glusterd
+  service: name=glusterd enabled=yes
+
+- name: start glusterd
+  service: name=glusterd state=started
+
+
+
diff --git a/extras/devel-tools/devel-vagrant/ansible/setup.yml b/extras/devel-tools/devel-vagrant/ansible/setup.yml
new file mode 100644
index 00000000000..c26bd7d6051
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/ansible/setup.yml
@@ -0,0 +1,23 @@
+---
+- hosts: all
+  become: yes
+  become_method: sudo
+  roles:
+    - install-pkgs
+    - prepare-brick
+    - selinux
+    - iptables
+
+- hosts: all
+  become: yes
+  become_method: sudo
+  serial: 1
+  roles:
+    - compile-gluster
+    - service
+
+- hosts: origin
+  become: yes
+  become_method: sudo
+  roles:
+    - cluster
diff --git a/extras/devel-tools/devel-vagrant/bootstrap.sh b/extras/devel-tools/devel-vagrant/bootstrap.sh
new file mode 100644
index 00000000000..bbf9fa2c063
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/bootstrap.sh
@@ -0,0 +1,3 @@
+dnf install -y nfs-utils
+dnf install -y vim
+dnf install -y python2 python2-dnf libselinux-python libsemanage-python
diff --git a/extras/devel-tools/devel-vagrant/up.sh b/extras/devel-tools/devel-vagrant/up.sh
new file mode 100755
index 00000000000..35cbe79d835
--- /dev/null
+++ b/extras/devel-tools/devel-vagrant/up.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+vagrant up --no-provision $@
+vagrant provision
diff --git a/extras/devel-tools/gdb_macros b/extras/devel-tools/gdb_macros
new file mode 100644
index 00000000000..aae4ccb3b37
--- /dev/null
+++ b/extras/devel-tools/gdb_macros
@@ -0,0 +1,84 @@
+
+#
+# gdb_macros is a gdb script file which can assist
+# developer in debugging. This script provides
+# following functions for debugging gluster processes
+# effectively:
+#
+# pdict : This function will iterate through all the
+#         dictionary (dict_t) members and print the
+#         key-value pair.
+#
+# plist : This function will print address of each member
+#         of gluster list (list_head).
+#
+# gdb script should be loaded in gdb before using these
+# functions. gdb script can be loaded on-demand or on gdb
+# start-up. Use the following command on gdb prompt for
+# loading it on-demand.
+#       source <script filename>
+# e.g.
+# (gdb) source /mnt/gdb_macros
+#
+# To automatically load gdb script on startup use .gdbinit
+# file. This file is automatically loaded by gdb on start.
+# Edit (or create) ~/.gdbinit file and add the following
+# entry:
+# source /mnt/gdb_macros
+#
+
+
+
+# This is an internal helper function
+define _print_dict_data
+        set $data = ($arg0)
+        set $len = $data->len - 1
+        set $i = 0
+        set $ishex = 0
+        while ($i < $len)
+                set $ch = $data->data [$i]
+
+                # Print only alpha-numeric values as char
+                # and remaining as hex value. This is done
+                # in this way because using %s screws up
+                # the display if the string has non-displayable
+                # characters
+                if ($ishex == 0) && ($ch > 31) && ($ch < 127)
+                        printf "%c", $ch
+                else
+                        printf "%x", ($ch & 0xFF)
+                        set $ishex = 1
+                end
+                set $i = $i + 1
+        end
+end
+
+
+define pdict
+        set $cnt = 1
+        set $temp = *($arg0->members)
+        while ($temp)
+                printf "[%d](%s::",$cnt, $temp->key
+                _print_dict_data ($temp)->value
+                printf ")\n"
+                set $temp = $temp->next
+                set $cnt = $cnt + 1
+        end
+        printf "Done\n"
+end
+
+
+define plist
+        set $node = &($arg0)
+        set $head = $node
+
+        printf "[0x%lx]", $node
+        set $node = $node->next
+
+        while ($node != $head)
+                printf "--> [0x%lx]", $node
+                set $node = $node->next
+        end
+        printf "\n"
+end
+
diff --git a/extras/devel-tools/print-backtrace.sh b/extras/devel-tools/print-backtrace.sh
new file mode 100755
index 00000000000..33fbae288bc
--- /dev/null
+++ b/extras/devel-tools/print-backtrace.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+# sample unresolved backtrace lines picked up from a brick log that should go
+# into a backtrace file eg. bt-file.txt:
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6]
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323]
+#
+# following is the output of the script for the above backtrace lines:
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3ec81)[0x7fe4bc271c81] __afr_selfheal_data_finalize_source inlined at /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:684 in __afr_selfheal_data_prepare /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:603
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3eecd)[0x7fe4bc271ecd] __afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:740
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x404cb)[0x7fe4bc2734cb] afr_selfheal_data /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-data.c:883
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d2b6)[0x7fe4bc2702b6] afr_selfheal_do /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:1968
+# /usr/lib64/glusterfs/3.8.4/xlator/cluster/replicate.so(+0x3d323)[0x7fe4bc270323] afr_selfheal /usr/src/debug/glusterfs-3.8.4/xlators/cluster/afr/src/afr-self-heal-common.c:2015
+#
+# Usage with debuginfo RPM:
+# print-backtrace.sh $HOME/Downloads/glusterfs-debuginfo-3.8.4-10.el7.x86_64.rpm bt-file.txt
+#
+# Usage with source install:
+# print-packtrace.sh none bt-file.txt
+
+function version_compare() { test $(echo $1|awk -F '.' '{print $1 $2 $3}') -gt $(echo $2|awk -F '.' '{print $1 $2 $3}'); }
+
+function Usage()
+{
+        echo -e "Usage:\n\t$0 { none | <debuginfo-rpm> } <backtrace-file>"
+        echo "none: implies we don't have a debuginfo rpm but want to resolve"
+        echo "      against a source install which already has the debuginfo"
+        echo "      NOTE: in this case you should have configured the build"
+        echo "            with --enable-debug and the linker options should"
+        echo "            have the option -rdynamic"
+}
+
+debuginfo_rpm=$1
+backtrace_file=$2
+
+if [ ! $debuginfo_rpm ] || [ ! $backtrace_file ]; then
+        Usage
+        exit 1
+fi
+
+if [ $debuginfo_rpm != "none" ]; then
+        if [ ! -f $debuginfo_rpm ]; then
+                echo "no such rpm file: $debuginfo_rpm"
+                exit 1
+        fi
+fi
+
+if [ ! -f $backtrace_file ]; then
+        echo "no such backtrace file: $backtrace_file"
+        exit 1
+fi
+
+if [ "$debuginfo_rpm" != "none" ]; then
+        if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then
+                echo "file does not look like an rpm: $debuginfo_rpm"
+                exit 1
+        fi
+fi
+
+cpio_version=$(cpio --version|grep cpio|cut -f 2 -d ')'|sed -e 's/^[[:space:]]*//')
+rpm_name=""
+debuginfo_path=""
+debuginfo_extension=""
+
+if [ $debuginfo_rpm != "none" ]; then
+        # extract the gluster debuginfo rpm to resolve the symbols against
+        rpm_name=$(basename $debuginfo_rpm '.rpm')
+        if [ -d $rpm_name ]; then
+                echo "directory already exists: $rpm_name"
+                echo "please remove/move it and reattempt"
+                exit 1
+        fi
+        mkdir -p $rpm_name
+        if version_compare $cpio_version "2.11"; then
+                rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name
+                ret=$?
+        else
+                current_dir="$PWD"
+                cd $rpm_name
+                rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time
+                ret=$?
+                cd $current_dir
+        fi
+        if [ $ret -eq 1 ]; then
+                echo "failed to extract rpm $debuginfo_rpm to $PWD/$rpm_name directory"
+                rm -rf $rpm_name
+                exit 1
+        fi
+        debuginfo_path="$PWD/$rpm_name/usr/lib/debug"
+        debuginfo_extension=".debug"
+else
+        debuginfo_path=""
+        debuginfo_extension=""
+fi
+
+# NOTE: backtrace file should contain only the lines which need to be resolved
+for bt in $(cat $backtrace_file)
+do
+        libname=$(echo $bt | cut -f 1 -d '(')
+        addr=$(echo $bt | cut -f 2 -d '(' | cut -f 1 -d ')')
+        libpath=${debuginfo_path}${libname}${debuginfo_extension}
+        if [ ! -f $libpath ]; then
+                continue
+        fi
+        newbt=( $(eu-addr2line --functions --exe=$libpath $addr) )
+        echo "$bt ${newbt[*]}"
+done
+
+# remove the temporary directory
+if [ -d $rpm_name ]; then
+        rm -rf $rpm_name
+fi
+
diff --git a/extras/devel-tools/strace-brick.sh b/extras/devel-tools/strace-brick.sh
new file mode 100755
index 00000000000..a140729111c
--- /dev/null
+++ b/extras/devel-tools/strace-brick.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+# Usage:
+# nice -n -19 strace-brick.sh glusterfsd 50
+
+brick_process_name=$1
+min_watch_cpu=$2
+if [ ! $brick_process_name ]; then
+        brick_process_name=glusterfsd
+fi
+
+if [ ! $min_watch_cpu ]; then
+        min_watch_cpu=50
+fi
+
+echo "min_watch_cpu: $min_watch_cpu"
+
+break=false
+
+while ! $break;
+do
+        mypids=( $(pgrep $brick_process_name) )
+        echo "mypids: ${mypids[*]}"
+
+        pid_args=$(echo ${mypids[*]} | sed -e 's/ / -p /g;s/^/-p /')
+        echo "pid_args: $pid_args"
+
+        pcpu=( $(ps $pid_args -o pcpu -h ) )
+        echo "pcpu: ${pcpu[*]}"
+
+        wait_longer=false
+
+        for i in $( seq 0 $((${#pcpu[*]} - 1)) )
+        do
+                echo "i: $i"
+                echo "mypids[$i]: ${mypids[$i]}"
+
+                int_pcpu=$(echo ${pcpu[$i]} | cut -f 1 -d '.')
+                echo "int_pcpu: $int_pcpu"
+                if [ ! $int_pcpu ] || [ ! $min_watch_cpu ]; then
+                        break=true
+                        echo "breaking"
+                fi
+                if [ $int_pcpu -ge $min_watch_cpu ]; then
+                        wait_longer=true
+                        mydirname="${brick_process_name}-${mypids[$i]}-$(date --utc +'%Y%m%d-%H%M%S.%N')"
+                        $(mkdir $mydirname && cd $mydirname && timeout --kill-after=5 --signal=KILL 60 nice -n -19 strace -p ${mypids[$i]} -ff -tt -T -o $brick_process_name) &
+                fi
+        done
+
+        if $wait_longer; then
+                sleep 90
+        else
+                sleep 15
+        fi
+done
diff --git a/extras/distributed-testing/README b/extras/distributed-testing/README
new file mode 100644
index 00000000000..928d943f211
--- /dev/null
+++ b/extras/distributed-testing/README
@@ -0,0 +1,28 @@
+PROBLEM
+
+The testing methodology of Gluster is extremely slow. It takes a very long time (6+ hrs) to run the basic tests on a single machine. It takes about 20+ hours to run code analysis version of tests like valgrind, asan, tsan etc.
+
+SOLUTION
+
+The fundamental problem is that the tests cannot be parallelized on a single machine. The natural solution is to run these tests on a cluster of machines. In a nutshell, apply map-reduce to run unit tests.
+
+WORK @ Facebook
+
+At Facebook we have applied the map-reduce approach to testing and have observed 10X improvements.
+
+The solution supports the following
+
+Distribute tests across machines, collect results/logs
+Share worker pool across different testers
+Try failure 3 times on 3 different machines before calling it a failure
+Support running asan, valgrind, asan-noleaks
+Self management of worker pools. The clients will manage the worker pool including version update, no manual maintenance required
+WORK
+
+Port the code from gluster-fb-3.8 to gluster master
+
+HOW TO RUN
+
+./extras/distributed-testing/distributed-test.sh --hosts '<h1> <h2> <h3>'
+
+All hosts should have no password for ssh via root. This can be achieved with keys setup on the client and the server machines.
diff --git a/extras/distributed-testing/distributed-test-build-env b/extras/distributed-testing/distributed-test-build-env
new file mode 100644
index 00000000000..cd68ff717da
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-build-env
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+GF_CONF_OPTS="--localstatedir=/var --sysconfdir /var/lib --prefix /usr --libdir /usr/lib64 \
+            --enable-bd-xlator=yes --enable-debug --enable-gnfs"
+
+if [ -x /usr/lib/rpm/redhat/dist.sh ]; then
+  REDHAT_MAJOR=$(/usr/lib/rpm/redhat/dist.sh --distnum)
+else
+  REDHAT_MAJOR=0
+fi
+
+ASAN_ENABLED=${ASAN_ENABLED:=0}
+if [ "$ASAN_ENABLED" -eq "1" ]; then
+    GF_CONF_OPTS="$GF_CONF_OPTS --with-asan"
+fi
+
+GF_CONF_OPTS="$GF_CONF_OPTS --with-systemd"
+export GF_CONF_OPTS
+
+export CFLAGS="-O0 -ggdb -fPIC -Wall"
diff --git a/extras/distributed-testing/distributed-test-build.sh b/extras/distributed-testing/distributed-test-build.sh
new file mode 100755
index 00000000000..e8910d8425c
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+EXTRA_CONFIGURE_ARGS="$@"
+ASAN_REQUESTED=false
+for arg in $EXTRA_CONFIGURE_ARGS; do
+  if [ $arg == "--with-asan" ]; then
+    echo "Requested ASAN, cleaning build first."
+    make -j distclean || true
+    touch .with_asan
+    ASAN_REQUESTED=true
+  fi
+done
+
+if [ $ASAN_REQUESTED == false ]; then
+  if [ -f .with_asan ]; then
+    echo "Previous build was with ASAN, cleaning build first."
+    make -j distclean || true
+    rm -v .with_asan
+  fi
+fi
+
+source extras/distributed-testing/distributed-test-build-env
+./autogen.sh
+./configure $GF_CONF_OPTS $EXTRA_CONFIGURE_ARGS
+make -j
diff --git a/extras/distributed-testing/distributed-test-env b/extras/distributed-testing/distributed-test-env
new file mode 100644
index 00000000000..36fdd82e5dd
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-env
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+SMOKE_TESTS="\
+    tests/basic/*.t\
+    tests/basic/afr/*.t\
+    tests/basic/distribute/*.t\
+    tests/bugs/fb*.t\
+    tests/features/brick-min-free-space.t\
+"
+
+KNOWN_FLAKY_TESTS="\
+"
+
+BROKEN_TESTS="\
+  tests/features/lock_revocation.t\
+  tests/features/recon.t\
+  tests/features/fdl-overflow.t\
+  tests/features/fdl.t\
+  tests/features/ipc.t\
+  tests/bugs/distribute/bug-1247563.t\
+  tests/bugs/distribute/bug-1543279.t\
+  tests/bugs/distribute/bug-1066798.t\
+  tests/bugs/ec/bug-1304988.t\
+  tests/bugs/unclassified/bug-1357397.t\
+  tests/bugs/quota/bug-1235182.t\
+  tests/bugs/fuse/bug-1309462.t\
+  tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\
+  tests/bugs/stripe/bug-1002207.t\
+  tests/bugs/stripe/bug-1111454.t\
+  tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
+  tests/bugs/write-behind/bug-1279730.t\
+  tests/bugs/gfapi/bug-1093594.t\
+  tests/bugs/replicate/bug-1473026.t\
+  tests/bugs/replicate/bug-802417.t\
+  tests/basic/inode-leak.t\
+  tests/basic/distribute/force-migration.t\
+  tests/basic/ec/heal-info.t\
+  tests/basic/ec/ec-seek.t\
+  tests/basic/jbr/jbr-volgen.t\
+  tests/basic/jbr/jbr.t\
+  tests/basic/afr/tarissue.t\
+  tests/basic/tier/tierd_check.t\
+  tests/basic/gfapi/bug1291259.t\
+"
+
+SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ')
+KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ')
+BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ')
diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py
new file mode 100755
index 00000000000..5a07e2feab1
--- /dev/null
+++ b/extras/distributed-testing/distributed-test-runner.py
@@ -0,0 +1,859 @@
+#!/usr/bin/python3
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+import re
+import sys
+import fcntl
+import base64
+import threading
+import socket
+import os
+import shlex
+import argparse
+import subprocess
+import time
+import SimpleXMLRPCServer
+import xmlrpclib
+import md5
+import httplib
+import uuid
+
+DEFAULT_PORT = 9999
+TEST_TIMEOUT_S = 15 * 60
+CLIENT_CONNECT_TIMEOUT_S = 10
+CLIENT_TIMEOUT_S = 60
+PATCH_FILE_UID = str(uuid.uuid4())
+SSH_TIMEOUT_S = 10
+MAX_ATTEMPTS = 3
+ADDRESS_FAMILY = 'IPv4'
+
+
+def socket_instance(address_family):
+    if address_family.upper() == 'ipv4'.upper():
+        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    elif address_family.upper() == 'ipv6'.upper():
+        return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        Log.error("Invalid IP address family")
+        sys.exit(1)
+
+
+def patch_file():
+    return "/tmp/%s-patch.tar.gz" % PATCH_FILE_UID
+
+# ..............................................................................
+# SimpleXMLRPCServer IPvX Wrapper
+# ..............................................................................
+
+
+class GeneralXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
+    def __init__(self, addr):
+        SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr)
+
+    def server_bind(self):
+        if self.socket:
+            self.socket.close()
+        self.socket = socket_instance(args.address_family)
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)
+
+
+class HTTPConnection(httplib.HTTPConnection):
+    def __init__(self, host):
+        self.host = host
+        httplib.HTTPConnection.__init__(self, host)
+
+    def connect(self):
+        old_timeout = socket.getdefaulttimeout()
+        self.sock = socket.create_connection((self.host, self.port),
+                                             timeout=CLIENT_CONNECT_TIMEOUT_S)
+        self.sock.settimeout(old_timeout)
+
+
+class IPTransport(xmlrpclib.Transport):
+    def __init__(self, *args, **kwargs):
+        xmlrpclib.Transport.__init__(self, *args, **kwargs)
+
+    def make_connection(self, host):
+        return HTTPConnection(host)
+
+
+# ..............................................................................
+# Common
+# ..............................................................................
+
+
+class Timer:
+    def __init__(self):
+        self.start = time.time()
+
+    def elapsed_s(self):
+        return int(time.time() - self.start)
+
+    def reset(self):
+        ret = self.elapsed_s()
+        self.start = time.time()
+        return ret
+
+
+def encode(buf):
+    return base64.b16encode(buf)
+
+
+def decode(buf):
+    return base64.b16decode(buf)
+
+
+def get_file_content(path):
+    with open(path, "r") as f:
+        return f.read()
+
+
+def write_to_file(path, data):
+    with open(path, "w") as f:
+        f.write(data)
+
+
+def failsafe(fn, args=()):
+    try:
+        return (True, fn(*args))
+    except (xmlrpclib.Fault, xmlrpclib.ProtocolError, xmlrpclib.ResponseError,
+            Exception) as err:
+        Log.debug(str(err))
+    return (False, None)
+
+
+class LogLevel:
+    DEBUG = 2
+    ERROR = 1
+    CLI = 0
+
+
+class Log:
+    LOGLEVEL = LogLevel.ERROR
+
+    @staticmethod
+    def _normalize(msg):
+        return msg[:100]
+
+    @staticmethod
+    def debug(msg):
+        if Log.LOGLEVEL >= LogLevel.DEBUG:
+            sys.stdout.write("<debug> %s\n" % Log._normalize(msg))
+            sys.stdout.flush()
+
+    @staticmethod
+    def error(msg):
+        sys.stderr.write("<error> %s\n" % Log._normalize(msg))
+
+    @staticmethod
+    def header(msg):
+        sys.stderr.write("* %s *\n" % Log._normalize(msg))
+
+    @staticmethod
+    def cli(msg):
+        sys.stderr.write("%s\n" % msg)
+
+
+class Shell:
+    def __init__(self, cwd=None, logpath=None):
+        self.cwd = cwd
+        self.shell = True
+        self.redirect = open(os.devnull if not logpath else logpath, "wr+")
+
+    def __del__(self):
+        self.redirect.close()
+
+    def cd(self, cwd):
+        Log.debug("cd %s" % cwd)
+        self.cwd = cwd
+
+    def truncate(self):
+        self.redirect.truncate(0)
+
+    def read_logs(self):
+        self.redirect.seek(0)
+        return self.redirect.read()
+
+    def check_call(self, cmd):
+        status = self.call(cmd)
+        if status:
+            raise Exception("Error running command %s. status=%s"
+                            % (cmd, status))
+
+    def call(self, cmd):
+        if isinstance(cmd, list):
+            return self._calls(cmd)
+
+        return self._call(cmd)
+
+    def ssh(self, hostname, cmd, id_rsa=None):
+        flags = "" if not id_rsa else "-i " + id_rsa
+        return self.call("timeout %s ssh %s root@%s \"%s\"" %
+                            (SSH_TIMEOUT_S, flags, hostname, cmd))
+
+    def scp(self, hostname, src, dest, id_rsa=None):
+        flags = "" if not id_rsa else "-i " + id_rsa
+        return self.call("timeout %s scp %s %s root@%s:%s" %
+                            (SSH_TIMEOUT_S, flags, src, hostname, dest))
+
+    def output(self, cmd, cwd=None):
+        Log.debug("%s> %s" % (cwd, cmd))
+        return subprocess.check_output(shlex.split(cmd), cwd=self.cwd)
+
+    def _calls(self, cmds):
+        Log.debug("Running commands. %s" % cmds)
+        for c in cmds:
+            status = self.call(c)
+            if status:
+                Log.error("Commands failed with %s" % status)
+                return status
+        return 0
+
+    def _call(self, cmd):
+        if not self.shell:
+            cmd = shlex.split(cmd)
+
+        Log.debug("%s> %s" % (self.cwd, cmd))
+
+        status = subprocess.call(cmd, cwd=self.cwd, shell=self.shell,
+                                 stdout=self.redirect, stderr=self.redirect)
+
+        Log.debug("return %s" % status)
+        return status
+
+
+# ..............................................................................
+# Server role
+# ..............................................................................
+
+class TestServer:
+    def __init__(self, port, scratchdir):
+        self.port = port
+        self.scratchdir = scratchdir
+        self.shell = Shell()
+        self.rpc = None
+        self.pidf = None
+
+        self.shell.check_call("mkdir -p %s" % self.scratchdir)
+        self._process_lock()
+
+    def __del__(self):
+        if self.pidf:
+            self.pidf.close()
+
+    def init(self):
+        Log.debug("Starting xmlrpc server on port %s" % self.port)
+        self.rpc = GeneralXMLRPCServer(("", self.port))
+        self.rpc.register_instance(Handlers(self.scratchdir))
+
+    def serve(self):
+        (status, _) = failsafe(self.rpc.serve_forever)
+        Log.cli("== End ==")
+
+    def _process_lock(self):
+        pid_filename = os.path.basename(__file__).replace("/", "-")
+        pid_filepath = "%s/%s.pid" % (self.scratchdir, pid_filename)
+        self.pidf = open(pid_filepath, "w")
+        try:
+            fcntl.lockf(self.pidf, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # We have the lock, kick anybody listening on this port
+            self.shell.call("kill $(lsof -t -i:%s)" % self.port)
+        except IOError:
+            Log.error("Another process instance is running")
+            sys.exit(0)
+
+#
+# Server Handler
+#
+
+
+handler_lock = threading.Lock()
+handler_serving_since = Timer()
+
+
+def synchronized(func):
+    def decorator(*args, **kws):
+        handler_lock.acquire()
+        h = args[0]
+        try:
+            h.shell.truncate()
+            ret = func(*args, **kws)
+            return ret
+        except Exception() as err:
+            Log.error(str(err))
+            Log.error(decode(h._log_content()))
+            raise
+        finally:
+            handler_lock.release()
+            handler_serving_since.reset()
+
+    return decorator
+
+
+class Handlers:
+    def __init__(self, scratchdir):
+        self.client_id = None
+        self.scratchdir = scratchdir
+        self.gluster_root = "%s/glusterfs" % self.scratchdir
+        self.shell = Shell(logpath="%s/test-handlers.log" % self.scratchdir)
+
+    def hello(self, id):
+        if not handler_lock.acquire(False):
+            return False
+        try:
+            return self._hello_locked(id)
+        finally:
+            handler_lock.release()
+
+    def _hello_locked(self, id):
+        if handler_serving_since.elapsed_s() > CLIENT_TIMEOUT_S:
+            Log.debug("Disconnected client %s" % self.client_id)
+            self.client_id = None
+
+        if not self.client_id:
+            self.client_id = id
+            handler_serving_since.reset()
+            return True
+
+        return (id == self.client_id)
+
+    @synchronized
+    def ping(self, id=None):
+        if id:
+            return id == self.client_id
+        return True
+
+    @synchronized
+    def bye(self, id):
+        assert id == self.client_id
+        self.client_id = None
+        handler_serving_since.reset()
+        return True
+
+    @synchronized
+    def cleanup(self, id):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        self.shell.check_call("PATH=.:$PATH; sudo ./clean_gfs_devserver.sh")
+        return True
+
+    @synchronized
+    def copy(self, id, name, content):
+        with open("%s/%s" % (self.scratchdir, name), "w+") as f:
+            f.write(decode(content))
+        return True
+
+    @synchronized
+    def copygzip(self, id, content):
+        assert id == self.client_id
+        gzipfile = "%s/tmp.tar.gz" % self.scratchdir
+        tarfile = "%s/tmp.tar" % self.scratchdir
+        self.shell.check_call("rm -f %s" % gzipfile)
+        self.shell.check_call("rm -f %s" % tarfile)
+        write_to_file(gzipfile, decode(content))
+
+        self.shell.cd(self.scratchdir)
+        self.shell.check_call("rm -r -f %s" % self.gluster_root)
+        self.shell.check_call("mkdir -p %s" % self.gluster_root)
+
+        self.shell.cd(self.gluster_root)
+        cmds = [
+            "gunzip -f -q %s" % gzipfile,
+            "tar -xvf %s" % tarfile
+        ]
+        return self.shell.call(cmds) == 0
+
+    @synchronized
+    def build(self, id, asan=False):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        self.shell.call("make clean")
+        env = "ASAN_ENABLED=1" if asan else ""
+        return self.shell.call(
+		"%s ./extras/distributed-testing/distributed-test-build.sh" % env) == 0
+
+    @synchronized
+    def install(self, id):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        return self.shell.call("make install") == 0
+
+    @synchronized
+    def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True):
+        assert id == self.client_id
+        self.shell.cd(self.gluster_root)
+        env = "DEBUG=1 "
+        if valgrind == "memcheck" or valgrind == "yes":
+            cmd = "valgrind"
+            cmd += " --tool=memcheck --leak-check=full --track-origins=yes"
+            cmd += " --show-leak-kinds=all -v prove -v"
+        elif valgrind == "drd":
+            cmd = "valgrind"
+            cmd += " --tool=drd -v prove -v"
+        elif asan_noleaks:
+            cmd = "prove -v"
+            env += "ASAN_OPTIONS=detect_leaks=0 "
+        else:
+            cmd = "prove -v"
+
+        status = self.shell.call(
+		"%s timeout %s %s %s" % (env, timeout, cmd, test))
+
+        if status != 0:
+            return (False, self._log_content())
+        return (True, "")
+
+    def _log_content(self):
+        return encode(self.shell.read_logs())
+
+# ..............................................................................
+# Cli role
+# ..............................................................................
+
+
+class RPCConnection((threading.Thread)):
+    def __init__(self, host, port, path, cb):
+        threading.Thread.__init__(self)
+        self.host = host
+        self.port = port
+        self.path = path
+        self.shell = Shell()
+        self.cb = cb
+        self.stop = False
+        self.proxy = None
+        self.logid = "%s:%s" % (self.host, self.port)
+
+    def connect(self):
+        (status, ret) = failsafe(self._connect)
+        return (status and ret)
+
+    def _connect(self):
+        url = "http://%s:%s" % (self.host, self.port)
+        self.proxy = xmlrpclib.ServerProxy(url, transport=IPTransport())
+        return self.proxy.hello(self.cb.id)
+
+    def disconnect(self):
+        self.stop = True
+
+    def ping(self):
+        return self.proxy.ping()
+
+    def init(self):
+        return self._copy() and self._compile_and_install()
+
+    def run(self):
+        (status, ret) = failsafe(self.init)
+        if not status:
+            self.cb.note_lost_connection(self)
+            return
+        elif not ret:
+            self.cb.note_setup_failed(self)
+            return
+
+        while not self.stop:
+            (status, ret) = failsafe(self._run)
+            if not status or not ret:
+                self.cb.note_lost_connection(self)
+                break
+            time.sleep(0)
+
+        failsafe(self.proxy.bye, (self.cb.id,))
+        Log.debug("%s connection thread stopped" % self.host)
+
+    def _run(self):
+        test = self.cb.next_test()
+        (status, _) = failsafe(self._execute_next, (test,))
+        if not status:
+            self.cb.note_retry(test)
+            return False
+        return True
+
+    def _execute_next(self, test):
+        if not test:
+            time.sleep(1)
+            return
+
+        (status, error) = self.proxy.prove(self.cb.id, test,
+                                           self.cb.test_timeout,
+                                           self.cb.valgrind,
+                                           self.cb.asan_noleaks)
+        if status:
+            self.cb.note_done(test)
+        else:
+            self.cb.note_error(test, error)
+
+    def _compile_and_install(self):
+        Log.debug("<%s> Build " % self.logid)
+        asan = self.cb.asan or self.cb.asan_noleaks
+        return (self.proxy.build(self.cb.id, asan) and
+                self.proxy.install(self.cb.id))
+
+    def _copy(self):
+        return self._copy_gzip()
+
+    def _copy_gzip(self):
+        Log.cli("<%s> copying and compiling %s to remote" %
+                 (self.logid, self.path))
+        data = encode(get_file_content(patch_file()))
+        Log.debug("GZIP size = %s B" % len(data))
+        return self.proxy.copygzip(self.cb.id, data)
+
+
+class RPCConnectionPool:
+    def __init__(self, gluster_path, hosts, n, id_rsa):
+        self.gluster_path = gluster_path
+        self.hosts = hosts
+        self.conns = []
+        self.faulty = []
+        self.n = int(len(hosts) / 2) + 1 if not n else n
+        self.id_rsa = id_rsa
+        self.stop = False
+        self.scanner = threading.Thread(target=self._scan_hosts_loop)
+        self.kicker = threading.Thread(target=self._kick_hosts_loop)
+        self.shell = Shell()
+        self.since_start = Timer()
+
+        self.shell.check_call("rm -f %s" % patch_file())
+        self.shell.check_call("tar -zcvf %s ." % patch_file())
+        self.id = md5.new(get_file_content(patch_file())).hexdigest()
+        Log.cli("client UID %s" % self.id)
+        Log.cli("patch UID %s" % PATCH_FILE_UID)
+
+    def __del__(self):
+        self.shell.check_call("rm -f %s" % patch_file())
+
+    def pool_status(self):
+        elapsed_m = int(self.since_start.elapsed_s() / 60)
+        return "%s/%s connected, %smin elapsed" % (len(self.conns), self.n,
+                                                   elapsed_m)
+
+    def connect(self):
+        Log.debug("Starting scanner")
+        self.scanner.start()
+        self.kicker.start()
+
+    def disconnect(self):
+        self.stop = True
+        for conn in self.conns:
+            conn.disconnect()
+
+    def note_lost_connection(self, conn):
+        Log.cli("lost connection to %s" % conn.host)
+        self.conns.remove(conn)
+        self.hosts.append((conn.host, conn.port))
+
+    def note_setup_failed(self, conn):
+        Log.error("Setup failed on %s:%s" % (conn.host, conn.port))
+        self.conns.remove(conn)
+        self.faulty.append((conn.host, conn.port))
+
+    def _scan_hosts_loop(self):
+        Log.debug("Scanner thread started")
+        while not self.stop:
+            failsafe(self._scan_hosts)
+            time.sleep(5)
+
+    def _scan_hosts(self):
+        if len(self.hosts) == 0 and len(self.conns) == 0:
+            Log.error("no more hosts available to loadbalance")
+            sys.exit(1)
+
+        for (host, port) in self.hosts:
+            if (len(self.conns) >= self.n) or self.stop:
+                break
+            self._scan_host(host, port)
+
+    def _scan_host(self, host, port):
+        Log.debug("scanning %s:%s" % (host, port))
+        c = RPCConnection(host, port, self.gluster_path, self)
+        (status, result) = failsafe(c.connect)
+        if status and result:
+            self.hosts.remove((host, port))
+            Log.debug("Connected to %s:%s" % (host, port))
+            self.conns.append(c)
+            c.start()
+            Log.debug("%s / %s connected" % (len(self.conns), self.n))
+        else:
+            Log.debug("Failed to connect to %s:%s" % (host, port))
+
+    def _kick_hosts_loop(self):
+        Log.debug("Kick thread started")
+        while not self.stop:
+            time.sleep(10)
+            failsafe(self._kick_hosts)
+
+        Log.debug("Kick thread stopped")
+
+    def _is_pingable(self, host, port):
+        c = RPCConnection(host, port, self.gluster_path, self)
+        failsafe(c.connect)
+        (status, result) = failsafe(c.ping)
+        return status and result
+
+    def _kick_hosts(self):
+        # Do not kick hosts if we have the optimal number of connections
+        if (len(self.conns) >= self.n) or self.stop:
+            Log.debug("Skip kicking hosts")
+            return
+
+        # Check and if dead kick all hosts
+        for (host, port) in self.hosts:
+            if self.stop:
+                Log.debug("Break kicking hosts")
+                break
+
+            if self._is_pingable(host, port):
+                Log.debug("Host=%s is alive. Won't kick" % host)
+                continue
+
+            Log.debug("Kicking %s" % host)
+            mypath = sys.argv[0]
+            myname = os.path.basename(mypath)
+            destpath = "/tmp/%s" % myname
+            sh = Shell()
+            sh.scp(host, mypath, destpath, self.id_rsa)
+            sh.ssh(host, "nohup %s --server &>> %s.log &" %
+                         (destpath, destpath), self.id_rsa)
+
+    def join(self):
+        self.scanner.join()
+        self.kicker.join()
+        for c in self.conns:
+            c.join()
+
+
+# ..............................................................................
+# test role
+# ..............................................................................
+
+class TestRunner(RPCConnectionPool):
+    def __init__(self, gluster_path, hosts, n, tests, flaky_tests, valgrind,
+                 asan, asan_noleaks, id_rsa, test_timeout):
+        RPCConnectionPool.__init__(self, gluster_path, self._parse_hosts(hosts),
+                                   n, id_rsa)
+        self.flaky_tests = flaky_tests.split(" ")
+        self.pending = []
+        self.done = []
+        self.error = []
+        self.retry = {}
+        self.error_logs = []
+        self.stats_timer = Timer()
+        self.valgrind = valgrind
+        self.asan = asan
+        self.asan_noleaks = asan_noleaks
+        self.test_timeout = test_timeout
+
+        self.tests = self._get_tests(tests)
+
+        Log.debug("tests: %s" % self.tests)
+
+    def _get_tests(self, tests):
+        if not tests or tests == "all":
+            return self._not_flaky(self._all())
+        elif tests == "flaky":
+            return self.flaky_tests
+        else:
+            return self._not_flaky(tests.strip().split(" "))
+
+    def run(self):
+        self.connect()
+        self.join()
+        return len(self.error)
+
+    def _pretty_print(self, data):
+        if isinstance(data, list):
+            str = ""
+            for i in data:
+                str = "%s %s" % (str, i)
+            return str
+        return "%s" % data
+
+    def print_result(self):
+        Log.cli("== RESULTS ==")
+        Log.cli("SUCCESS  : %s" % len(self.done))
+        Log.cli("ERRORS   : %s" % len(self.error))
+        Log.cli("== ERRORS ==")
+        Log.cli(self._pretty_print(self.error))
+        Log.cli("== LOGS ==")
+        Log.cli(self._pretty_print(self.error_logs))
+        Log.cli("== END ==")
+
+    def next_test(self):
+        if len(self.tests):
+            test = self.tests.pop()
+            self.pending.append(test)
+            return test
+
+        if not len(self.pending):
+            self.disconnect()
+
+        return None
+
+    def _pct_completed(self):
+        total = len(self.tests) + len(self.pending) + len(self.done)
+        total += len(self.error)
+        completed = len(self.done) + len(self.error)
+        return 0 if not total else int(completed / total * 100)
+
+    def note_done(self, test):
+        Log.cli("%s PASS (%s%% done) (%s)" % (test, self._pct_completed(),
+                                              self.pool_status()))
+        self.pending.remove(test)
+        self.done.append(test)
+        if test in self.retry:
+            del self.retry[test]
+
+    def note_error(self, test, errstr):
+        Log.cli("%s FAIL" % test)
+        self.pending.remove(test)
+        if test not in self.retry:
+            self.retry[test] = 1
+
+        if errstr:
+            path = "%s/%s-%s.log" % ("/tmp", test.replace("/", "-"),
+                                     self.retry[test])
+            failsafe(write_to_file, (path, decode(errstr),))
+            self.error_logs.append(path)
+
+        if self.retry[test] < MAX_ATTEMPTS:
+            self.retry[test] += 1
+            Log.debug("retry test %s attempt %s" % (test, self.retry[test]))
+            self.tests.append(test)
+        else:
+            Log.debug("giveup attempt test %s" % test)
+            del self.retry[test]
+            self.error.append(test)
+
+    def note_retry(self, test):
+        Log.cli("retry %s on another host" % test)
+        self.pending.remove(test)
+        self.tests.append(test)
+
+    #
+    # test classifications
+    #
+    def _all(self):
+        return self._list_tests(["tests"], recursive=True)
+
+    def _not_flaky(self, tests):
+        for t in self.flaky_tests:
+            if t in tests:
+                tests.remove(t)
+        return tests
+
+    def _list_tests(self, prefixes, recursive=False, ignore_ifnotexist=False):
+        tests = []
+        for prefix in prefixes:
+            real_path = "%s/%s" % (self.gluster_path, prefix)
+            if not os.path.exists(real_path) and ignore_ifnotexist:
+                continue
+            for f in os.listdir(real_path):
+                if os.path.isdir(real_path + "/" + f):
+                    if recursive:
+                        tests += self._list_tests([prefix + "/" + f], recursive)
+                else:
+                    if re.match(r".*\.t$", f):
+                        tests += [prefix + "/" + f]
+        return tests
+
+    def _parse_hosts(self, hosts):
+        ret = []
+        for h in args.hosts.split(" "):
+            ret.append((h, DEFAULT_PORT))
+        Log.debug(ret)
+        return ret
+
+# ..............................................................................
+# Roles entry point
+# ..............................................................................
+
+
+def run_as_server(args):
+    if not args.server_path:
+        Log.error("please provide server path")
+        return 1
+
+    server = TestServer(args.port, args.server_path)
+    server.init()
+    server.serve()
+    return 0
+
+
+def run_as_tester(args):
+    Log.header("GLUSTER TEST CLI")
+
+    Log.debug("args = %s" % args)
+
+    tests = TestRunner(args.gluster_path, args.hosts, args.n, args.tests,
+                       args.flaky_tests, valgrind=args.valgrind,
+                       asan=args.asan, asan_noleaks=args.asan_noleaks,
+                       id_rsa=args.id_rsa, test_timeout=args.test_timeout)
+    result = tests.run()
+    tests.print_result()
+    return result
+
+# ..............................................................................
+# main
+# ..............................................................................
+
+
+def main(args):
+    if args.v:
+        Log.LOGLEVEL = LogLevel.DEBUG
+
+    if args.server and args.tester:
+        Log.error("Invalid arguments. More than one role specified")
+        sys.exit(1)
+
+    if args.server:
+        sys.exit(run_as_server(args))
+    elif args.tester:
+        sys.exit(run_as_tester(args))
+    else:
+        Log.error("please specify a mode for CI")
+        parser.print_help()
+        sys.exit(1)
+
+
+parser = argparse.ArgumentParser(description="Gluster CI")
+
+# server role
+parser.add_argument("--server", help="start server", action="store_true")
+parser.add_argument("--server_path", help="server scratch space",
+                    default="/tmp/gluster-test")
+parser.add_argument("--host", help="server address to listen", default="")
+parser.add_argument("--port", help="server port to listen",
+                    type=int, default=DEFAULT_PORT)
+# test role
+parser.add_argument("--tester", help="start tester", action="store_true")
+parser.add_argument("--valgrind[=memcheck,drd]",
+                    help="run tests with valgrind tool 'memcheck' or 'drd'",
+                    default="no")
+parser.add_argument("--asan", help="test with asan enabled",
+                    action="store_true")
+parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks",
+                    action="store_true")
+parser.add_argument("--tests", help="all/flaky/list of tests", default=None)
+parser.add_argument("--flaky_tests", help="list of flaky tests", default=None)
+parser.add_argument("--n", help="max number of machines to use", type=int,
+                    default=0)
+parser.add_argument("--hosts", help="list of worker machines")
+parser.add_argument("--gluster_path", help="gluster path to test",
+                    default=os.getcwd())
+parser.add_argument("--id-rsa", help="private key to use for ssh",
+                    default=None)
+parser.add_argument("--test-timeout",
+                    help="test timeout in sec (default 15min)",
+                    default=TEST_TIMEOUT_S)
+# general
+parser.add_argument("-v", help="verbose", action="store_true")
+parser.add_argument("--address_family", help="IPv6 or IPv4 to use",
+                    default=ADDRESS_FAMILY)
+
+args = parser.parse_args()
+
+main(args)
diff --git a/extras/distributed-testing/distributed-test.sh b/extras/distributed-testing/distributed-test.sh
new file mode 100755
index 00000000000..8f1e0310f33
--- /dev/null
+++ b/extras/distributed-testing/distributed-test.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+source ./extras/distributed-testing/distributed-test-env
+
+N=0
+TESTS='all'
+FLAKY=$KNOWN_FLAKY_TESTS
+BROKEN=$BROKEN_TESTS
+TEST_TIMEOUT_S=900
+ADDRESS_FAMILY='IPv4'
+
+FLAGS=""
+
+function print_env {
+    echo "Settings:"
+    echo "N=$N"
+    echo -e "-------\nHOSTS\n$HOSTS\n-------"
+    echo -e "TESTS\n$TESTS\n-------"
+    echo -e "SKIP\n$FLAKY $BROKEN\n-------"
+    echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n"
+}
+
+function cleanup {
+    rm -f /tmp/test*.log
+}
+
+function usage {
+    echo "Usage: $0 [-h or --help] [-v or --verbose]
+             [--all] [--flaky] [--smoke] [--broken]
+             [--valgrind] [--asan] [--asan-noleaks]
+             [--hosts <hosts>] [-n <parallelism>]
+             [--tests <tests>]
+             [--id-rsa <ssh private key>]
+             [--address_family <IPv4 or IPv6>]
+    "
+}
+
+function parse_args () {
+    args=`getopt \
+            -o hvn: \
+            --long help,verbose,address_family:,valgrind,asan,asan-noleaks,all,\
+smoke,flaky,broken,hosts:,tests:,id-rsa:,test-timeout: \
+            -n 'fb-remote-test.sh' --  "$@"`
+
+    if [ $? != 0 ]; then
+        echo "Error parsing getopt"
+        exit 1
+    fi
+
+    eval set -- "$args"
+
+    while true; do
+        case "$1" in
+            -h | --help) usage ; exit 1 ;;
+            -v | --verbose) FLAGS="$FLAGS -v" ; shift ;;
+            --address_family) ADDRESS_FAMILY=$2; shift 2 ;;
+            --valgrind) FLAGS="$FLAGS --valgrind" ; shift ;;
+            --asan-noleaks) FLAGS="$FLAGS --asan-noleaks"; shift ;;
+            --asan) FLAGS="$FLAGS --asan" ; shift ;;
+            --hosts) HOSTS=$2; shift 2 ;;
+            --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;;
+            --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;;
+            --all) TESTS='all' ; shift 1 ;;
+            --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;;
+            --smoke) TESTS=$SMOKE_TESTS; shift 1 ;;
+            --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;;
+            --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;;
+            -n) N=$2; shift 2 ;;
+            *) break ;;
+            esac
+        done
+        run_tests_args="$@"
+}
+
+function main {
+    parse_args "$@"
+
+    if [ -z "$HOSTS" ]; then
+        echo "Please provide hosts to run the tests in"
+	exit -1
+    fi
+
+    print_env
+
+    cleanup
+
+    "extras/distributed-testing/distributed-test-runner.py" $FLAGS --tester \
+        --n "$N" --hosts "$HOSTS" --tests "$TESTS" \
+        --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S" \
+        --address_family "$ADDRESS_FAMILY"
+
+    exit $?
+}
+
+main "$@"
diff --git a/extras/ec-heal-script/README.md b/extras/ec-heal-script/README.md
new file mode 100644
index 00000000000..aaefd6681f6
--- /dev/null
+++ b/extras/ec-heal-script/README.md
@@ -0,0 +1,69 @@
+# gluster-heal-scripts
+Scripts to correct extended attributes of fragments of files to make them healble.
+
+Following are the guidelines/suggestions to use these scripts.
+
+1 - Passwordless ssh should be setup for all the nodes of the cluster.
+
+2 - Scripts should be executed from one of these nodes.
+
+3 - Make sure NO "IO" is going on for the files for which we are running
+these two scripts.
+
+4 - There should be no heal going on for the file for which xattrs are being
+set by correct_pending_heals.sh. Disable the self heal while running this script.
+
+5 - All the bricks of the volume should be UP to identify good and bad fragments
+and to decide if an entry is healble or not.
+
+6 - If correct_pending_heals.sh is stopped in the middle while it was processing
+healble entries, it is suggested to re-run gfid_needing_heal_parallel.sh to create
+latest list of healble and non healble entries and "potential_heal" "can_not_heal" files.
+
+7 - Based on the number of entries, these files might take time to get and set the
+stats and xattrs of entries.
+
+8 - A backup of the fragments will be taken on <brick path>/.glusterfs/correct_pending_heals
+    directory with a file name same as gfid.
+
+9 - Once the correctness of the file gets verified by user, these backup should be removed.
+
+10 - Make sure we have enough space on bricks to take these backups.
+
+11 - At the end this will create two files -
+     1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed.
+     2 - can_not_heal - Contains list of files which can not be healed.
+
+12 - It is suggested that the integrity of the data of files, which were modified and healed,
+     should be checked by the user.
+
+
+Usage:
+
+Following are the sequence of steps to use these scripts -
+
+1 - ./gfid_needing_heal_parallel.sh <volume name>
+
+    Execute gfid_needing_heal_parallel.sh with volume name to create list of files which could
+    be healed and can not be healed. It creates "potential_heal" and "can_not_heal" files.
+    During execution, it also displays the list of files on consol with the verdict.
+
+2 - ./correct_pending_heals.sh
+
+    Execute correct_pending_heals.sh without any argument. This script processes entries present
+    in "heal" file. It asks user to enter how many files we want to process in one attempt.
+    Once the count is provided, this script will fetch the entries one by one from "potential_heal" file and takes necessary action.
+    If at this point also a file can not be healed, it will be pushed to "can_not_heal" file.
+    If a file can be healed, this script will modify the xattrs of that file fragments and create an entry in "modified_and_backedup_files" file
+
+3 - At the end, all the entries of "potential_heal" will be processed and based on the processing only two files will be left.
+
+     1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed.
+     2 - can_not_heal - Contains list of files which can not be healed.
+
+Logs and other files -
+
+1 - modified_and_backedup_files  - It contains all the files which could be healed and the location of backup of each fragments.
+2 - can_not_heal - It contains all the files which can not be healed.
+3 - potential_heal - List of files which could be healed and should be processed by "correct_pending_heals.sh"
+4 - /var/log/glusterfs/ec-heal-script.log - It contains logs of both the files.
diff --git a/extras/ec-heal-script/correct_pending_heals.sh b/extras/ec-heal-script/correct_pending_heals.sh
new file mode 100755
index 00000000000..c9f19dd7c89
--- /dev/null
+++ b/extras/ec-heal-script/correct_pending_heals.sh
@@ -0,0 +1,415 @@
+#!/bin/bash
+#  Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+# This script finally resets the xattrs of all the fragments of a file
+# which can be healed as per gfid_needing_heal_parallel.sh.
+# gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal.
+# This script takes potential_heal as input and resets xattrs of all the fragments
+# of those files present in this file and which could be healed as per
+# trusted.ec.size xattar of the file else it will place the entry in can_not_heal
+# file.  Those entries which must be healed will be place in must_heal file
+# after setting xattrs so that user can track those files.
+
+
+MOD_BACKUP_FILES="modified_and_backedup_files"
+CAN_NOT_HEAL="can_not_heal"
+LOG_DIR="/var/log/glusterfs"
+LOG_FILE="$LOG_DIR/ec-heal-script.log"
+LINE_SEP="==================================================="
+
+function heal_log()
+{
+    echo "$1" >> "$LOG_FILE"
+}
+
+function desc ()
+{
+    echo ""
+    echo "This script finally resets the xattrs of all the fragments of a file
+which can be healed as per gfid_needing_heal_parallel.sh.
+gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal.
+This script takes potential_heal as input and resets xattrs of all the fragments
+of those files present in this file and which could be healed as per
+trusted.ec.size xattar of the file else it will place the entry in can_not_heal
+file.  Those entries which must be healed will be place in must_heal file
+after setting xattrs so that user can track those files."
+}
+
+function _init ()
+{
+    if [ $# -ne 0 ]
+    then
+        echo "usage: $0"
+        desc
+        exit 2
+    fi
+
+    if [ ! -f "potential_heal" ]
+    then
+        echo "Nothing to correct. File "potential_heal" does not exist"
+        echo ""
+        desc
+        exit 2
+    fi
+}
+
+function total_file_size_in_hex()
+{
+    local frag_size=$1
+    local size=0
+    local hex_size=""
+
+    size=$((frag_size * 4))
+    hex_size=$(printf '0x%016x' $size)
+    echo "$hex_size"
+}
+
+function backup_file_fragment()
+{
+    local file_host=$1
+    local file_entry=$2
+    local gfid_actual_paths=$3
+    local brick_root=""
+    local temp=""
+    local backup_dir=""
+    local cmd=""
+    local gfid=""
+
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+    temp=$(echo "$(basename "$BASH_SOURCE")" | cut -d '.' -f 1)
+    backup_dir=$(echo "${brick_root}/.glusterfs/${temp}")
+    file_entry=${file_entry//#}
+
+    gfid=$(echo "${gfid_actual_paths}" | cut -d '|' -f 1 | cut -d '/' -f 5)
+    echo "${file_host}:${backup_dir}/${gfid}" >> "$MOD_BACKUP_FILES"
+
+    cmd="mkdir -p ${backup_dir} && yes | cp -af ${file_entry} ${backup_dir}/${gfid} 2>/dev/null"
+    ssh -n "${file_host}" "${cmd}"
+}
+
+function set_frag_xattr ()
+{
+    local file_host=$1
+    local file_entry=$2
+    local good=$3
+    local cmd1=""
+    local cmd2=""
+    local cmd=""
+    local version="0x00000000000000010000000000000001"
+    local dirty="0x00000000000000010000000000000001"
+
+    if [[ $good -eq 0 ]]
+    then
+            version="0x00000000000000000000000000000000"
+    fi
+
+    cmd1=" setfattr -n trusted.ec.version -v ${version} ${file_entry} &&"
+    cmd2=" setfattr -n trusted.ec.dirty -v ${dirty} ${file_entry}"
+    cmd=${cmd1}${cmd2}
+    ssh -n "${file_host}" "${cmd}"
+}
+
+function set_version_dirty_xattr ()
+{
+    local file_paths=$1
+    local good=$2
+    local gfid_actual_paths=$3
+    local file_entry=""
+    local file_host=""
+    local bpath=""
+
+    for bpath in ${file_paths//,/ }
+    do
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        backup_file_fragment "$file_host" "$file_entry" "$gfid_actual_paths"
+        file_entry=${file_entry//#}
+        set_frag_xattr "$file_host" "$file_entry" "$good"
+    done
+}
+
+function match_size_xattr_quorum ()
+{
+    local file_paths=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local size_xattr=""
+    local bpath=""
+    declare -A xattr_count
+
+    for bpath in ${file_paths//,/ }
+    do
+        size_xattr=""
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        file_entry=${file_entry//#}
+
+        cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2"
+        size_xattr=$(ssh -n "${file_host}" "${cmd}")
+        if [[ -n $size_xattr ]]
+        then
+            count=$((xattr_count["$size_xattr"] + 1))
+            xattr_count["$size_xattr"]=${count}
+            if [[ $count -ge 4 ]]
+            then
+                echo "${size_xattr}"
+                return
+            fi
+        fi
+    done
+    echo "False"
+}
+
+function match_version_xattr ()
+{
+    local file_paths=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local version=""
+    local bpath=""
+    declare -A ver_count
+
+    for bpath in ${file_paths//,/ }
+    do
+        version=""
+        file_host=$(echo "$bpath" | cut -d ":" -f 1)
+        file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+        file_entry=${file_entry//#}
+
+        cmd="getfattr -n trusted.ec.version -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.version" | cut -d '=' -f 2"
+        version=$(ssh -n "${file_host}" "${cmd}")
+        ver_count["$version"]=$((ver_count["$version"] + 1))
+    done
+    for key in "${ver_count[@]}"
+    do
+        if [[ $key -ge 4 ]]
+        then
+            echo "True"
+            return
+        else
+            echo "False"
+            return
+        fi
+    done
+}
+
+function match_stat_size_with_xattr ()
+{
+    local bpath=$1
+    local size=$2
+    local file_stat=$3
+    local xattr=$4
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local stat_output=""
+    local hex_size=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+
+    file_entry=${file_entry//#}
+    cmd="stat --format=%F:%B:%s $file_entry 2>/dev/null"
+    stat_output=$(ssh -n "${file_host}" "${cmd}")
+    echo "$stat_output" | grep -w "${file_stat}" > /dev/null
+
+    if [[ $? -eq 0 ]]
+    then
+        cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2"
+        hex_size=$(ssh -n "${file_host}" "${cmd}")
+
+        if [[ -z $hex_size || "$hex_size" != "$xattr" ]]
+        then
+            echo "False"
+            return
+        fi
+        size_diff=$(printf '%d' $(( size - hex_size )))
+        if [[ $size_diff -gt 2047 ]]
+        then
+            echo "False"
+            return
+        else
+            echo "True"
+            return
+        fi
+    else
+        echo "False"
+        return
+    fi
+}
+
+function find_file_paths ()
+{
+    local bpath=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local brick_root=""
+    local gfid=""
+    local actual_path=""
+    local gfid_path=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+
+    gfid=$(echo "${file_entry}" | grep ".glusterfs")
+    if [[ -n "$gfid" ]]
+    then
+        gfid_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' "
+        actual_path=$(ssh -n "${file_host}" "${cmd}")
+        #removing absolute path so that user can refer this from mount point
+        actual_path=${actual_path#"$brick_root"}
+    else
+        actual_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' "
+        gfid_path=$(ssh -n "${file_host}" "${cmd}")
+        gfid_path=${gfid_path#"$brick_root"}
+    fi
+
+    echo "${gfid_path}|${actual_path}"
+}
+
+function log_can_not_heal ()
+{
+    local gfid_actual_paths=$1
+    local file_paths=$2
+    file_paths=${file_paths//#}
+
+    echo "${LINE_SEP}" >> "$CAN_NOT_HEAL"
+    echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL"
+    for bpath in ${file_paths//,/ }
+    do
+        echo "${bpath}" >> "$CAN_NOT_HEAL"
+    done
+}
+
+function check_all_frag_and_set_xattr ()
+{
+    local file_paths=$1
+    local total_size=$2
+    local file_stat=$3
+    local bpath=""
+    local healthy_count=0
+    local match="False"
+    local matching_bricks=""
+    local bad_bricks=""
+    local gfid_actual_paths=""
+
+    for bpath in ${file_paths//,/ }
+    do
+        if [[ -n "$gfid_actual_paths" ]]
+        then
+            break
+        fi
+        gfid_actual_paths=$(find_file_paths "$bpath")
+    done
+
+   match=$(match_size_xattr_quorum "$file_paths")
+
+#   echo "${match} : $bpath" >> "$MOD_BACKUP_FILES"
+
+    if [[ "$match" != "False" ]]
+    then
+        xattr="$match"
+        for bpath in ${file_paths//,/ }
+        do
+            match="False"
+            match=$(match_stat_size_with_xattr "$bpath" "$total_size" "$file_stat" "$xattr")
+            if [[ "$match" == "True" ]]
+            then
+                matching_bricks="${bpath},${matching_bricks}"
+                healthy_count=$((healthy_count + 1))
+            else
+                bad_bricks="${bpath},${bad_bricks}"
+            fi
+        done
+    fi
+
+    if [[ $healthy_count -ge 4 ]]
+    then
+        match="True"
+        echo "${LINE_SEP}" >> "$MOD_BACKUP_FILES"
+        echo "Modified : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$MOD_BACKUP_FILES"
+        set_version_dirty_xattr  "$matching_bricks" 1 "$gfid_actual_paths"
+        set_version_dirty_xattr  "$bad_bricks" 0 "$gfid_actual_paths"
+    else
+        log_can_not_heal "$gfid_actual_paths" "${file_paths}"
+    fi
+
+    echo "$match"
+}
+function set_xattr()
+{
+    local count=$1
+    local heal_entry=""
+    local file_stat=""
+    local frag_size=""
+    local total_size=""
+    local file_paths=""
+    local num=""
+    local can_heal_count=0
+
+    heal_log "Started $(basename $BASH_SOURCE) on $(date) "
+
+    while read -r heal_entry
+    do
+        heal_log "$LINE_SEP"
+        heal_log "${heal_entry}"
+
+        file_stat=$(echo "$heal_entry" | cut -d "|" -f 1)
+        frag_size=$(echo "$file_stat" | rev | cut -d ":" -f 1 | rev)
+        total_size="$(total_file_size_in_hex "$frag_size")"
+        file_paths=$(echo "$heal_entry" | cut -d "|" -f 2)
+        match=$(check_all_frag_and_set_xattr "$file_paths" "$total_size" "$file_stat")
+        if [[ "$match" == "True" ]]
+        then
+            can_heal_count=$((can_heal_count + 1))
+        fi
+
+        sed -i '1d' potential_heal
+        count=$((count - 1))
+        if [ $count == 0 ]
+        then
+            num=$(cat potential_heal | wc -l)
+            heal_log "$LINE_SEP"
+            heal_log "${1} : Processed"
+            heal_log "${can_heal_count} : Modified to Heal"
+            heal_log "$((${1} - can_heal_count)) : Moved to can_not_heal."
+            heal_log "${num} : Pending as Potential Heal"
+            exit 0
+        fi
+
+    done < potential_heal
+}
+
+function main ()
+{
+    local count=0
+
+    read -p "Number of files to correct: [choose between 1-1000] (0 for All):" count
+    if [[ $count -lt 0 || $count -gt 1000 ]]
+    then
+        echo "Provide correct value:"
+        exit 2
+    fi
+
+    if [[ $count -eq 0 ]]
+    then
+        count=$(cat potential_heal | wc -l)
+    fi
+    set_xattr "$count"
+}
+
+_init "$@" && main "$@"
diff --git a/extras/ec-heal-script/gfid_needing_heal_parallel.sh b/extras/ec-heal-script/gfid_needing_heal_parallel.sh
new file mode 100755
index 00000000000..d7f53c97c33
--- /dev/null
+++ b/extras/ec-heal-script/gfid_needing_heal_parallel.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+#  Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+# This script provides a list of all the files which can be healed or not healed.
+# It also generates two files, potential_heal and can_not_heal, which contains the information
+# of all theose files. These files could be used by correct_pending_heals.sh to correct
+# the fragmnets so that files could be healed by shd.
+
+CAN_NOT_HEAL="can_not_heal"
+CAN_HEAL="potential_heal"
+LINE_SEP="==================================================="
+LOG_DIR="/var/log/glusterfs"
+LOG_FILE="$LOG_DIR/ec-heal-script.log"
+
+function heal_log()
+{
+    echo "$1" >> "$LOG_FILE"
+}
+
+function _init ()
+{
+    if [ $# -ne 1 ]; then
+    echo "usage: $0 <gluster volume name>";
+    echo "This script provides a list of all the files which can be healed or not healed.
+It also generates two files, potential_heal and can_not_heal, which contains the information
+of all theose files. These files could be used by correct_pending_heals.sh to correct
+the fragmnets so that files could be healed by shd."
+    exit 2;
+    fi
+
+    volume=$1;
+}
+
+function get_pending_entries ()
+{
+    local volume_name=$1
+
+    gluster volume heal "$volume_name" info | grep -v ":/" | grep -v "Number of entries" | grep -v "Status:" | sort -u | sed '/^$/d'
+}
+
+function get_entry_path_on_brick()
+{
+    local path="$1"
+    local gfid_string=""
+    if [[ "${path:0:1}" == "/" ]];
+    then
+        echo "$path"
+    else
+        gfid_string="$(echo "$path" | cut -f2 -d':' | cut -f1 -d '>')"
+        echo "/.glusterfs/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string"
+    fi
+}
+
+function run_command_on_server()
+{
+	local subvolume="$1"
+	local host="$2"
+	local cmd="$3"
+	local output
+        output=$(ssh -n "${host}" "${cmd}")
+	if [ -n "$output" ]
+	then
+            echo "$subvolume:$output"
+	fi
+}
+
+function get_entry_path_all_bricks ()
+{
+    local entry="$1"
+    local bricks="$2"
+    local cmd=""
+    for brick in $bricks
+    do
+	    echo "${brick}#$(get_entry_path_on_brick "$entry")"
+    done | tr '\n' ','
+}
+
+function get_stat_for_entry_from_all_bricks ()
+{
+    local entry="$1"
+    local bricks="$2"
+    local subvolume=0
+    local host=""
+    local bpath=""
+    local cmd=""
+
+    for brick in $bricks
+    do
+        if [[ "$((subvolume % 6))" == "0" ]]
+        then
+            subvolume=$((subvolume+1))
+        fi
+        host=$(echo "$brick" | cut -f1 -d':')
+        bpath=$(echo "$brick" | cut -f2 -d':')
+
+	cmd="stat --format=%F:%B:%s $bpath$(get_entry_path_on_brick "$entry") 2>/dev/null"
+    run_command_on_server "$subvolume" "${host}" "${cmd}" &
+    done | sort | uniq -c | sort -rnk1
+}
+
+function get_bricks_from_volume()
+{
+    local v=$1
+    gluster volume info "$v" | grep -E "^Brick[0-9][0-9]*:" | cut -f2- -d':'
+}
+
+function print_entry_gfid()
+{
+    local host="$1"
+    local dirpath="$2"
+    local entry="$3"
+    local gfid
+    gfid="$(ssh -n "${host}" "getfattr -d -m. -e hex $dirpath/$entry 2>/dev/null | grep trusted.gfid=|cut -f2 -d'='")"
+    echo "$entry" - "$gfid"
+}
+
+function print_brick_directory_info()
+{
+    local h="$1"
+    local dirpath="$2"
+    while read -r e
+    do
+        print_entry_gfid "${h}" "${dirpath}" "${e}"
+    done < <(ssh -n "${h}" "ls $dirpath 2>/dev/null")
+}
+
+function print_directory_info()
+{
+    local entry="$1"
+    local bricks="$2"
+    local h
+    local b
+    local gfid
+    for brick in $bricks;
+    do
+        h="$(echo "$brick" | cut -f1 -d':')"
+        b="$(echo "$brick" | cut -f2 -d':')"
+        dirpath="$b$(get_entry_path_on_brick "$entry")"
+	print_brick_directory_info "${h}" "${dirpath}" &
+    done | sort | uniq -c
+}
+
+function print_entries_needing_heal()
+{
+    local quorum=0
+    local entry="$1"
+    local bricks="$2"
+    while read -r line
+    do
+        quorum=$(echo "$line" | awk '{print $1}')
+        if [[ "$quorum" -lt 4 ]]
+        then
+            echo "$line - Not in Quorum"
+        else
+            echo "$line - In Quorum"
+        fi
+    done < <(print_directory_info "$entry" "$bricks")
+}
+
+function find_file_paths ()
+{
+    local bpath=$1
+    local file_entry=""
+    local file_host=""
+    local cmd=""
+    local brick_root=""
+    local gfid=""
+    local actual_path=""
+    local gfid_path=""
+
+    file_host=$(echo "$bpath" | cut -d ":" -f 1)
+    file_entry=$(echo "$bpath" | cut -d ":" -f 2)
+    brick_root=$(echo "$file_entry" | cut -d "#" -f 1)
+
+    gfid=$(echo "${file_entry}" | grep ".glusterfs")
+
+    if [[ -n "$gfid" ]]
+    then
+        gfid_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' "
+        actual_path=$(ssh -n "${file_host}" "${cmd}")
+        #removing absolute path so that user can refer this from mount point
+        actual_path=${actual_path#"$brick_root"}
+    else
+        actual_path=$(echo "$file_entry" | cut -d "#" -f 2)
+        file_entry=${file_entry//#}
+        cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' "
+        gfid_path=$(ssh -n "${file_host}" "${cmd}")
+        gfid_path=${gfid_path#"$brick_root"}
+    fi
+
+    echo "${gfid_path}|${actual_path}"
+}
+
+function log_can_not_heal ()
+{
+    local gfid_actual_paths=$1
+    local file_paths=$2
+    file_paths=${file_paths//#}
+
+    echo "${LINE_SEP}" >> "$CAN_NOT_HEAL"
+    echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL"
+    for bpath in ${file_paths//,/ }
+    do
+        echo "${bpath}" >> "$CAN_NOT_HEAL"
+    done
+}
+
+function main ()
+{
+    local bricks=""
+    local quorum=0
+    local stat_info=""
+    local file_type=""
+    local gfid_actual_paths=""
+    local bpath=""
+    local file_paths=""
+    local good=0
+    local bad=0
+    bricks=$(get_bricks_from_volume "$volume")
+    rm -f "$CAN_HEAL"
+    rm -f "$CAN_NOT_HEAL"
+    mkdir "$LOG_DIR" -p
+
+    heal_log "Started $(basename "$BASH_SOURCE") on $(date) "
+    while read -r heal_entry
+    do
+        heal_log "------------------------------------------------------------------"
+        heal_log "$heal_entry"
+
+        gfid_actual_paths=""
+        file_paths="$(get_entry_path_all_bricks "$heal_entry" "$bricks")"
+        stat_info="$(get_stat_for_entry_from_all_bricks "$heal_entry" "$bricks")"
+        heal_log "$stat_info"
+
+        quorum=$(echo "$stat_info" | head -1 | awk '{print $1}')
+        good_stat=$(echo "$stat_info" | head -1 | awk '{print $3}')
+        file_type="$(echo "$stat_info" | head -1 | cut -f2 -d':')"
+        if [[ "$file_type" == "directory" ]]
+        then
+            print_entries_needing_heal "$heal_entry" "$bricks"
+        else
+            if [[ "$quorum" -ge 4 ]]
+            then
+                good=$((good + 1))
+                heal_log "Verdict: Healable"
+
+                echo "${good_stat}|$file_paths" >> "$CAN_HEAL"
+            else
+                bad=$((bad + 1))
+                heal_log "Verdict: Not Healable"
+                for bpath in ${file_paths//,/ }
+                do
+                    if [[ -z "$gfid_actual_paths" ]]
+                    then
+                        gfid_actual_paths=$(find_file_paths "$bpath")
+                    else
+                        break
+                    fi
+                done
+                log_can_not_heal "$gfid_actual_paths" "${file_paths}"
+            fi
+        fi
+    done < <(get_pending_entries "$volume")
+    heal_log "========================================="
+    heal_log "Total number of  potential heal : ${good}"
+    heal_log "Total number of can not heal    : ${bad}"
+    heal_log "========================================="
+}
+
+_init "$@" && main "$@"
diff --git a/extras/failed-tests.py b/extras/failed-tests.py
new file mode 100755
index 00000000000..f7f110246b5
--- /dev/null
+++ b/extras/failed-tests.py
@@ -0,0 +1,180 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import blessings
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import re
+import argparse
+from collections import defaultdict
+from datetime import timedelta, datetime
+from pystache import render
+
+# This tool goes though the Gluster regression links and checks for failures
+
+BASE = 'https://build.gluster.org'
+TERM = blessings.Terminal()
+MAX_BUILDS = 1000
+summary = defaultdict(list)
+VERBOSE = None
+
+
+def process_failure(url, node):
+    text = requests.get(url, verify=False).text
+    accum = []
+    for t in text.split('\n'):
+        if t.find("Result: FAIL") != -1:
+            for t2 in accum:
+                if VERBOSE:
+                    print(t2.encode('utf-8'))
+                if t2.find("Wstat") != -1:
+                    test_case = re.search('\./tests/.*\.t', t2)
+                    if test_case:
+                        summary[test_case.group()].append((url, node))
+            accum = []
+        elif t.find("cur_cores=/") != -1:
+            summary["core"].append([t.split("/")[1]])
+            summary["core"].append(url)
+        else:
+            accum.append(t)
+
+
+def print_summary(failed_builds, total_builds, html=False):
+    # All the templates
+    count = [
+            '{{failed}} of {{total}} regressions failed',
+            '<p><b>{{failed}}</b> of <b>{{total}}</b> regressions failed</p>'
+    ]
+    regression_link = [
+            '\tRegression Link: {{link}}\n'
+            '\tNode: {{node}}',
+            '<p>&emsp;Regression Link: {{link}}</p>'
+            '<p>&emsp;Node: {{node}}</p>'
+    ]
+    component = [
+            '\tComponent: {{comp}}',
+            '<p>&emsp;Component: {{comp}}</p>'
+    ]
+    failure_count = [
+            ''.join([
+                TERM.red,
+                '{{test}} ; Failed {{count}} times',
+                TERM.normal
+            ]),
+            (
+                '<p><font color="red"><b>{{test}};</b> Failed <b>{{count}}'
+                '</b> times</font></p>'
+            )
+    ]
+
+    template = 0
+    if html:
+        template = 1
+    print(render(
+            count[template],
+            {'failed': failed_builds, 'total': total_builds}
+    ))
+    for k, v in summary.items():
+        if k == 'core':
+            print(''.join([TERM.red, "Found cores:", TERM.normal]))
+            for comp, link in zip(v[::2], v[1::2]):
+                print(render(component[template], {'comp': comp}))
+                print(render(
+                        regression_link[template],
+                        {'link': link[0], 'node': link[1]}
+                ))
+        else:
+            print(render(failure_count[template], {'test': k, 'count': len(v)}))
+            for link in v:
+                print(render(
+                        regression_link[template],
+                        {'link': link[0], 'node': link[1]}
+                ))
+
+
+def get_summary(cut_off_date, reg_link):
+    '''
+    Get links to the failed jobs
+    '''
+    success_count = 0
+    failure_count = 0
+    for page in range(0, MAX_BUILDS, 100):
+        build_info = requests.get(''.join([
+                BASE,
+                reg_link,
+                'api/json?depth=1&tree=allBuilds'
+                '[url,result,timestamp,builtOn]',
+                '{{{0},{1}}}'.format(page, page+100)
+        ]), verify=False).json()
+        for build in build_info.get('allBuilds'):
+            if datetime.fromtimestamp(build['timestamp']/1000) < cut_off_date:
+                # stop when timestamp older than cut off date
+                return failure_count, failure_count + success_count
+            if build['result'] in [None, 'SUCCESS']:
+                # pass when build is a success or ongoing
+                success_count += 1
+                continue
+            if VERBOSE:
+                print(''.join([
+                    TERM.red,
+                    'FAILURE on {0}'.format(build['url']),
+                    TERM.normal
+                ]))
+            url = ''.join([build['url'], 'consoleText'])
+            failure_count += 1
+            process_failure(url, build['builtOn'])
+    return failure_count, failure_count + success_count
+
+
+def main(num_days, regression_link, html_report):
+    cut_off_date = datetime.today() - timedelta(days=num_days)
+    failure = 0
+    total = 0
+    for reg in regression_link:
+        if reg == 'centos':
+            reg_link = '/job/centos6-regression/'
+        elif reg == 'netbsd':
+            reg_link = '/job/netbsd7-regression/'
+        else:
+            reg_link = reg
+        counts = get_summary(cut_off_date, reg_link)
+        failure += counts[0]
+        total += counts[1]
+    print_summary(failure, total, html_report)
+
+
+if __name__ == '__main__':
+    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("get-summary")
+    parser.add_argument(
+            "last_no_of_days",
+            default=1,
+            type=int,
+            help="Regression summary of last number of days"
+    )
+    parser.add_argument(
+            "regression_link",
+            default="centos",
+            nargs='+',
+            help="\"centos\" | \"netbsd\" | any other regression link"
+    )
+    parser.add_argument(
+            "--verbose",
+            default=False,
+            action="store_true",
+            help="Print a detailed report of each test case that is failed"
+    )
+    parser.add_argument(
+            "--html-report",
+            default=False,
+            action="store_true",
+            help="Print a brief report of failed regressions in html format"
+    )
+    args = parser.parse_args()
+    VERBOSE = args.verbose
+    main(
+        num_days=args.last_no_of_days,
+        regression_link=args.regression_link,
+        html_report=args.html_report
+    )
diff --git a/extras/file_size_contri.sh b/extras/file_size_contri.sh
new file mode 100755
index 00000000000..4f52a9a89b4
--- /dev/null
+++ b/extras/file_size_contri.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This script checks whether the contribution and disk-usage of a file is same.
+
+CONTRIBUTION_HEX=`getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri $1 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2`
+
+BLOCKS=`stat -c %b $1`
+SIZE=$(($BLOCKS * 512))
+
+CONTRIBUTION=`printf "%d" $CONTRIBUTION_HEX`
+
+if [ $CONTRIBUTION -ne $SIZE ]; then
+    printf "contribution of %s:%d\n" $1 $CONTRIBUTION
+    echo "size of $1: $SIZE"
+    echo "==================================================="
+fi
+ 
diff --git a/extras/firewalld/Makefile.am b/extras/firewalld/Makefile.am
new file mode 100644
index 00000000000..530881fb8eb
--- /dev/null
+++ b/extras/firewalld/Makefile.am
@@ -0,0 +1,8 @@
+EXTRA_DIST = glusterfs.xml
+
+if USE_FIREWALLD
+if WITH_SERVER
+staticdir = /usr/lib/firewalld/services/
+static_DATA = glusterfs.xml
+endif
+endif
diff --git a/extras/firewalld/glusterfs.xml b/extras/firewalld/glusterfs.xml
new file mode 100644
index 00000000000..7e176442f5b
--- /dev/null
+++ b/extras/firewalld/glusterfs.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<service>
+<short>glusterfs-static</short>
+<description>Default ports for gluster-distributed storage</description>
+<port protocol="tcp" port="24007"/>    <!--For glusterd -->
+<port protocol="tcp" port="24008"/>    <!--For glusterd RDMA port management -->
+<port protocol="tcp" port="24009"/>    <!--For glustereventsd -->
+<port protocol="tcp" port="38465"/>    <!--Gluster NFS service -->
+<port protocol="tcp" port="38466"/>    <!--Gluster NFS service -->
+<port protocol="tcp" port="38467"/>    <!--Gluster NFS service -->
+<port protocol="tcp" port="38468"/>    <!--Gluster NFS service -->
+<port protocol="tcp" port="38469"/>    <!--Gluster NFS service -->
+<port protocol="tcp" port="49152-49664"/>  <!--512 ports for bricks -->
+</service>
diff --git a/extras/ganesha/Makefile.am b/extras/ganesha/Makefile.am
new file mode 100644
index 00000000000..9eaa401b6c8
--- /dev/null
+++ b/extras/ganesha/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = scripts config ocf
+CLEANFILES =
diff --git a/extras/ganesha/config/Makefile.am b/extras/ganesha/config/Makefile.am
new file mode 100644
index 00000000000..c729273096e
--- /dev/null
+++ b/extras/ganesha/config/Makefile.am
@@ -0,0 +1,4 @@
+EXTRA_DIST= ganesha-ha.conf.sample
+
+confdir = $(sysconfdir)/ganesha
+conf_DATA = ganesha-ha.conf.sample
diff --git a/extras/ganesha/config/ganesha-ha.conf.sample b/extras/ganesha/config/ganesha-ha.conf.sample
new file mode 100644
index 00000000000..c22892bde56
--- /dev/null
+++ b/extras/ganesha/config/ganesha-ha.conf.sample
@@ -0,0 +1,19 @@
+# Name of the HA cluster created.
+# must be unique within the subnet
+HA_NAME="ganesha-ha-360"
+#
+# N.B. you may use short names or long names; you may not use IP addrs.
+# Once you select one, stay with it as it will be mildly unpleasant to
+# clean up if you switch later on. Ensure that all names - short and/or
+# long - are in DNS or /etc/hosts on all machines in the cluster.
+#
+# The subset of nodes of the Gluster Trusted Pool that form the ganesha
+# HA cluster. Hostname is specified.
+HA_CLUSTER_NODES="server1,server2,..."
+#HA_CLUSTER_NODES="server1.lab.redhat.com,server2.lab.redhat.com,..."
+#
+# Virtual IPs for each of the nodes specified above.
+VIP_server1="10.0.2.1"
+VIP_server2="10.0.2.2"
+#VIP_server1_lab_redhat_com="10.0.2.1"
+#VIP_server2_lab_redhat_com="10.0.2.2"
diff --git a/extras/ganesha/ocf/Makefile.am b/extras/ganesha/ocf/Makefile.am
new file mode 100644
index 00000000000..990a609f254
--- /dev/null
+++ b/extras/ganesha/ocf/Makefile.am
@@ -0,0 +1,11 @@
+EXTRA_DIST= ganesha_grace ganesha_mon ganesha_nfsd
+
+# The root of the OCF resource agent hierarchy
+# Per the OCF standard, it's always "lib",
+# not "lib64" (even on 64-bit platforms).
+ocfdir = $(prefix)/lib/ocf
+
+# The provider directory
+radir = $(ocfdir)/resource.d/heartbeat
+
+ra_SCRIPTS = ganesha_grace ganesha_mon ganesha_nfsd
diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace
new file mode 100644
index 00000000000..825f7164597
--- /dev/null
+++ b/extras/ganesha/ocf/ganesha_grace
@@ -0,0 +1,221 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 Anand Subramanian anands@redhat.com
+# Copyright (c) 2015 Red Hat Inc.
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+#
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+if [ -n "$OCF_DEBUG_LIBRARY" ]; then
+	. $OCF_DEBUG_LIBRARY
+else
+	: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+	. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+fi
+
+OCF_RESKEY_grace_active_default="grace-active"
+: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}}
+
+ganesha_meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="ganesha_grace">
+<version>1.0</version>
+
+<longdesc lang="en">
+This Linux-specific resource agent acts as a dummy
+resource agent for nfs-ganesha.
+</longdesc>
+
+<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc>
+
+<parameters>
+<parameter name="grace_active">
+<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc>
+<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc>
+<content type="string" default="grace-active" />
+</parameter>
+</parameters>
+
+<actions>
+<action name="start"   timeout="40s" />
+<action name="stop"    timeout="40s" />
+<action name="status"  timeout="20s" interval="60s" />
+<action name="monitor" depth="0" timeout="10s" interval="5s" />
+<action name="notify"  timeout="10s" />
+<action name="meta-data"  timeout="20s" />
+</actions>
+</resource-agent>
+END
+
+return ${OCF_SUCCESS}
+}
+
+ganesha_grace_usage() {
+	echo "ganesha.nfsd USAGE"
+}
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+	meta-data)	ganesha_meta_data
+			exit ${OCF_SUCCESS}
+			;;
+	usage|help)	ganesha_usage
+			exit ${OCF_SUCCESS}
+			;;
+	*)
+			;;
+esac
+
+ganesha_grace_start()
+{
+	local rc=${OCF_ERR_GENERIC}
+	local host=$(hostname -s)
+
+	ocf_log debug "ganesha_grace_start()"
+	# give ganesha_mon RA a chance to set the crm_attr first
+	# I mislike the sleep, but it's not clear that looping
+	# with a small sleep is necessarily better
+	# start has a 40sec timeout, so a 5sec sleep here is okay
+        sleep 5
+	attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+        if [ $? -ne 0 ]; then
+		host=$(hostname)
+		attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null )
+                if [ $? -ne 0 ]; then
+	                ocf_log info "grace start: crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed"
+                fi
+        fi
+
+	# Three possibilities:
+	# 1. There is no attribute at all and attr_updater returns
+	#    a zero length string. This happens when
+	#    ganesha_mon::monitor hasn't run at least once to set
+	#    the attribute. The assumption here is that the system
+	#    is coming up. We pretend, for now, that the node is
+	#    healthy, to allow the system to continue coming up.
+	#    It will cure itself in a few seconds
+	# 2. There is an attribute, and it has the value "1"; this
+	#    node is healthy.
+	# 3. There is an attribute, but it has no value or the value
+	#    "0"; this node is not healthy.
+
+	# case 1
+	if [[ -z "${attr}" ]]; then
+		return ${OCF_SUCCESS}
+	fi
+
+	# case 2
+	if [[ "${attr}" = *"value=1" ]]; then
+		return ${OCF_SUCCESS}
+	fi
+
+	# case 3
+	return ${OCF_NOT_RUNNING}
+}
+
+ganesha_grace_stop()
+{
+
+	ocf_log debug "ganesha_grace_stop()"
+	return ${OCF_SUCCESS}
+}
+
+ganesha_grace_notify()
+{
+        # since this is a clone RA we should only ever see pre-start
+        # or post-stop
+	mode="${OCF_RESKEY_CRM_meta_notify_type}-${OCF_RESKEY_CRM_meta_notify_operation}"
+	case "${mode}" in
+	pre-start | post-stop)
+		dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname}
+		if [ $? -ne 0 ]; then
+			ocf_log info "dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} failed"
+		fi
+		;;
+	esac
+
+	return ${OCF_SUCCESS}
+}
+
+ganesha_grace_monitor()
+{
+	local host=$(hostname -s)
+
+	ocf_log debug "monitor"
+
+	attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+        if [ $? -ne 0 ]; then
+		host=$(hostname)
+	        attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+                if [ $? -ne 0 ]; then
+	                ocf_log info "crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed"
+                fi
+        fi
+
+	# if there is no attribute (yet), maybe it's because
+	# this RA started before ganesha_mon (nfs-mon) has had
+	# chance to create it. In which case we'll pretend
+	# everything is okay this time around
+	if [[ -z "${attr}" ]]; then
+		return ${OCF_SUCCESS}
+	fi
+
+	if [[ "${attr}" = *"value=1" ]]; then
+		return ${OCF_SUCCESS}
+	fi
+
+	return ${OCF_NOT_RUNNING}
+}
+
+ganesha_grace_validate()
+{
+	return ${OCF_SUCCESS}
+}
+
+ganesha_grace_validate
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          ganesha_grace_start
+		;;
+stop)           ganesha_grace_stop
+		;;
+status|monitor) ganesha_grace_monitor
+		;;
+notify)         ganesha_grace_notify
+		;;
+*)              ganesha_grace_usage
+		exit ${OCF_ERR_UNIMPLEMENTED}
+		;;
+esac
+
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
+exit $rc
diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon
new file mode 100644
index 00000000000..2b4a9d6da84
--- /dev/null
+++ b/extras/ganesha/ocf/ganesha_mon
@@ -0,0 +1,234 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 Anand Subramanian anands@redhat.com
+# Copyright (c) 2015 Red Hat Inc.
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+#
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+if [ -n "${OCF_DEBUG_LIBRARY}" ]; then
+	. ${OCF_DEBUG_LIBRARY}
+else
+	: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+	. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+fi
+
+# Defaults
+OCF_RESKEY_ganesha_active_default="ganesha-active"
+OCF_RESKEY_grace_active_default="grace-active"
+OCF_RESKEY_grace_delay_default="5"
+
+: ${OCF_RESKEY_ganesha_active=${OCF_RESKEY_ganesha_active_default}}
+: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}}
+: ${OCF_RESKEY_grace_delay=${OCF_RESKEY_grace_delay_default}}
+
+ganesha_meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="ganesha_mon">
+<version>1.0</version>
+
+<longdesc lang="en">
+This Linux-specific resource agent acts as a dummy
+resource agent for nfs-ganesha.
+</longdesc>
+
+<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc>
+
+<parameters>
+<parameter name="ganesha_active">
+<longdesc lang="en">NFS-Ganesha daemon active attribute</longdesc>
+<shortdesc lang="en">NFS-Ganesha daemon active attribute</shortdesc>
+<content type="string" default="ganesha-active" />
+</parameter>
+<parameter name="grace_active">
+<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc>
+<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc>
+<content type="string" default="grace-active" />
+</parameter>
+<parameter name="grace_delay">
+<longdesc lang="en">
+NFS-Ganesha grace delay.
+When changing this, adjust the ganesha_grace RA's monitor interval to match.
+</longdesc>
+<shortdesc lang="en">NFS-Ganesha grace delay</shortdesc>
+<content type="string" default="5" />
+</parameter>
+</parameters>
+
+<actions>
+<action name="start"   timeout="40s" />
+<action name="stop"    timeout="40s" />
+<action name="status"  timeout="20s" interval="60s" />
+<action name="monitor" depth="0"  timeout="10s" interval="10s" />
+<action name="meta-data"  timeout="20s" />
+</actions>
+</resource-agent>
+END
+
+return ${OCF_SUCCESS}
+}
+
+ganesha_mon_usage() {
+	echo "ganesha.nfsd USAGE"
+}
+
+# Make sure meta-data and usage always succeed
+case ${__OCF_ACTION} in
+	meta-data)	ganesha_meta_data
+			exit ${OCF_SUCCESS}
+			;;
+	usage|help)	ganesha_usage
+			exit ${OCF_SUCCESS}
+			;;
+	*)
+			;;
+esac
+
+ganesha_mon_start()
+{
+	ocf_log debug "ganesha_mon_start"
+	ganesha_mon_monitor
+	return $OCF_SUCCESS
+}
+
+ganesha_mon_stop()
+{
+	ocf_log debug "ganesha_mon_stop"
+	return $OCF_SUCCESS
+}
+
+ganesha_mon_monitor()
+{
+	local host=$(hostname -s)
+	local pid_file="/var/run/ganesha.pid"
+	local rhel6_pid_file="/var/run/ganesha.nfsd.pid"
+	local proc_pid="/proc/"
+
+	# RHEL6 /etc/init.d/nfs-ganesha adds -p /var/run/ganesha.nfsd.pid
+	# RHEL7 systemd does not. Would be nice if all distros used the
+	# same pid file.
+	if [ -e ${rhel6_pid_file} ]; then
+		pid_file=${rhel6_pid_file}
+	fi
+	if [ -e ${pid_file} ]; then
+		proc_pid="${proc_pid}$(cat ${pid_file})"
+	fi
+
+	if [ "x${proc_pid}" != "x/proc/" -a -d ${proc_pid} ]; then
+
+		attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1
+		if [ $? -ne 0 ]; then
+			ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 failed"
+		fi
+
+		# ganesha_grace (nfs-grace) RA follows grace-active attr
+		# w/ constraint location
+		attrd_updater -n ${OCF_RESKEY_grace_active} -v 1
+		if [ $? -ne 0 ]; then
+			ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 failed"
+		fi
+
+		# ganesha_mon (nfs-mon) and ganesha_grace (nfs-grace)
+		# track grace-active crm_attr (attr != crm_attr)
+		# we can't just use the attr as there's no way to query
+		# its value in RHEL6 pacemaker
+
+		crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
+		if [ $? -ne 0 ]; then
+			host=$(hostname)
+			crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
+			if [ $? -ne 0 ]; then
+				ocf_log info "mon monitor warning: crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed"
+			fi
+		fi
+
+		return ${OCF_SUCCESS}
+	fi
+
+	# VIP fail-over is triggered by clearing the
+	# ganesha-active node attribute on this node.
+	#
+	# Meanwhile the ganesha_grace notify() runs when its
+	# nfs-grace resource is disabled on a node; which
+	# is triggered by clearing the grace-active attribute
+	# on this node.
+	#
+	# We need to allow time for it to run and put
+	# the remaining ganesha.nfsds into grace before
+	# initiating the VIP fail-over.
+
+	attrd_updater -D -n ${OCF_RESKEY_grace_active}
+	if [ $? -ne 0 ]; then
+		ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_grace_active} failed"
+	fi
+
+	host=$(hostname -s)
+	crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
+	if [ $? -ne 0 ]; then
+		host=$(hostname)
+		crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
+		if [ $? -ne 0 ]; then
+			ocf_log info "mon monitor warning: crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 failed"
+		fi
+	fi
+
+	sleep ${OCF_RESKEY_grace_delay}
+
+	attrd_updater -D -n ${OCF_RESKEY_ganesha_active}
+	if [ $? -ne 0 ]; then
+		ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_ganesha_active} failed"
+	fi
+
+	return ${OCF_SUCCESS}
+}
+
+ganesha_mon_validate()
+{
+	return ${OCF_SUCCESS}
+}
+
+ganesha_mon_validate
+
+# Translate each action into the appropriate function call
+case ${__OCF_ACTION} in
+start)          ganesha_mon_start
+		;;
+stop)           ganesha_mon_stop
+		;;
+status|monitor) ganesha_mon_monitor
+		;;
+*)              ganesha_mon_usage
+		exit ${OCF_ERR_UNIMPLEMENTED}
+		;;
+esac
+
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
+exit $rc
diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd
new file mode 100644
index 00000000000..f91e8b6b8f7
--- /dev/null
+++ b/extras/ganesha/ocf/ganesha_nfsd
@@ -0,0 +1,167 @@
+#!/bin/bash
+#
+# Copyright (c) 2014 Anand Subramanian anands@redhat.com
+# Copyright (c) 2015 Red Hat Inc.
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+#
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+if [ -n "${OCF_DEBUG_LIBRARY}" ]; then
+	. ${OCF_DEBUG_LIBRARY}
+else
+	: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+	. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+fi
+
+OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage"
+: ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}}
+
+ganesha_meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="ganesha_nfsd">
+<version>1.0</version>
+
+<longdesc lang="en">
+This Linux-specific resource agent acts as a dummy
+resource agent for nfs-ganesha.
+</longdesc>
+
+<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc>
+
+<parameters>
+<parameter name="ha_vol_mnt">
+<longdesc lang="en">HA State Volume Mount Point</longdesc>
+<shortdesc lang="en">HA_State Volume Mount Point</shortdesc>
+<content type="string" default="" />
+</parameter>
+</parameters>
+
+<actions>
+<action name="start"   timeout="5s" />
+<action name="stop"    timeout="5s" />
+<action name="status" depth="0"  timeout="5s" interval="0" />
+<action name="monitor" depth="0"  timeout="5s" interval="0" />
+<action name="meta-data"  timeout="20s" />
+</actions>
+</resource-agent>
+END
+
+return ${OCF_SUCCESS}
+}
+
+ganesha_nfsd_usage() {
+	echo "ganesha.nfsd USAGE"
+}
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+	meta-data)	ganesha_meta_data
+			exit ${OCF_SUCCESS}
+			;;
+	usage|help)	ganesha_usage
+			exit ${OCF_SUCCESS}
+			;;
+	*)
+			;;
+esac
+
+ganesha_nfsd_start()
+{
+	local long_host=$(hostname)
+
+	if [[ -d /var/lib/nfs ]]; then
+		mv /var/lib/nfs /var/lib/nfs.backup
+		if [ $? -ne 0 ]; then
+			ocf_log notice "mv /var/lib/nfs /var/lib/nfs.backup failed"
+		fi
+		ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs
+		if [ $? -ne 0 ]; then
+			ocf_log notice "ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs failed"
+		fi
+	fi
+
+	return ${OCF_SUCCESS}
+}
+
+ganesha_nfsd_stop()
+{
+
+	if [ -L /var/lib/nfs -a -d /var/lib/nfs.backup ]; then
+		rm -f /var/lib/nfs
+		if [ $? -ne 0 ]; then
+			ocf_log notice "rm -f /var/lib/nfs failed"
+		fi
+		mv /var/lib/nfs.backup /var/lib/nfs
+		if [ $? -ne 0 ]; then
+			ocf_log notice "mv /var/lib/nfs.backup /var/lib/nfs failed"
+		fi
+	fi
+
+	return ${OCF_SUCCESS}
+}
+
+ganesha_nfsd_monitor()
+{
+	# pacemaker checks to see if RA is already running before starting it.
+	# if we return success, then it's presumed it's already running and
+	# doesn't need to be started, i.e. invoke the start action.
+	# return something other than success to make pacemaker invoke the
+	# start action
+	if [[ -L /var/lib/nfs ]]; then
+		return ${OCF_SUCCESS}
+	fi
+	return ${OCF_NOT_RUNNING}
+}
+
+ganesha_nfsd_validate()
+{
+	return ${OCF_SUCCESS}
+}
+
+ganesha_nfsd_validate
+
+# ocf_log notice "ganesha_nfsd ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION"
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          ganesha_nfsd_start
+		;;
+stop)           ganesha_nfsd_stop
+		;;
+status|monitor) ganesha_nfsd_monitor
+		;;
+*)              ganesha_nfsd_usage
+		exit ${OCF_ERR_UNIMPLEMENTED}
+		;;
+esac
+
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc"
+exit $rc
diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am
new file mode 100644
index 00000000000..7e345fd5f19
--- /dev/null
+++ b/extras/ganesha/scripts/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST= create-export-ganesha.sh generate-epoch.py dbus-send.sh \
+			ganesha-ha.sh
+
+scriptsdir = $(libexecdir)/ganesha
+scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh generate-epoch.py \
+				  ganesha-ha.sh
diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh
new file mode 100755
index 00000000000..3040e8138b0
--- /dev/null
+++ b/extras/ganesha/scripts/create-export-ganesha.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+#This script is called by glusterd when the user
+#tries to export a volume via NFS-Ganesha.
+#An export file specific to a volume
+#is created in GANESHA_DIR/exports.
+
+# Try loading the config from any of the distro
+# specific configuration locations
+if [ -f /etc/sysconfig/ganesha ]
+        then
+        . /etc/sysconfig/ganesha
+fi
+if [ -f /etc/conf.d/ganesha ]
+        then
+        . /etc/conf.d/ganesha
+fi
+if [ -f /etc/default/ganesha ]
+        then
+        . /etc/default/ganesha
+fi
+
+GANESHA_DIR=${1%/}
+OPTION=$2
+VOL=$3
+CONF=$GANESHA_DIR"/ganesha.conf"
+declare -i EXPORT_ID
+
+function check_cmd_status()
+{
+        if [ "$1" != "0" ]
+                 then
+                 rm -rf $GANESHA_DIR/exports/export.$VOL.conf
+                 sed -i /$VOL.conf/d $CONF
+                 exit 1
+        fi
+}
+
+
+if [ ! -d "$GANESHA_DIR/exports" ];
+        then
+        mkdir $GANESHA_DIR/exports
+        check_cmd_status `echo $?`
+fi
+
+function write_conf()
+{
+echo -e "# WARNING : Using Gluster CLI will overwrite manual
+# changes made to this file. To avoid it, edit the
+# file and run ganesha-ha.sh --refresh-config."
+
+echo "EXPORT{"
+echo "      Export_Id = 2;"
+echo "      Path = \"/$VOL\";"
+echo "      FSAL {"
+echo "           name = "GLUSTER";"
+echo "           hostname=\"localhost\";"
+echo  "          volume=\"$VOL\";"
+echo "           }"
+echo "      Access_type = RW;"
+echo "      Disable_ACL = true;"
+echo '      Squash="No_root_squash";'
+echo "      Pseudo=\"/$VOL\";"
+echo '      Protocols = "3", "4" ;'
+echo '      Transports = "UDP","TCP";'
+echo '      SecType = "sys";'
+echo '      Security_Label = False;'
+echo "     }"
+}
+if [ "$OPTION" = "on" ];
+then
+        if ! (cat $CONF | grep  $VOL.conf\"$ )
+        then
+                write_conf $@ > $GANESHA_DIR/exports/export.$VOL.conf
+                echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF
+                count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l`
+                if [ "$count" = "1" ] ; then
+                        EXPORT_ID=2
+                else
+                        EXPORT_ID=`cat $GANESHA_DIR/.export_added`
+                        check_cmd_status `echo $?`
+                        EXPORT_ID=EXPORT_ID+1
+                        sed -i s/Export_Id.*/"Export_Id= $EXPORT_ID ;"/ \
+                        $GANESHA_DIR/exports/export.$VOL.conf
+                        check_cmd_status `echo $?`
+                fi
+                echo $EXPORT_ID > $GANESHA_DIR/.export_added
+        fi
+else
+        rm -rf $GANESHA_DIR/exports/export.$VOL.conf
+        sed -i /$VOL.conf/d $CONF
+fi
diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh
new file mode 100755
index 00000000000..9d613a0e7ad
--- /dev/null
+++ b/extras/ganesha/scripts/dbus-send.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Try loading the config from any of the distro
+# specific configuration locations
+if [ -f /etc/sysconfig/ganesha ]
+        then
+        . /etc/sysconfig/ganesha
+fi
+if [ -f /etc/conf.d/ganesha ]
+        then
+        . /etc/conf.d/ganesha
+fi
+if [ -f /etc/default/ganesha ]
+        then
+        . /etc/default/ganesha
+fi
+
+GANESHA_DIR=${1%/}
+OPTION=$2
+VOL=$3
+CONF=$GANESHA_DIR"/ganesha.conf"
+
+function check_cmd_status()
+{
+        if [ "$1" != "0" ]
+        then
+                logger "dynamic export failed on node :${hostname -s}"
+        fi
+}
+
+#This function keeps track of export IDs and increments it with every new entry
+function dynamic_export_add()
+{
+        dbus-send  --system \
+--dest=org.ganesha.nfsd  /org/ganesha/nfsd/ExportMgr \
+org.ganesha.nfsd.exportmgr.AddExport  string:$GANESHA_DIR/exports/export.$VOL.conf \
+string:"EXPORT(Path=/$VOL)"
+        check_cmd_status `echo $?`
+}
+
+#This function removes an export dynamically(uses the export_id of the export)
+function dynamic_export_remove()
+{
+        # Below bash fetch all the export from ShowExport command and search
+        # export entry based on path and then get its export entry.
+        # There are two possiblities for path, either entire volume will be
+        # exported or subdir. It handles both cases. But it remove only first
+        # entry from the list based on assumption that entry exported via cli
+        # has lowest export id value
+	removed_id=$(dbus-send --type=method_call --print-reply --system \
+                    --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
+                    org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \
+                    "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \
+		    | head -1)
+
+        dbus-send --print-reply --system \
+--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
+org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id
+        check_cmd_status `echo $?`
+}
+
+if [ "$OPTION" = "on" ];
+then
+        dynamic_export_add $@
+fi
+
+if [ "$OPTION" = "off" ];
+then
+        dynamic_export_remove $@
+fi
diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh
new file mode 100644
index 00000000000..9790a719e10
--- /dev/null
+++ b/extras/ganesha/scripts/ganesha-ha.sh
@@ -0,0 +1,1199 @@
+#!/bin/bash
+
+# Copyright 2015-2016 Red Hat Inc.  All Rights Reserved
+#
+# Pacemaker+Corosync High Availability for NFS-Ganesha
+#
+# setup, teardown, add, delete, refresh-config, and status
+#
+# Each participating node in the cluster is assigned a virtual IP (VIP)
+# which fails over to another node when its associated ganesha.nfsd dies
+# for any reason. After the VIP is moved to another node all the
+# ganesha.nfsds are send a signal using DBUS to put them into NFS GRACE.
+#
+# There are six resource agent types used: ganesha_mon, ganesha_grace,
+# ganesha_nfsd, IPaddr, and Dummy. ganesha_mon is used to monitor the
+# ganesha.nfsd. ganesha_grace is used to send the DBUS signal to put
+# the remaining ganesha.nfsds into grace. ganesha_nfsd is used to start
+# and stop the ganesha.nfsd during setup and teardown. IPaddr manages
+# the VIP. A Dummy resource named $hostname-trigger_ip-1 is used to
+# ensure that the NFS GRACE DBUS signal is sent after the VIP moves to
+# the new host.
+
+GANESHA_HA_SH=$(realpath $0)
+HA_NUM_SERVERS=0
+HA_SERVERS=""
+HA_VOL_NAME="gluster_shared_storage"
+HA_VOL_MNT="/run/gluster/shared_storage"
+HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha"
+SERVICE_MAN="DISTRO_NOT_FOUND"
+
+# rhel, fedora id, version
+ID=""
+VERSION_ID=""
+
+PCS9OR10_PCS_CNAME_OPTION=""
+PCS9OR10_PCS_CLONE_OPTION="clone"
+SECRET_PEM="/var/lib/glusterd/nfs/secret.pem"
+
+# UNBLOCK RA uses shared_storage which may become unavailable
+# during any of the nodes reboot. Hence increase timeout value.
+PORTBLOCK_UNBLOCK_TIMEOUT="60s"
+
+# Try loading the config from any of the distro
+# specific configuration locations
+if [ -f /etc/sysconfig/ganesha ]
+        then
+        . /etc/sysconfig/ganesha
+fi
+if [ -f /etc/conf.d/ganesha ]
+        then
+        . /etc/conf.d/ganesha
+fi
+if [ -f /etc/default/ganesha ]
+        then
+        . /etc/default/ganesha
+fi
+
+GANESHA_CONF=
+
+function find_rhel7_conf
+{
+ while [[ $# > 0 ]]
+        do
+                key="$1"
+                case $key in
+                        -f)
+                         CONFFILE="$2"
+                         break;
+                         ;;
+                         *)
+                         ;;
+                 esac
+                 shift
+         done
+}
+
+if [ -z ${CONFFILE} ]
+        then
+        find_rhel7_conf ${OPTIONS}
+
+fi
+
+GANESHA_CONF=${CONFFILE:-/etc/ganesha/ganesha.conf}
+
+usage() {
+
+        echo "Usage      : add|delete|refresh-config|status"
+        echo "Add-node   : ganesha-ha.sh --add <HA_CONF_DIR> \
+<NODE-HOSTNAME>  <NODE-VIP>"
+        echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR> \
+<NODE-HOSTNAME>"
+        echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR> \
+<volume>"
+        echo "Status : ganesha-ha.sh --status <HA_CONFDIR>"
+}
+
+determine_service_manager () {
+
+        if [ -e "/bin/systemctl" ];
+        then
+                SERVICE_MAN="/bin/systemctl"
+        elif [ -e "/sbin/invoke-rc.d" ];
+        then
+                SERVICE_MAN="/sbin/invoke-rc.d"
+        elif [ -e "/sbin/service" ];
+        then
+                SERVICE_MAN="/sbin/service"
+        fi
+        if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]]
+        then
+                logger "Service manager not recognized, exiting"
+                exit 1
+        fi
+}
+
+manage_service ()
+{
+        local action=${1}
+        local new_node=${2}
+        local option=
+
+        if [[ "${action}" == "start" ]]; then
+                option="yes"
+        else
+                option="no"
+        fi
+        ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option"
+
+        if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]
+        then
+                ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${new_node} "${SERVICE_MAN}  ${action} nfs-ganesha"
+        else
+                ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}"
+        fi
+}
+
+
+check_cluster_exists()
+{
+    local name=${1}
+    local cluster_name=""
+
+    if [ -e /var/run/corosync.pid ]; then
+        cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3)
+        if [[ "${cluster_name}X" == "${name}X" ]]; then
+            logger "$name already exists, exiting"
+            exit 0
+        fi
+    fi
+}
+
+
+determine_servers()
+{
+    local cmd=${1}
+    local num_servers=0
+    local tmp_ifs=${IFS}
+    local ha_servers=""
+
+    if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then
+        ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//')
+        IFS=$' '
+        for server in ${ha_servers} ; do
+            num_servers=$(expr ${num_servers} + 1)
+        done
+        IFS=${tmp_ifs}
+        HA_NUM_SERVERS=${num_servers}
+        HA_SERVERS="${ha_servers}"
+    else
+        IFS=$','
+        for server in ${HA_CLUSTER_NODES} ; do
+            num_servers=$(expr ${num_servers} + 1)
+        done
+        IFS=${tmp_ifs}
+        HA_NUM_SERVERS=${num_servers}
+        HA_SERVERS="${HA_CLUSTER_NODES//,/ }"
+    fi
+}
+
+stop_ganesha_all()
+{
+    local serverlist=${1}
+    for node in ${serverlist} ; do
+        manage_service "stop" ${node}
+    done
+}
+
+setup_cluster()
+{
+    local name=${1}
+    local num_servers=${2}
+    local servers=${3}
+    local unclean=""
+    local quorum_policy="stop"
+
+    logger "setting up cluster ${name} with the following ${servers}"
+
+    # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers}
+    pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers}
+    if [ $? -ne 0 ]; then
+        logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out"
+        #set up failed stop all ganesha process and clean up symlinks in cluster
+        stop_ganesha_all "${servers}"
+        exit 1;
+    fi
+
+    # pcs cluster auth ${servers}
+    pcs cluster auth
+    if [ $? -ne 0 ]; then
+        logger "pcs cluster auth failed"
+    fi
+
+    pcs cluster start --all
+    if [ $? -ne 0 ]; then
+        logger "pcs cluster start failed"
+        exit 1;
+    fi
+
+    sleep 1
+    # wait for the cluster to elect a DC before querying or writing
+    # to the CIB. BZ 1334092
+    crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1
+    while [ $? -ne 0 ]; do
+        crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1
+    done
+
+    unclean=$(pcs status | grep -u "UNCLEAN")
+    while [[ "${unclean}X" == "UNCLEANX" ]]; do
+         sleep 1
+         unclean=$(pcs status | grep -u "UNCLEAN")
+    done
+    sleep 1
+
+    if [ ${num_servers} -lt 3 ]; then
+        quorum_policy="ignore"
+    fi
+    pcs property set no-quorum-policy=${quorum_policy}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed"
+    fi
+
+    pcs property set stonith-enabled=false
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs property set stonith-enabled=false failed"
+    fi
+}
+
+
+setup_finalize_ha()
+{
+    local cibfile=${1}
+    local stopped=""
+
+    stopped=$(pcs status | grep -u "Stopped")
+    while [[ "${stopped}X" == "StoppedX" ]]; do
+         sleep 1
+         stopped=$(pcs status | grep -u "Stopped")
+    done
+}
+
+
+refresh_config ()
+{
+        local short_host=$(hostname -s)
+        local VOL=${1}
+        local HA_CONFDIR=${2}
+        local short_host=$(hostname -s)
+
+        local export_id=$(grep ^[[:space:]]*Export_Id $HA_CONFDIR/exports/export.$VOL.conf |\
+                          awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]')
+
+
+        if [ -e ${SECRET_PEM} ]; then
+        while [[ ${3} ]]; do
+            current_host=`echo ${3} | cut -d "." -f 1`
+            if [[ ${short_host} != ${current_host} ]]; then
+                output=$(ssh -oPasswordAuthentication=no \
+-oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \
+"dbus-send --print-reply --system --dest=org.ganesha.nfsd \
+/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \
+string:$HA_CONFDIR/exports/export.$VOL.conf \
+string:\"EXPORT(Export_Id=$export_id)\" 2>&1")
+                ret=$?
+                logger <<< "${output}"
+                if [ ${ret} -ne 0 ]; then
+                        echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}"
+                else
+                        echo "Refresh-config completed on ${current_host}."
+                fi
+
+          fi
+          shift
+        done
+    else
+        echo "Error: refresh-config failed. Passwordless ssh is not enabled."
+        exit 1
+    fi
+
+    # Run the same command on the localhost,
+        output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \
+/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \
+string:$HA_CONFDIR/exports/export.$VOL.conf \
+string:"EXPORT(Export_Id=$export_id)" 2>&1)
+        ret=$?
+        logger <<< "${output}"
+        if [ ${ret} -ne 0 ] ; then
+                echo "Refresh-config failed on localhost."
+        else
+                echo "Success: refresh-config completed."
+        fi
+}
+
+
+teardown_cluster()
+{
+    local name=${1}
+
+    for server in ${HA_SERVERS} ; do
+        if [[ ${HA_CLUSTER_NODES} != *${server}* ]]; then
+            logger "info: ${server} is not in config, removing"
+
+            pcs cluster stop ${server} --force
+            if [ $? -ne 0 ]; then
+                logger "warning: pcs cluster stop ${server} failed"
+            fi
+
+            pcs cluster node remove ${server}
+            if [ $? -ne 0 ]; then
+                logger "warning: pcs cluster node remove ${server} failed"
+            fi
+        fi
+    done
+
+    # BZ 1193433 - pcs doesn't reload cluster.conf after modification
+    # after teardown completes, a subsequent setup will appear to have
+    # 'remembered' the deleted node. You can work around this by
+    # issuing another `pcs cluster node remove $node`,
+    # `crm_node -f -R $server`, or
+    # `cibadmin --delete --xml-text '<node id="$server"
+    # uname="$server"/>'
+
+    pcs cluster stop --all
+    if [ $? -ne 0 ]; then
+        logger "warning pcs cluster stop --all failed"
+    fi
+
+    pcs cluster destroy
+    if [ $? -ne 0 ]; then
+        logger "error pcs cluster destroy failed"
+        exit 1
+    fi
+}
+
+
+cleanup_ganesha_config ()
+{
+    rm -f /etc/corosync/corosync.conf
+    rm -rf /etc/cluster/cluster.conf*
+    rm -rf /var/lib/pacemaker/cib/*
+    sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d'  $HA_CONFDIR/ganesha.conf
+}
+
+do_create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+    local primary=${1}; shift
+    local weight="1000"
+
+    # first a constraint location rule that says the VIP must be where
+    # there's a ganesha.nfsd running
+    pcs -f ${cibfile} constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 failed"
+    fi
+
+    # then a set of constraint location prefers to set the prefered order
+    # for where a VIP should move
+    while [[ ${1} ]]; do
+        pcs -f ${cibfile} constraint location ${primary}-group prefers ${1}=${weight}
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs constraint location ${primary}-group prefers ${1}=${weight} failed"
+        fi
+        weight=$(expr ${weight} + 1000)
+        shift
+    done
+    # and finally set the highest preference for the VIP to its home node
+    # default weight when created is/was 100.
+    # on Fedora setting appears to be additive, so to get the desired
+    # value we adjust the weight
+    # weight=$(expr ${weight} - 100)
+    pcs -f ${cibfile} constraint location ${primary}-group prefers ${primary}=${weight}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs constraint location ${primary}-group prefers ${primary}=${weight} failed"
+    fi
+}
+
+
+wrap_create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+    local primary=${1}; shift
+    local head=""
+    local tail=""
+
+    # build a list of peers, e.g. for a four node cluster, for node1,
+    # the result is "node2 node3 node4"; for node2, "node3 node4 node1"
+    # and so on.
+    while [[ ${1} ]]; do
+        if [[ ${1} == ${primary} ]]; then
+            shift
+            while [[ ${1} ]]; do
+                tail=${tail}" "${1}
+                shift
+            done
+        else
+            head=${head}" "${1}
+        fi
+        shift
+    done
+    do_create_virt_ip_constraints ${cibfile} ${primary} ${tail} ${head}
+}
+
+
+create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+
+    while [[ ${1} ]]; do
+        wrap_create_virt_ip_constraints ${cibfile} ${1} ${HA_SERVERS}
+        shift
+    done
+}
+
+
+setup_create_resources()
+{
+    local cibfile=$(mktemp -u)
+
+    # fixup /var/lib/nfs
+    logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}"
+    pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed"
+    fi
+
+    pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed"
+    fi
+
+    # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace
+    # start method. Allow time for ganesha_mon to start and set the
+    # ganesha-active crm_attribute
+    sleep 5
+
+    pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed"
+    fi
+
+    pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1"
+    fi
+
+    pcs cluster cib ${cibfile}
+
+    while [[ ${1} ]]; do
+
+        # this is variable indirection
+        # from a nvs like 'VIP_host1=10.7.6.5' or 'VIP_host1="10.7.6.5"'
+        # (or VIP_host-1=..., or VIP_host-1.my.domain.name=...)
+        # a variable 'clean_name' is created (e.g. w/ value 'VIP_host_1')
+        # and a clean nvs (e.g. w/ value 'VIP_host_1="10_7_6_5"')
+        # after the `eval ${clean_nvs}` there is a variable VIP_host_1
+        # with the value '10_7_6_5', and the following \$$ magic to
+        # reference it, i.e. `eval tmp_ipaddr=\$${clean_name}` gives us
+        # ${tmp_ipaddr} with 10_7_6_5 and then convert the _s back to .s
+        # to give us ipaddr="10.7.6.5". whew!
+        name="VIP_${1}"
+        clean_name=${name//[-.]/_}
+        nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf)
+        clean_nvs=${nvs//[-.]/_}
+        eval ${clean_nvs}
+        eval tmp_ipaddr=\$${clean_name}
+        ipaddr=${tmp_ipaddr//_/.}
+
+        pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=block ip=${ipaddr} --group ${1}-group
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-nfs_block failed"
+        fi
+        pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+        cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+            cidr_netmask=32 op monitor interval=15s failed"
+        fi
+
+        pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed"
+        fi
+
+        pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \
+        tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \
+        op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \
+        op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-nfs_unblock failed"
+        fi
+
+
+        shift
+    done
+
+    create_virt_ip_constraints ${cibfile} ${HA_SERVERS}
+
+    pcs cluster cib-push ${cibfile}
+    if [ $? -ne 0 ]; then
+        logger "warning pcs cluster cib-push ${cibfile} failed"
+    fi
+    rm -f ${cibfile}
+}
+
+
+teardown_resources()
+{
+    # local mntpt=$(grep ha-vol-mnt ${HA_CONFIG_FILE} | cut -d = -f 2)
+
+    # restore /var/lib/nfs
+    logger "notice: pcs resource delete nfs_setup-clone"
+    pcs resource delete nfs_setup-clone
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource delete nfs_setup-clone failed"
+    fi
+
+    # delete -clone resource agents
+    # in particular delete the ganesha monitor so we don't try to
+    # trigger anything when we shut down ganesha next.
+    pcs resource delete nfs-mon-clone
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource delete nfs-mon-clone failed"
+    fi
+
+    pcs resource delete nfs-grace-clone
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs resource delete nfs-grace-clone failed"
+    fi
+
+    while [[ ${1} ]]; do
+        pcs resource delete ${1}-group
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs resource delete ${1}-group failed"
+        fi
+        shift
+    done
+
+}
+
+
+recreate_resources()
+{
+    local cibfile=${1}; shift
+
+    while [[ ${1} ]]; do
+        # this is variable indirection
+        # see the comment on the same a few lines up
+        name="VIP_${1}"
+        clean_name=${name//[-.]/_}
+        nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf)
+        clean_nvs=${nvs//[-.]/_}
+        eval ${clean_nvs}
+        eval tmp_ipaddr=\$${clean_name}
+        ipaddr=${tmp_ipaddr//_/.}
+
+        pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=block ip=${ipaddr} --group ${1}-group
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-nfs_block failed"
+        fi
+        pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+        cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \
+            cidr_netmask=32 op monitor interval=15s failed"
+        fi
+
+        pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed"
+        fi
+
+        pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \
+        portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \
+        tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \
+        op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \
+        op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
+        if [ $? -ne 0 ]; then
+            logger "warning pcs resource create ${1}-nfs_unblock failed"
+        fi
+
+        shift
+    done
+}
+
+
+addnode_recreate_resources()
+{
+    local cibfile=${1}; shift
+    local add_node=${1}; shift
+    local add_vip=${1}; shift
+
+    recreate_resources ${cibfile} ${HA_SERVERS}
+
+    pcs -f ${cibfile} resource create ${add_node}-nfs_block ocf:heartbeat:portblock \
+    protocol=tcp portno=2049 action=block ip=${add_vip} --group ${add_node}-group
+    if [ $? -ne 0 ]; then
+        logger "warning pcs resource create ${add_node}-nfs_block failed"
+    fi
+    pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \
+    ip=${add_vip} cidr_netmask=32 op monitor interval=15s --group ${add_node}-group \
+    --after ${add_node}-nfs_block
+    if [ $? -ne 0 ]; then
+        logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \
+        ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed"
+    fi
+
+    pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 failed"
+    fi
+    pcs -f ${cibfile} resource create ${add_node}-nfs_unblock ocf:heartbeat:portblock \
+    protocol=tcp portno=2049 action=unblock ip=${add_vip} reset_local_on_unblock_stop=true \
+    tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${add_node}-group --after \
+    ${add_node}-cluster_ip-1 op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start \
+    timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op monitor interval=10s \
+    timeout=${PORTBLOCK_UNBLOCK_TIMEOUT}
+    if [ $? -ne 0 ]; then
+        logger "warning pcs resource create ${add_node}-nfs_unblock failed"
+    fi
+}
+
+
+clear_resources()
+{
+    local cibfile=${1}; shift
+
+    while [[ ${1} ]]; do
+        pcs -f ${cibfile} resource delete ${1}-group
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs -f ${cibfile} resource delete ${1}-group"
+        fi
+
+        shift
+    done
+}
+
+
+addnode_create_resources()
+{
+    local add_node=${1}; shift
+    local add_vip=${1}; shift
+    local cibfile=$(mktemp -u)
+
+    # start HA on the new node
+    pcs cluster start ${add_node}
+    if [ $? -ne 0 ]; then
+       logger "warning: pcs cluster start ${add_node} failed"
+    fi
+
+    pcs cluster cib ${cibfile}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs cluster cib ${cibfile} failed"
+    fi
+
+    # delete all the -cluster_ip-1 resources, clearing
+    # their constraints, then create them again so we can
+    # recompute their constraints
+    clear_resources ${cibfile} ${HA_SERVERS}
+    addnode_recreate_resources ${cibfile} ${add_node} ${add_vip}
+
+    HA_SERVERS="${HA_SERVERS} ${add_node}"
+    create_virt_ip_constraints ${cibfile} ${HA_SERVERS}
+
+    pcs cluster cib-push ${cibfile}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs cluster cib-push ${cibfile} failed"
+    fi
+    rm -f ${cibfile}
+}
+
+
+deletenode_delete_resources()
+{
+    local node=${1}; shift
+    local ha_servers=$(echo "${HA_SERVERS}" | sed s/${node}//)
+    local cibfile=$(mktemp -u)
+
+    pcs cluster cib ${cibfile}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs cluster cib ${cibfile} failed"
+    fi
+
+    # delete all the -cluster_ip-1 and -trigger_ip-1 resources,
+    # clearing their constraints, then create them again so we can
+    # recompute their constraints
+    clear_resources ${cibfile} ${HA_SERVERS}
+    recreate_resources ${cibfile} ${ha_servers}
+    HA_SERVERS=$(echo "${ha_servers}" | sed -e "s/  / /")
+
+    create_virt_ip_constraints ${cibfile} ${HA_SERVERS}
+
+    pcs cluster cib-push ${cibfile}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs cluster cib-push ${cibfile} failed"
+    fi
+    rm -f ${cibfile}
+
+}
+
+
+deletenode_update_haconfig()
+{
+    local name="VIP_${1}"
+    local clean_name=${name//[-.]/_}
+
+    ha_servers=$(echo ${HA_SERVERS} | sed -e "s/ /,/")
+    sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf
+}
+
+
+setup_state_volume()
+{
+    local mnt=${HA_VOL_MNT}
+    local longname=""
+    local shortname=""
+    local dname=""
+    local dirname=""
+
+    longname=$(hostname)
+    dname=${longname#$(hostname -s)}
+
+    while [[ ${1} ]]; do
+
+        if [[ ${1} == *${dname} ]]; then
+            dirname=${1}
+        else
+            dirname=${1}${dname}
+        fi
+
+        if [ ! -d ${mnt}/nfs-ganesha/tickle_dir ]; then
+            mkdir ${mnt}/nfs-ganesha/tickle_dir
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+        fi
+        if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then
+            touch ${mnt}/nfs-ganesha/${dirname}/nfs/state
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+        fi
+        if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then
+            mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+            chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+        fi
+        if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then
+            touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state
+        fi
+        for server in ${HA_SERVERS} ; do
+            if [[ ${server} != ${dirname} ]]; then
+                ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server}
+                ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server}
+            fi
+        done
+        shift
+    done
+
+}
+
+
+enable_pacemaker()
+{
+    while [[ ${1} ]]; do
+        if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then
+            ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker"
+        else
+            ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+${SECRET_PEM} root@${1} "${SERVICE_MAN} pacemaker enable"
+        fi
+        shift
+    done
+}
+
+
+addnode_state_volume()
+{
+    local newnode=${1}; shift
+    local mnt=${HA_VOL_MNT}
+    local longname=""
+    local dname=""
+    local dirname=""
+
+    longname=$(hostname)
+    dname=${longname#$(hostname -s)}
+
+    if [[ ${newnode} == *${dname} ]]; then
+        dirname=${newnode}
+    else
+        dirname=${newnode}${dname}
+    fi
+
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd
+    fi
+    if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then
+        touch ${mnt}/nfs-ganesha/${dirname}/nfs/state
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm
+    fi
+    if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then
+        mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+        chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak
+    fi
+    if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then
+        touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state
+    fi
+
+    for server in ${HA_SERVERS} ; do
+
+        if [[ ${server} != ${dirname} ]]; then
+            ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server}
+            ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server}
+
+            ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname}
+            ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/statd ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname}
+        fi
+    done
+
+}
+
+
+delnode_state_volume()
+{
+    local delnode=${1}; shift
+    local mnt=${HA_VOL_MNT}
+    local longname=""
+    local dname=""
+    local dirname=""
+
+    longname=$(hostname)
+    dname=${longname#$(hostname -s)}
+
+    if [[ ${delnode} == *${dname} ]]; then
+        dirname=${delnode}
+    else
+        dirname=${delnode}${dname}
+    fi
+
+    rm -rf ${mnt}/nfs-ganesha/${dirname}
+
+    for server in ${HA_SERVERS} ; do
+        if [[ ${server} != ${dirname} ]]; then
+            rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname}
+            rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname}
+        fi
+    done
+}
+
+
+status()
+{
+    local scratch=$(mktemp)
+    local regex_str="^${1}-cluster_ip-1"
+    local healthy=0
+    local index=1
+    local nodes
+
+    # change tabs to spaces, strip leading spaces, including any 
+    # new '*' at the beginning of a line introduced in pcs-0.10.x
+    pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch}
+
+    nodes[0]=${1}; shift
+
+    # make a regex of the configured nodes
+    # and initalize the nodes array for later
+    while [[ ${1} ]]; do
+
+        regex_str="${regex_str}|^${1}-cluster_ip-1"
+        nodes[${index}]=${1}
+        ((index++))
+        shift
+    done
+
+    # print the nodes that are expected to be online
+    grep -E "Online:" ${scratch}
+
+    echo
+
+    # print the VIPs and which node they are on
+    grep -E "${regex_str}" < ${scratch} | cut -d ' ' -f 1,4
+
+    echo
+
+    # check if the VIP and port block/unblock RAs are on the expected nodes
+    for n in ${nodes[*]}; do
+
+        grep -E -x "${n}-nfs_block \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+        grep -E -x "${n}-cluster_ip-1 \(ocf::heartbeat:IPaddr\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+        grep -E -x "${n}-nfs_unblock \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch}
+        result=$?
+        ((healthy+=${result}))
+    done
+
+    grep -E "\):\ Stopped|FAILED" > /dev/null 2>&1 ${scratch}
+    result=$?
+
+    if [ ${result} -eq 0 ]; then
+        echo "Cluster HA Status: BAD"
+    elif [ ${healthy} -eq 0 ]; then
+        echo "Cluster HA Status: HEALTHY"
+    else
+        echo "Cluster HA Status: FAILOVER"
+    fi
+
+    rm -f ${scratch}
+}
+
+create_ganesha_conf_file()
+{
+        if [[ "$1" == "yes" ]];
+        then
+                if [  -e $GANESHA_CONF ];
+                then
+                        rm -rf $GANESHA_CONF
+                fi
+        # The symlink /etc/ganesha/ganesha.conf need to be
+        # created using ganesha conf file mentioned in the
+        # shared storage. Every node will only have this
+        # link and actual file will stored in shared storage,
+        # so that ganesha conf editing of ganesha conf will
+        # be easy as well as it become more consistent.
+
+                ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+        else
+        # Restoring previous file
+                rm -rf $GANESHA_CONF
+                cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+                sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $GANESHA_CONF
+        fi
+}
+
+set_quorum_policy()
+{
+    local quorum_policy="stop"
+    local num_servers=${1}
+
+    if [ ${num_servers} -lt 3 ]; then
+        quorum_policy="ignore"
+    fi
+    pcs property set no-quorum-policy=${quorum_policy}
+    if [ $? -ne 0 ]; then
+        logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed"
+    fi
+}
+
+main()
+{
+
+    local cmd=${1}; shift
+    if [[ ${cmd} == *help ]]; then
+        usage
+        exit 0
+    fi
+
+    if (selinuxenabled) ;then
+     semanage boolean -m gluster_use_execmem --on
+    fi
+
+    local osid=""
+
+    osid=$(grep ^ID= /etc/os-release)
+    eval $(echo ${osid} | grep -F ID=)
+    osid=$(grep ^VERSION_ID= /etc/os-release)
+    eval $(echo ${osid} | grep -F VERSION_ID=)
+
+    HA_CONFDIR=${1%/}; shift
+    local ha_conf=${HA_CONFDIR}/ganesha-ha.conf
+    local node=""
+    local vip=""
+
+    # ignore any comment lines
+    cfgline=$(grep  ^HA_NAME= ${ha_conf})
+    eval $(echo ${cfgline} | grep -F HA_NAME=)
+    cfgline=$(grep  ^HA_CLUSTER_NODES= ${ha_conf})
+    eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=)
+
+    case "${cmd}" in
+
+    setup | --setup)
+        logger "setting up ${HA_NAME}"
+
+        check_cluster_exists ${HA_NAME}
+
+        determine_servers "setup"
+
+        # Fedora 29+ and rhel/centos 8 has PCS-0.10.x
+        # default is pcs-0.10.x options but check for
+        # rhel/centos 7 (pcs-0.9.x) and adjust accordingly
+        if [[ ! ${ID} =~ {rhel,centos} ]]; then
+            if [[ ${VERSION_ID} == 7.* ]]; then
+                PCS9OR10_PCS_CNAME_OPTION="--name"
+                PCS9OR10_PCS_CLONE_OPTION="--clone"
+            fi
+        fi
+
+        if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then
+
+            determine_service_manager
+
+            setup_cluster ${HA_NAME} ${HA_NUM_SERVERS} "${HA_SERVERS}"
+
+            setup_create_resources ${HA_SERVERS}
+
+            setup_finalize_ha
+
+            setup_state_volume ${HA_SERVERS}
+
+            enable_pacemaker ${HA_SERVERS}
+
+        else
+
+            logger "insufficient servers for HA, aborting"
+        fi
+        ;;
+
+    teardown | --teardown)
+        logger "tearing down ${HA_NAME}"
+
+        determine_servers "teardown"
+
+        teardown_resources ${HA_SERVERS}
+
+        teardown_cluster ${HA_NAME}
+
+        cleanup_ganesha_config ${HA_CONFDIR}
+        ;;
+
+    cleanup | --cleanup)
+        cleanup_ganesha_config ${HA_CONFDIR}
+        ;;
+
+    add | --add)
+        node=${1}; shift
+        vip=${1}; shift
+
+        logger "adding ${node} with ${vip} to ${HA_NAME}"
+
+        determine_service_manager
+
+        manage_service "start" ${node}
+
+        determine_servers "add"
+
+        pcs cluster node add ${node}
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs cluster node add ${node} failed"
+        fi
+
+        addnode_create_resources ${node} ${vip}
+        # Subsequent add-node recreates resources for all the nodes
+        # that already exist in the cluster. The nodes are picked up
+        # from the entries in the ganesha-ha.conf file. Adding the
+        # newly added node to the file so that the resources specfic
+        # to this node is correctly recreated in the future.
+        clean_node=${node//[-.]/_}
+        echo "VIP_${node}=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf
+
+        NEW_NODES="$HA_CLUSTER_NODES,${node}"
+
+        sed -i s/HA_CLUSTER_NODES.*/"HA_CLUSTER_NODES=\"$NEW_NODES\""/ \
+$HA_CONFDIR/ganesha-ha.conf
+
+        addnode_state_volume ${node}
+
+        # addnode_create_resources() already appended ${node} to
+        # HA_SERVERS, so only need to increment HA_NUM_SERVERS
+        # and set quorum policy
+        HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} + 1)
+        set_quorum_policy ${HA_NUM_SERVERS}
+        ;;
+
+    delete | --delete)
+        node=${1}; shift
+
+        logger "deleting ${node} from ${HA_NAME}"
+
+        determine_servers "delete"
+
+        deletenode_delete_resources ${node}
+
+        pcs cluster node remove ${node}
+        if [ $? -ne 0 ]; then
+            logger "warning: pcs cluster node remove ${node} failed"
+        fi
+
+        deletenode_update_haconfig ${node}
+
+        delnode_state_volume ${node}
+
+        determine_service_manager
+
+        manage_service "stop" ${node}
+
+        HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} - 1)
+        set_quorum_policy ${HA_NUM_SERVERS}
+        ;;
+
+    status | --status)
+        determine_servers "status"
+
+        status ${HA_SERVERS}
+        ;;
+
+    refresh-config | --refresh-config)
+        VOL=${1}
+
+        determine_servers "refresh-config"
+
+        refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS}
+        ;;
+
+    setup-ganesha-conf-files | --setup-ganesha-conf-files)
+
+        create_ganesha_conf_file ${1}
+        ;;
+
+    *)
+        # setup and teardown are not intended to be used by a
+        # casual user
+        usage
+        logger "Usage: ganesha-ha.sh add|delete|status"
+        ;;
+
+    esac
+
+    if (selinuxenabled) ;then
+     semanage boolean -m gluster_use_execmem --off
+    fi
+}
+
+main $*
diff --git a/extras/ganesha/scripts/generate-epoch.py b/extras/ganesha/scripts/generate-epoch.py
new file mode 100755
index 00000000000..77af014bab9
--- /dev/null
+++ b/extras/ganesha/scripts/generate-epoch.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+#
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+# Generates unique epoch value on each gluster node to be used by
+# nfs-ganesha service on that node.
+#
+# Configure 'EPOCH_EXEC' option to this script path in
+# '/etc/sysconfig/ganesha' file used by nfs-ganesha service.
+#
+# Construct epoch as follows -
+#        first 32-bit contains the now() time
+#        rest 32-bit value contains the local glusterd node uuid
+
+import time
+import binascii
+
+# Calculate the now() time into a 64-bit integer value
+def epoch_now():
+        epoch_time = int(time.mktime(time.localtime())) << 32
+        return epoch_time
+
+# Read glusterd UUID and extract first 32-bit of it
+def epoch_uuid():
+        file_name = '/var/lib/glusterd/glusterd.info'
+
+        for line in open(file_name):
+                if "UUID" in line:
+                        glusterd_uuid = line.split('=')[1].strip()
+
+        uuid_bin = binascii.unhexlify(glusterd_uuid.replace("-",""))
+
+        epoch_uuid = int(binascii.hexlify(uuid_bin), 32) & 0xFFFF0000
+        return epoch_uuid
+
+# Construct epoch as follows -
+#        first 32-bit contains the now() time
+#        rest 32-bit value contains the local glusterd node uuid
+epoch = (epoch_now() | epoch_uuid())
+print((str(epoch)))
+
+exit(0)
diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am
new file mode 100644
index 00000000000..09eff308ac4
--- /dev/null
+++ b/extras/geo-rep/Makefile.am
@@ -0,0 +1,16 @@
+scriptsdir = $(libexecdir)/glusterfs/scripts
+scripts_SCRIPTS =  gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \
+	slave-upgrade.sh schedule_georep.py
+
+scripts_PROGRAMS = gsync-sync-gfid
+gsync_sync_gfid_CFLAGS = $(GF_CFLAGS) -Wall -I$(top_srcdir)/libglusterfs/src
+gsync_sync_gfid_LDFLAGS = $(GF_LDFLAGS)
+gsync_sync_gfid_LDADD = $(GF_LDADD) $(top_builddir)/libglusterfs/src/libglusterfs.la
+gsync_sync_gfid_SOURCES = gsync-sync-gfid.c
+gsync_sync_gfid_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+EXTRA_DIST = gsync-sync-gfid.c gsync-upgrade.sh generate-gfid-file.sh \
+	get-gfid.sh slave-upgrade.sh schedule_georep.py.in
+
+CLEANFILES = schedule_georep.py
diff --git a/extras/geo-rep/generate-gfid-file.sh b/extras/geo-rep/generate-gfid-file.sh
new file mode 100644
index 00000000000..14f104b986d
--- /dev/null
+++ b/extras/geo-rep/generate-gfid-file.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+#Usage: generate-gfid-file.sh <master-volfile-server:master-volume> <path-to-get-gfid.sh> <output-file> [dirs-list-file]
+
+function get_gfids()
+{
+    GET_GFID_CMD=$1
+    OUTPUT_FILE=$2
+    DIR_PATH=$3
+    find "$DIR_PATH" -exec $GET_GFID_CMD {} \; >> $OUTPUT_FILE
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+
+    VOLFILE_SERVER=$1;
+    VOLUME=$2;
+    GFID_CMD=$3;
+    OUTPUT=$4;
+
+    T=$(mktemp -d -t ${0##*/}.XXXXXX);
+
+    glusterfs -s $VOLFILE_SERVER --volfile-id $VOLUME $T;
+
+    i=$(stat -c '%i' $T);
+
+    [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T";
+
+    cd $T;
+    rm -f $OUTPUT;
+    touch $OUTPUT;
+
+    if [ "$DIRS_FILE" = "." ]
+    then
+        get_gfids $GFID_CMD $OUTPUT "."
+    else
+        while read line
+        do
+            get_gfids $GFID_CMD $OUTPUT "$line"
+        done < $DIRS_FILE
+    fi;
+
+    cd -;
+
+    umount $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+
+function main()
+{
+    SLAVE=$1
+    GET_GFID_CMD=$2
+    OUTPUT=$3
+
+    VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    if [ "$#" -lt 4 ]
+    then
+        DIRS_FILE="."
+    else
+        DIRS_FILE=$4
+    fi
+    mount_client $VOLFILE_SERVER $VOLUME_NAME $GET_GFID_CMD $OUTPUT $DIRS_FILE
+}
+
+main "$@";
diff --git a/extras/geo-rep/get-gfid.sh b/extras/geo-rep/get-gfid.sh
new file mode 100755
index 00000000000..a4d609b0bc5
--- /dev/null
+++ b/extras/geo-rep/get-gfid.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+ATTR_STR=`getfattr -h $1 -n glusterfs.gfid.string`
+GLFS_PATH=`echo $ATTR_STR | sed -e 's/# file: \(.*\) glusterfs.gfid.string*/\1/g'`
+GFID=`echo $ATTR_STR | sed -e 's/.*glusterfs.gfid.string="\(.*\)"/\1/g'`
+
+echo "$GFID $GLFS_PATH"
diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c
new file mode 100644
index 00000000000..47dca0413e9
--- /dev/null
+++ b/extras/geo-rep/gsync-sync-gfid.c
@@ -0,0 +1,109 @@
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <libgen.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+
+#ifndef UUID_CANONICAL_FORM_LEN
+#define UUID_CANONICAL_FORM_LEN 36
+#endif
+
+#ifndef GF_FUSE_AUX_GFID_HEAL
+#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal"
+#endif
+
+#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN))
+
+int
+main(int argc, char *argv[])
+{
+    char *file = NULL;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    char *parent_dir = NULL;
+    char *gfid = NULL;
+    char *bname = NULL;
+    int ret = -1;
+    int len = 0;
+    FILE *fp = NULL;
+    char line[GLFS_LINE_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    void *blob = NULL;
+    void *tmp_blob = NULL;
+
+    if (argc != 2) {
+        /* each line in the file has the following format
+         * uuid-in-canonical-form path-relative-to-gluster-mount.
+         * Both uuid and relative path are from master mount.
+         */
+        fprintf(stderr, "usage: %s <file-of-paths-to-be-synced>\n", argv[0]);
+        goto out;
+    }
+
+    file = argv[1];
+
+    fp = fopen(file, "r");
+    if (fp == NULL) {
+        fprintf(stderr, "cannot open %s for reading (%s)\n", file,
+                strerror(errno));
+        goto out;
+    }
+
+    while (fgets(line, GLFS_LINE_MAX, fp) != NULL) {
+        tmp = line;
+        path = gfid = line;
+
+        path += UUID_CANONICAL_FORM_LEN + 1;
+
+        while (isspace(*path))
+            path++;
+
+        len = strlen(line);
+        if ((len < GLFS_LINE_MAX) && (line[len - 1] == '\n'))
+            line[len - 1] = '\0';
+
+        line[UUID_CANONICAL_FORM_LEN] = '\0';
+
+        tmp = strdup(path);
+        tmp1 = strdup(path);
+        parent_dir = dirname(tmp);
+        bname = basename(tmp1);
+
+        /* gfid + '\0' + bname + '\0' */
+        len = UUID_CANONICAL_FORM_LEN + 1 + strlen(bname) + 1;
+
+        blob = malloc(len);
+
+        memcpy(blob, gfid, UUID_CANONICAL_FORM_LEN);
+
+        tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1;
+
+        memcpy(tmp_blob, bname, strlen(bname));
+
+        ret = sys_lsetxattr(parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len, 0);
+        if (ret < 0) {
+            fprintf(stderr, "setxattr on %s/%s failed (%s)\n", parent_dir,
+                    bname, strerror(errno));
+        }
+        memset(line, 0, GLFS_LINE_MAX);
+
+        free(blob);
+        free(tmp);
+        free(tmp1);
+        blob = NULL;
+    }
+
+    ret = 0;
+out:
+    if (fp)
+        fclose(fp);
+    return ret;
+}
diff --git a/extras/geo-rep/gsync-upgrade.sh b/extras/geo-rep/gsync-upgrade.sh
new file mode 100644
index 00000000000..0f73a33884b
--- /dev/null
+++ b/extras/geo-rep/gsync-upgrade.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+#usage: gsync-upgrade.sh <slave-volfile-server:slave-volume> <gfid-file>
+#                        <path-to-gsync-sync-gfid> <ssh-identity-file>
+#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info.
+#                        slave-volfile-server defaults to localhost.
+#
+#<gfid-file>: a file containing paths and their associated gfids
+#            on master. The paths are relative to master mount point
+#            (not absolute). An example extract of <gfid-file> can be,
+#
+#            <extract>
+#            22114455-57c5-46e9-a783-c40f83a72b09 /dir
+#            25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file
+#            </extract>
+#
+#<ssh-identity-file>: file from which the identity (private key) for public key authentication is read.
+
+SLAVE_MOUNT='/tmp/glfs_slave'
+
+function SSH()
+{
+    HOST=$1
+    SSHKEY=$2
+
+    shift 2
+
+    ssh -qi $SSHKEY \
+        -oPasswordAuthentication=no \
+        -oStrictHostKeyChecking=no \
+        "$HOST" "$@";
+}
+
+function get_bricks()
+{
+    SSHKEY=$3
+
+    SSH $1 $SSHKEY "gluster volume info $2" | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g'
+}
+
+function cleanup_brick()
+{
+    HOST=$1
+    BRICK=$2
+    SSHKEY=$3
+
+    # TODO: write a C program to receive a list of files and does cleanup on
+    # them instead of spawning a new setfattr process for each file if
+    # performance is bad.
+    SSH -i $SSHKEY $HOST  "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \;"
+}
+
+function cleanup_slave()
+{
+    SSHKEY=$2
+
+    VOLFILE_SERVER=`echo $1 | sed -e 's/\(.*\):.*/\1/'`
+    VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'`
+
+    BRICKS=`get_bricks $VOLFILE_SERVER $VOLUME_NAME $SSHKEY`
+
+    for i in $BRICKS; do
+	HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'`
+	BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'`
+	cleanup_brick $HOST $BRICK $SSHKEY
+    done
+
+    SSH -i $SSHKEY $VOLFILE_SERVER "gluster --mode=script volume stop $VOLUME_NAME; gluster volume start $VOLUME_NAME";
+
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+    GFID_FILE=$3
+    SYNC_CMD=$4
+
+    T=$(mktemp -d -t ${0##*/}.XXXXXX);
+
+    glusterfs --aux-gfid-mount -s $1 --volfile-id $2 $T;
+
+    i=$(stat -c '%i' $T);
+
+    [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T";
+
+    cd $T;
+
+    $SYNC_CMD $GFID_FILE
+
+    cd -;
+
+    umount -l $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+function sync_gfids()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+
+    SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then
+        SLAVE_VOLFILE_SERVER="localhost"
+    fi
+
+    mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $3
+}
+
+function upgrade()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+    SSHKEY=$4
+
+    cleanup_slave $SLAVE $SSHKEY
+    sync_gfids $SLAVE $GFID_FILE $SYNC_CMD
+}
+
+upgrade "$@"
diff --git a/extras/geo-rep/schedule_georep.py.in b/extras/geo-rep/schedule_georep.py.in
new file mode 100644
index 00000000000..48b2b507060
--- /dev/null
+++ b/extras/geo-rep/schedule_georep.py.in
@@ -0,0 +1,492 @@
+#!/usr/bin/python3
+"""
+Schedule Geo-replication
+------------------------
+A tool to run Geo-replication when required. This can be used to
+schedule the Geo-replication to run once in a day using
+
+   # Run daily at 08:30pm
+   30 20 * * * root python /usr/share/glusterfs/scripts/schedule_georep.py \\
+      --no-color gv1 fvm1 gv2 >> /var/log/glusterfs/schedule_georep.log 2>&1
+
+This tool does the following,
+
+1. Stop Geo-replication if Started
+2. Start Geo-replication
+3. Set Checkpoint
+4. Check the Status and see Checkpoint is Complete.(LOOP)
+5. If checkpoint complete, Stop Geo-replication
+
+Usage:
+
+    python /usr/share/glusterfs/scripts/schedule_georep.py <MASTERVOL> \\
+         <SLAVEHOST> <SLAVEVOL>
+
+For example,
+
+    python /usr/share/glusterfs/scripts/schedule_georep.py gv1 fvm1 gv2
+
+"""
+import subprocess
+import time
+import xml.etree.cElementTree as etree
+import sys
+from contextlib import contextmanager
+import tempfile
+import os
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError
+cache_data = {}
+
+SESSION_MOUNT_LOG_FILE = ("/var/log/glusterfs/geo-replication"
+                          "/schedule_georep.mount.log")
+
+USE_CLI_COLOR = True
+mnt_list = []
+
+class GlusterBadXmlFormat(Exception):
+    """
+    Exception class for XML Parse Errors
+    """
+    pass
+
+
+def output_notok(msg, err="", exitcode=1):
+    if USE_CLI_COLOR:
+        out = "\033[31m[NOT OK]\033[0m {0}\n{1}\n"
+    else:
+        out = "[NOT OK] {0}\n{1}\n"
+    sys.stderr.write(out.format(msg, err))
+    sys.exit(exitcode)
+
+
+def output_warning(msg):
+    if USE_CLI_COLOR:
+        out = "\033[33m[  WARN]\033[0m {0}\n"
+    else:
+        out = "[  WARN] {0}\n"
+    sys.stderr.write(out.format(msg))
+
+
+def output_ok(msg):
+    if USE_CLI_COLOR:
+        out = "\033[32m[    OK]\033[0m {0}\n"
+    else:
+        out = "[    OK] {0}\n"
+    sys.stderr.write(out.format(msg))
+
+
+def execute(cmd, success_msg="", failure_msg="", exitcode=-1):
+    """
+    Generic wrapper to execute the CLI commands. Returns Output if success.
+    On success it can print message in stdout if specified.
+    On failure, exits after writing to stderr.
+    """
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    out, err = p.communicate()
+    if p.returncode == 0:
+        if success_msg:
+            output_ok(success_msg)
+        return out
+    else:
+        if exitcode == 0:
+            return
+        err_msg = err if err else out
+        output_notok(failure_msg, err=err_msg, exitcode=exitcode)
+
+
+def cache_output_with_args(func):
+    """
+    Decorator function to remember the output of any function
+    """
+    def wrapper(*args, **kwargs):
+        global cache_data
+        key = "_".join([func.func_name] + list(args))
+        if cache_data.get(key, None) is None:
+            cache_data[key] = func(*args, **kwargs)
+
+        return cache_data[key]
+    return wrapper
+
+
+def cleanup(hostname, volname, mnt):
+    """
+    Unmount the Volume and Remove the temporary directory
+    """
+    execute(["umount", "-l", mnt],
+            failure_msg="Unable to Unmount Gluster Volume "
+            "{0}:{1}(Mounted at {2})".format(hostname, volname, mnt))
+    execute(["rmdir", mnt],
+            failure_msg="Unable to Remove temp directory "
+            "{0}".format(mnt), exitcode=0)
+
+
+@contextmanager
+def glustermount(hostname, volname):
+    """
+    Context manager for Mounting Gluster Volume
+    Use as
+        with glustermount(HOSTNAME, VOLNAME) as MNT:
+            # Do your stuff
+    Automatically unmounts it in case of Exceptions/out of context
+    """
+    mnt = tempfile.mkdtemp(prefix="georepsetup_")
+    mnt_list.append(mnt)
+    execute(["@SBIN_DIR@/glusterfs",
+             "--volfile-server", hostname,
+             "--volfile-id", volname,
+             "-l", SESSION_MOUNT_LOG_FILE,
+             mnt],
+            failure_msg="Unable to Mount Gluster Volume "
+            "{0}:{1}".format(hostname, volname))
+    if os.path.ismount(mnt):
+        yield mnt
+    else:
+        output_notok("Unable to Mount Gluster Volume "
+                     "{0}:{1}".format(hostname, volname))
+    cleanup(hostname, volname, mnt)
+
+
+@cache_output_with_args
+def get_bricks(volname):
+    """
+    Returns Bricks list, caches the Bricks list for a volume once
+    parsed.
+    """
+    value = []
+    cmd = ["@SBIN_DIR@/gluster", "volume", "info", volname, "--xml"]
+    info = execute(cmd)
+    try:
+        tree = etree.fromstring(info)
+        volume_el = tree.find('volInfo/volumes/volume')
+        for b in volume_el.findall('bricks/brick'):
+            value.append({"name": b.find("name").text,
+                          "hostUuid": b.find("hostUuid").text})
+    except ParseError:
+        raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd))
+
+    return value
+
+
+def get_georep_status(mastervol, slave):
+    session_keys = set()
+    out = {}
+    cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication"]
+    if mastervol is not None:
+        cmd += [mastervol]
+        if slave:
+            cmd += [slave]
+
+    cmd += ["status", "--xml"]
+    info = execute(cmd)
+
+    try:
+        tree = etree.fromstring(info)
+        # Get All Sessions
+        for volume_el in tree.findall("geoRep/volume"):
+            sessions_el = volume_el.find("sessions")
+            # Master Volume name if multiple Volumes
+            mvol = volume_el.find("name").text
+
+            # For each session, collect the details
+            for session in sessions_el.findall("session"):
+                session_slave = "{0}:{1}".format(mvol, session.find(
+                    "session_slave").text)
+                session_keys.add(session_slave)
+                out[session_slave] = {}
+
+                for pair in session.findall('pair'):
+                    master_brick = "{0}:{1}".format(
+                        pair.find("master_node").text,
+                        pair.find("master_brick").text
+                    )
+
+                    out[session_slave][master_brick] = {
+                        "mastervol": mvol,
+                        "slavevol": pair.find("slave").text.split("::")[-1],
+                        "master_node": pair.find("master_node").text,
+                        "master_brick": pair.find("master_brick").text,
+                        "slave_user": pair.find("slave_user").text,
+                        "slave": pair.find("slave").text,
+                        "slave_node": pair.find("slave_node").text,
+                        "status": pair.find("status").text,
+                        "crawl_status": pair.find("crawl_status").text,
+                        "entry": pair.find("entry").text,
+                        "data": pair.find("data").text,
+                        "meta": pair.find("meta").text,
+                        "failures": pair.find("failures").text,
+                        "checkpoint_completed": pair.find(
+                            "checkpoint_completed").text,
+                        "master_node_uuid": pair.find("master_node_uuid").text,
+                        "last_synced": pair.find("last_synced").text,
+                        "checkpoint_time": pair.find("checkpoint_time").text,
+                        "checkpoint_completion_time":
+                        pair.find("checkpoint_completion_time").text
+                    }
+    except ParseError:
+        raise GlusterBadXmlFormat("Bad XML Format: %s" % " ".join(cmd))
+
+    return session_keys, out
+
+
+def get_offline_status(volname, brick, node_uuid, slave):
+    node, brick = brick.split(":")
+    if "@" not in slave:
+        slave_user = "root"
+    else:
+        slave_user, _ = slave.split("@")
+
+    return {
+        "mastervol": volname,
+        "slavevol": slave.split("::")[-1],
+        "master_node": node,
+        "master_brick": brick,
+        "slave_user": slave_user,
+        "slave": slave,
+        "slave_node": "N/A",
+        "status": "Offline",
+        "crawl_status": "N/A",
+        "entry": "N/A",
+        "data": "N/A",
+        "meta": "N/A",
+        "failures": "N/A",
+        "checkpoint_completed": "N/A",
+        "master_node_uuid": node_uuid,
+        "last_synced": "N/A",
+        "checkpoint_time": "N/A",
+        "checkpoint_completion_time": "N/A"
+    }
+
+
+def get(mastervol=None, slave=None):
+    """
+    This function gets list of Bricks of Master Volume and collects
+    respective Geo-rep status. Output will be always ordered as the
+    bricks list in Master Volume. If Geo-rep status is not available
+    for any brick then it updates OFFLINE status.
+    """
+    out = []
+    session_keys, gstatus = get_georep_status(mastervol, slave)
+
+    for session in session_keys:
+        mvol, _, slave = session.split(":", 2)
+        slave = slave.replace("ssh://", "")
+        master_bricks = get_bricks(mvol)
+        out.append([])
+        for brick in master_bricks:
+            bname = brick["name"]
+            if gstatus.get(session) and gstatus[session].get(bname, None):
+                out[-1].append(gstatus[session][bname])
+            else:
+                out[-1].append(
+                    get_offline_status(mvol, bname, brick["hostUuid"], slave))
+
+    return out
+
+
+def get_summary(mastervol, slave_url):
+    """
+    Wrapper function around Geo-rep Status and Gluster Volume Info
+    This combines the output from Bricks list and Geo-rep Status.
+    If a Master Brick node is down or Status is faulty then increments
+    the faulty counter. It also collects the checkpoint status from all
+    workers and compares with Number of Bricks.
+    """
+    down_rows = []
+    faulty_rows = []
+    out = []
+
+    status_data = get(mastervol, slave_url)
+
+    for session in status_data:
+        session_name = ""
+        summary = {
+            "active": 0,
+            "passive": 0,
+            "faulty": 0,
+            "initializing": 0,
+            "stopped": 0,
+            "created": 0,
+            "offline": 0,
+            "paused": 0,
+            "workers": 0,
+            "completed_checkpoints": 0,
+            "checkpoint": False,
+            "checkpoints_ok": False,
+            "ok": False
+        }
+
+        for row in session:
+            summary[row["status"].replace("...", "").lower()] += 1
+            summary["workers"] += 1
+            if row["checkpoint_completed"] == "Yes":
+                summary["completed_checkpoints"] += 1
+
+            session_name = "{0}=>{1}".format(
+                row["mastervol"],
+                row["slave"].replace("ssh://", "")
+            )
+
+            if row["status"] == "Faulty":
+                faulty_rows.append("{0}:{1}".format(row["master_node"],
+                                                    row["master_brick"]))
+
+            if row["status"] == "Offline":
+                down_rows.append("{0}:{1}".format(row["master_node"],
+                                                  row["master_brick"]))
+
+        if summary["active"] == summary["completed_checkpoints"] and \
+           summary["faulty"] == 0 and summary["offline"] == 0:
+            summary["checkpoints_ok"] = True
+
+        if summary["faulty"] == 0 and summary["offline"] == 0:
+            summary["ok"] = True
+
+        if session_name != "":
+                out.append([session_name, summary, faulty_rows, down_rows])
+
+    return out
+
+
+def touch_mount_root(mastervol):
+    # Create a Mount and Touch the Mount point root,
+    # Hack to make sure some event available after
+    # setting Checkpoint. Without this there is a chance of
+    # Checkpoint never completes.
+    with glustermount("localhost", mastervol) as mnt:
+        execute(["touch", mnt])
+
+
+def main(args):
+    turns = 1
+
+    # Stop Force
+    cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol,
+           "%s::%s" % (args.slave, args.slavevol), "stop", "force"]
+    execute(cmd)
+    output_ok("Stopped Geo-replication")
+
+    # Set Checkpoint to NOW
+    cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol,
+           "%s::%s" % (args.slave, args.slavevol), "config", "checkpoint",
+           "now"]
+    execute(cmd)
+    output_ok("Set Checkpoint")
+
+    # Start the Geo-replication
+    cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", args.mastervol,
+           "%s::%s" % (args.slave, args.slavevol), "start"]
+    execute(cmd)
+    output_ok("Started Geo-replication and watching Status for "
+              "Checkpoint completion")
+
+    start_time = int(time.time())
+    duration = 0
+
+    # Sleep till Geo-rep initializes
+    time.sleep(60)
+
+    touch_mount_root(args.mastervol)
+
+    slave_url = "{0}::{1}".format(args.slave, args.slavevol)
+
+    # Loop to Check the Geo-replication Status and Checkpoint
+    # If All Status OK and all Checkpoints complete,
+    # Stop the Geo-replication and Log the Completeness
+    while True:
+        session_summary = get_summary(args.mastervol,
+                                      slave_url)
+        if len(session_summary) == 0:
+            # If Status command fails with another transaction error
+            # or any other error. Gluster cmd still produces XML output
+            # with different message
+            output_warning("Unable to get Geo-replication Status")
+        else:
+            session_name, summary, faulty_rows, down_rows = session_summary[0]
+            chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \
+                           "NOT COMPLETE"
+            ok_status = "OK" if summary["ok"] else "NOT OK"
+
+            if summary["ok"]:
+                output_ok("All Checkpoints {1}, "
+                          "All status {2} (Turns {0:>3})".format(
+                              turns, chkpt_status, ok_status))
+            else:
+                output_warning("All Checkpoints {1}, "
+                               "All status {2} (Turns {0:>3})".format(
+                                   turns, chkpt_status, ok_status))
+
+                output_warning("Geo-rep workers Faulty/Offline, "
+                               "Faulty: {0} Offline: {1}".format(
+                                   repr(faulty_rows),
+                                   repr(down_rows)))
+
+            if summary["checkpoints_ok"]:
+                output_ok("Stopping Geo-replication session now")
+                cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
+                       args.mastervol,
+                       "%s::%s" % (args.slave, args.slavevol), "stop"]
+                execute(cmd)
+                break
+            else:
+                # If Checkpoint is not complete after a iteration means brick
+                # was down and came online now. SETATTR on mount is not
+                # recorded, So again issue touch on mount root So that
+                # Stime will increase and Checkpoint will complete.
+                touch_mount_root(args.mastervol)
+
+        # Increment the turns and Sleep for 10 sec
+        turns += 1
+        duration = int(time.time()) - start_time
+        if args.timeout > 0 and duration > (args.timeout * 60):
+            cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
+                   args.mastervol,
+                   "%s::%s" % (args.slave, args.slavevol), "stop", "force"]
+            execute(cmd)
+            output_notok("Timed out, Stopping Geo-replication("
+                         "Duration: {0}sec)".format(duration))
+
+        time.sleep(args.interval)
+
+    for mnt in mnt_list:
+        execute(["rmdir", mnt],
+                failure_msg="Unable to Remove temp directory "
+                "{0}".format(mnt), exitcode=0)
+
+if __name__ == "__main__":
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
+                            description=__doc__)
+    parser.add_argument("mastervol", help="Master Volume Name")
+    parser.add_argument("slave",
+                        help="Slave hostname "
+                        "(<username>@SLAVEHOST or SLAVEHOST)",
+                        metavar="SLAVE")
+    parser.add_argument("slavevol", help="Slave Volume Name")
+    parser.add_argument("--interval", help="Interval in Seconds. "
+                        "Wait time before each status check",
+                        type=int, default=10)
+    parser.add_argument("--timeout", help="Timeout in minutes. Script will "
+                        "stop Geo-replication if Checkpoint is not complete "
+                        "in the specified timeout time", type=int,
+                        default=0)
+    parser.add_argument("--no-color", help="Don't use Color in CLI output",
+                        action="store_true")
+    args = parser.parse_args()
+    if args.no_color:
+        USE_CLI_COLOR = False
+    try:
+        # Check for session existence
+        cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication",
+               args.mastervol, "%s::%s" % (args.slave, args.slavevol), "status"]
+        execute(cmd)
+        main(args)
+    except KeyboardInterrupt:
+        for mnt in mnt_list:
+            execute(["umount", "-l", mnt],
+                    failure_msg="Unable to Unmount Gluster Volume "
+                    "Mounted at {0}".format(mnt), exitcode=0)
+            execute(["rmdir", mnt],
+                    failure_msg="Unable to Remove temp directory "
+                    "{0}".format(mnt), exitcode=0)
+        output_notok("Exiting...")
diff --git a/extras/geo-rep/slave-upgrade.sh b/extras/geo-rep/slave-upgrade.sh
new file mode 100644
index 00000000000..3a37f8e3579
--- /dev/null
+++ b/extras/geo-rep/slave-upgrade.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+#usage: slave-upgrade.sh <volfile-server:volname> <gfid-file>
+#                        <path-to-gsync-sync-gfid>
+#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info.
+#                        slave-volfile-server defaults to localhost.
+#
+#<gfid-file>: a file containing paths and their associated gfids
+#            on master. The paths are relative to master mount point
+#            (not absolute). An example extract of <gfid-file> can be,
+#
+#            <extract>
+#            22114455-57c5-46e9-a783-c40f83a72b09 /dir
+#            25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file
+#            </extract>
+
+function get_bricks()
+{
+    gluster volume info $1 | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g'
+}
+
+function cleanup_brick()
+{
+    HOST=$1
+    BRICK=$2
+
+    # TODO: write a C program to receive a list of files and does cleanup on
+    # them instead of spawning a new setfattr process for each file if
+    # performance is bad.
+    ssh $HOST "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \; 2>/dev/null"
+}
+
+function cleanup_slave()
+{
+    VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'`
+
+    BRICKS=`get_bricks $VOLUME_NAME`
+
+    for i in $BRICKS; do
+	HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'`
+	BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'`
+	cleanup_brick $HOST $BRICK
+    done
+
+    # Now restart the volume
+    gluster --mode=script volume stop $VOLUME_NAME;
+    gluster volume start $VOLUME_NAME;
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+
+    VOLUME_NAME=$2;
+    GFID_FILE=$3
+    SYNC_CMD=$4
+
+    T=$(mktemp -d -t ${0##*/}.XXXXXX);
+
+    glusterfs --aux-gfid-mount -s $1 --volfile-id $VOLUME_NAME $T;
+
+    i=$(stat -c '%i' $T);
+
+    cd $T;
+
+    $SYNC_CMD $GFID_FILE
+
+    cd -;
+
+    umount $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+function sync_gfids()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+
+    SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then
+        SLAVE_VOLFILE_SERVER="localhost"
+    fi
+
+    mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $SYNC_CMD
+}
+
+function upgrade()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+
+    cleanup_slave $SLAVE
+
+    sync_gfids $SLAVE $GFID_FILE $SYNC_CMD
+}
+
+upgrade "$@"
diff --git a/extras/gfid-to-dirname.sh b/extras/gfid-to-dirname.sh
new file mode 100755
index 00000000000..fd359fab58a
--- /dev/null
+++ b/extras/gfid-to-dirname.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+function read_symlink()
+{
+        DOT_GLUSTERFS_PATH=$BRICK_PATH/.glusterfs
+        gfid_string=$1
+        symlink_path="$DOT_GLUSTERFS_PATH/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string"
+        #remove trailing '/'
+        symlink_path=${symlink_path%/}
+        linkname=$(readlink $symlink_path)
+        if [ $? -ne 0 ]; then
+                echo "readlink of $symlink_path returned an error." >&2
+                exit -1
+        fi
+        echo $linkname
+}
+
+main()
+{
+        if [ $# -lt 2 ] ;then
+                echo "Usage: $0 <brick-path> <gfid-string-of-directory>"
+                echo "Example: $0 /bricks/brick1 1b835012-1ae5-4f0d-9db4-64de574d891c"
+                exit -1
+        fi
+
+        BRICK_PATH=$1
+        name=$(read_symlink $2)
+        if [ $? -ne 0 ]; then
+                exit -1
+        fi
+
+        while [ ${name:12:36} != "00000000-0000-0000-0000-000000000001" ]
+        do
+                LOCATION=`basename $name`/$LOCATION
+                GFID_STRING=${name:12:36}
+                name=$(read_symlink $GFID_STRING)
+                if [ $? -ne 0 ]; then
+                        exit -1
+                fi
+        done
+
+        LOCATION=`basename $name`/$LOCATION
+        echo "Location of the directory corresponding to gfid:$2 is $BRICK_PATH/$LOCATION"
+}
+
+main "$@"
diff --git a/extras/git-branch-diff.py b/extras/git-branch-diff.py
new file mode 100755
index 00000000000..382513e069e
--- /dev/null
+++ b/extras/git-branch-diff.py
@@ -0,0 +1,285 @@
+#!/bin/python2
+
+"""
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
+
+"""
+  ABOUT:
+  This script helps in visualizing backported and missed commits between two
+  different branches, tags or commit ranges. In the list of missed commits,
+  it will help you identify patches which are posted for reviews on gerrit server.
+
+  USAGE:
+    $ ./extras/git-branch-diff.py --help
+    usage: git-branch-diff.py [-h] [-s SOURCE] -t TARGET [-a AUTHOR] [-p PATH]
+                              [-o OPTIONS]
+
+    git wrapper to diff local or remote branches/tags/commit-ranges
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -s SOURCE, --source SOURCE
+                            source pattern, it could be a branch, tag or a commit
+                            range
+      -t TARGET, --target TARGET
+                            target pattern, it could be a branch, tag or a commit
+                            range
+      -a AUTHOR, --author AUTHOR
+                            default: git config name/email, to provide multiple
+                            specify comma separated values
+      -p PATH, --path PATH  show source and target diff w.r.t given path, to
+                            provide multiple specify space in between them
+      -o OPTIONS, --options OPTIONS
+                            add other git options such as --after=<>, --before=<>
+                            etc. experts use;
+
+  SAMPLE EXECUTIONS:
+  $ ./extras/git-branch-diff.py -t origin/release-3.8
+
+  $ ./extras/git-branch-diff.py -s local_branch -t origin/release-3.7
+
+  $ ./extras/git-branch-diff.py -s 4517bf8..e66add8 -t origin/release-3.7
+  $ ./extras/git-branch-diff.py -s HEAD..c4efd39 -t origin/release-3.7
+
+  $ ./extras/git-branch-diff.py -t v3.7.11 --author="author@redhat.com"
+  $ ./extras/git-branch-diff.py -t v3.7.11 --author="authorX, authorY, authorZ"
+
+  $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="xlators/"
+  $ ./extras/git-branch-diff.py -t origin/release-3.8 --path="./xlators ./rpc"
+
+  $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="*"
+  $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="All"
+  $ ./extras/git-branch-diff.py -t origin/release-3.6 --author="Null"
+
+  $ ./extras/git-branch-diff.py -t v3.7.11 --options="--after=2015-03-01 \
+                                                      --before=2016-01-30"
+
+  DECLARATION:
+  While backporting commit to another branch only subject of the patch may
+  remain unchanged, all others such as commit message,  commit Id, change Id,
+  bug Id, may be changed. This script works by taking commit subject as the
+  key value for comparing two git branches, which can be local or remote.
+
+  Note: This script may ignore commits which have altered their commit subjects
+  while backporting patches. Also this script doesn't have any intelligence to
+  detect squashed commits.
+
+  AUTHOR:
+  Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+"""
+
+from __future__ import print_function
+import os
+import sys
+import argparse
+import commands
+import subprocess
+import requests
+
+class GitBranchDiff:
+    def __init__ (self):
+        " color symbols"
+        self.tick  = u'\033[1;32m[ \u2714 ]\033[0m'
+        self.cross = u'\033[1;31m[ \u2716 ]\033[0m'
+        self.green_set = u'\033[1;34m'
+        self.yello_set = u'\033[4;33m'
+        self.color_unset = '\033[0m'
+
+        self.parse_cmd_args()
+
+        " replace default values with actual values from command args"
+        self.g_author = self.argsdict['author']
+        self.s_pattern  = self.argsdict['source']
+        self.t_pattern  = self.argsdict['target']
+        self.r_path     = self.argsdict['path']
+        self.options    = ' '.join(self.argsdict['options'])
+
+        self.gerrit_server = "http://review.gluster.org"
+
+    def check_dir_exist (self, os_path):
+        " checks whether given path exist"
+        path_list = os_path.split()
+        for path in path_list:
+            if not os.path.exists(path):
+                raise argparse.ArgumentTypeError("'%s' path %s is not valid"
+                                                 %(os_path, path))
+        return os_path
+
+    def check_pattern_exist (self):
+        " defend to check given branch[s] exit"
+        status_sbr, op = commands.getstatusoutput('git log ' +
+                                                  self.s_pattern)
+        status_tbr, op = commands.getstatusoutput('git log ' +
+                                                  self.t_pattern)
+        if status_sbr != 0:
+            print("Error: --source=" + self.s_pattern + " doesn't exit\n")
+            self.parser.print_help()
+            exit(status_sbr)
+        elif status_tbr != 0:
+            print("Error: --target=" + self.t_pattern + " doesn't exit\n")
+            self.parser.print_help()
+            exit(status_tbr)
+
+    def check_author_exist (self):
+        " defend to check given author exist, format in case of multiple"
+        contrib_list = ['', '*', 'all', 'All', 'ALL', 'null', 'Null', 'NULL']
+        if self.g_author in contrib_list:
+            self.g_author = ""
+        else:
+            ide_list = self.g_author.split(',')
+            for ide in ide_list:
+                cmd4 = 'git log ' + self.s_pattern + ' --author=' + ide
+                c_list = subprocess.check_output(cmd4, shell = True)
+                if len(c_list) is 0:
+                    print("Error: --author=%s doesn't exit" %self.g_author)
+                    print("see '%s --help'" %__file__)
+                    exit(1)
+            if len(ide_list) > 1:
+                self.g_author = "\|".join(ide_list)
+
+    def connected_to_gerrit (self):
+        "check if gerrit server is reachable"
+        try:
+            r = requests.get(self.gerrit_server, timeout=3)
+            return True
+        except requests.Timeout as err:
+            " request timed out"
+            print("Warning: failed to get list of open review commits on " \
+                            "gerrit.\n" \
+                  "hint: Request timed out! gerrit server could possibly " \
+                  "slow ...\n")
+            return False
+        except requests.RequestException as err:
+            " handle other errors"
+            print("Warning: failed to get list of open review commits on " \
+                            "gerrit\n" \
+                  "hint: check with internet connection ...\n")
+            return False
+
+    def parse_cmd_args (self):
+        " command line parser"
+        author = subprocess.check_output('git config user.email',
+                                                  shell = True).rstrip('\n')
+        source = "remotes/origin/master"
+        options  = [' --pretty=format:"%h %s" ']
+        path = subprocess.check_output('git rev-parse --show-toplevel',
+                                            shell = True).rstrip('\n')
+        self.parser = argparse.ArgumentParser(description = 'git wrapper to '
+                                              'diff local or remote branches/'
+                                              'tags/commit-ranges')
+        self.parser.add_argument('-s',
+                                 '--source',
+                                 help = 'source pattern, it could be a branch,'
+                                        ' tag or a commit range',
+                                 default = source,
+                                 dest = 'source')
+        self.parser.add_argument('-t',
+                                 '--target',
+                                 help = 'target pattern, it could be a branch,'
+                                        ' tag or a commit range',
+                                 required = True,
+                                 dest = 'target')
+        self.parser.add_argument('-a',
+                                 '--author',
+                                 help = 'default: git config name/email, '
+                                        'to provide multiple specify comma'
+                                        ' separated values',
+                                 default = author,
+                                 dest = 'author')
+        self.parser.add_argument('-p',
+                                 '--path',
+                                 type = self.check_dir_exist,
+                                 help = 'show source and target diff w.r.t '
+                                        'given path, to provide multiple '
+                                        'specify space in between them',
+                                 default = path,
+                                 dest = 'path')
+        self.parser.add_argument('-o',
+                                 '--options',
+                                 help = 'add other git options such as '
+                                        '--after=<>, --before=<> etc. '
+                                        'experts use;',
+                                 default = options,
+                                 dest = 'options',
+                                 action='append')
+        self.argsdict = vars(self.parser.parse_args())
+
+    def print_output (self):
+        " display the result list"
+        print("\n------------------------------------------------------------\n")
+        print(self.tick + " Successfully Backported changes:")
+        print('      {' + 'from: ' + self.s_pattern + \
+              '  to: '+ self.t_pattern + '}\n')
+        for key, value in self.s_dict.items():
+            if value in self.t_dict.itervalues():
+                print("[%s%s%s] %s" %(self.yello_set,
+                                      key,
+                                      self.color_unset,
+                                      value))
+        print("\n------------------------------------------------------------\n")
+        print(self.cross + " Missing patches in " + self.t_pattern + ':\n')
+        if self.connected_to_gerrit():
+            cmd3 = "git review -r origin -l"
+            review_list = subprocess.check_output(cmd3, shell = True).split('\n')
+        else:
+            review_list = []
+
+        for key, value in self.s_dict.items():
+            if value not in self.t_dict.itervalues():
+                if any(value in s for s in review_list):
+                    print("[%s%s%s] %s %s(under review)%s" %(self.yello_set,
+                                                            key,
+                                                            self.color_unset,
+                                                            value,
+                                                            self.green_set,
+                                                            self.color_unset))
+                else:
+                    print("[%s%s%s] %s" %(self.yello_set,
+                                          key,
+                                          self.color_unset,
+                                          value))
+        print("\n------------------------------------------------------------\n")
+
+    def main (self):
+        self.check_pattern_exist()
+        self.check_author_exist()
+
+        " actual git commands"
+        cmd1 = 'git log' + self.options + ' ' + self.s_pattern + \
+               ' --author=\'' + self.g_author + '\' ' + self.r_path
+
+        " could be backported by anybody so --author doesn't apply here"
+        cmd2 = 'git log' + self.options + ' ' + self.t_pattern + \
+               ' ' + self.r_path
+
+        s_list = subprocess.check_output(cmd1, shell = True).split('\n')
+        t_list = subprocess.check_output(cmd2, shell = True)
+
+        if len(t_list) is 0:
+            print("No commits in the target: %s" %self.t_pattern)
+            print("see '%s --help'" %__file__)
+            exit()
+        else:
+            t_list = t_list.split('\n')
+
+        self.s_dict = dict()
+        self.t_dict = dict()
+
+        for item in s_list:
+            self.s_dict.update(dict([item.split(' ', 1)]))
+        for item in t_list:
+            self.t_dict.update(dict([item.split(' ', 1)]))
+
+        self.print_output()
+
+
+if __name__ == '__main__':
+    run = GitBranchDiff()
+    run.main()
diff --git a/extras/gluster-rsyslog-5.8.conf b/extras/gluster-rsyslog-5.8.conf
new file mode 100644
index 00000000000..2519999bcac
--- /dev/null
+++ b/extras/gluster-rsyslog-5.8.conf
@@ -0,0 +1,51 @@
+##### gluster.conf #####
+
+#
+## If you want to log every message to the log file instead of
+## intelligently suppressing repeated messages, set off to
+## RepeatedMsgReduction.  This change requires rsyslog restart
+## (eg. run 'service rsyslog restart')
+#
+#$RepeatedMsgReduction off
+$RepeatedMsgReduction on
+
+#
+## The mmcount module provides the capability to count log messages by 
+## severity or json property of given app-name.  The count value is added
+## into the log message as json property named '$msgid'
+#
+$ModLoad mmcount
+$mmcountKey gf_code # start counting value of gf_code
+
+$template Glusterfsd_dynLogFile,"/var/log/glusterfs/bricks/%app-name%.log"
+$template Gluster_dynLogFile,"/var/log/glusterfs/%app-name%.log"
+
+$template GLFS_Template,"%msgid%/%syslogfacility-text:::uppercase%/%syslogseverity-text:::uppercase% [%TIMESTAMP:::date-rfc3339%] %msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+
+#
+## Pass logs to mmcount if app-name is 'gluster'
+#
+if $app-name contains 'gluster' then :mmcount:
+
+if $app-name contains 'glusterfsd' then ?Glusterfsd_dynLogFile;GLFS_Template
+if $app-name contains 'gluster' and not ( $app-name contains 'glusterfsd' ) then ?Gluster_dynLogFile;GLFS_Template
+
+#
+## Sample configuration to send a email alert for every 50th mmcount
+#
+#$ModLoad ommail
+#$ActionMailSMTPServer smtp.example.com
+#$ActionMailFrom rsyslog@example.com
+#$ActionMailTo glusteradmin@example.com
+#$template mailSubject,"50th message of gf_code=9999 on %hostname%"
+#$template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'"
+#$ActionMailSubject mailSubject
+#$ActionExecOnlyOnceEveryInterval 30
+#if $app-name == 'glusterfsd' and $msgid != 0 and $msgid % 50 == 0 \
+#then :ommail:;RSYSLOG_SyslogProtocol23Format
+#
+
+#
+## discard logs where app-name is 'gluster' as we processed already
+#
+if $app-name contains 'gluster' then ~
diff --git a/extras/gluster-rsyslog-7.2.conf b/extras/gluster-rsyslog-7.2.conf
new file mode 100644
index 00000000000..8b2841543f4
--- /dev/null
+++ b/extras/gluster-rsyslog-7.2.conf
@@ -0,0 +1,76 @@
+##### gluster.conf #####
+#
+## If you want to log every message to the log file instead of
+## intelligently suppressing repeated messages, set off to
+## RepeatedMsgReduction.  This change requires rsyslog restart
+## (eg. run 'service rsyslog restart')
+#
+#$RepeatedMsgReduction off
+$RepeatedMsgReduction on
+
+$ModLoad mmjsonparse
+*.* :mmjsonparse:
+
+#
+## The mmcount module provides the capability to count log messages by 
+## severity or json property of given app-name.  The count value is added
+## into the log message as json property named 'mmcount'
+##
+## More info at http://www.rsyslog.com/doc/mmcount.html
+#
+#module(load="mmcount")
+#action(type="mmcount" appname="glusterd" key="!gf_code") # count each value of gf_code of appname glusterd
+#action(type="mmcount" appname="glusterfsd" key="!gf_code") # count each value of gf_code of appname glusterfsd
+#action(type="mmcount" appname="glusterfs" key="!gf_code") # count each value of gf_code of appname glusterfs
+
+template (name="Glusterfsd_dynLogFile" type="string" string="/var/log/glusterfs/bricks/%app-name%.log")
+template (name="Gluster_dynLogFile" type="string" string="/var/log/glusterfs/%app-name%.log")
+
+template(name="GLFS_template" type="list") {
+   property(name="$!mmcount")
+   constant(value="/")
+   property(name="syslogfacility-text" caseConversion="upper")
+   constant(value="/")
+   property(name="syslogseverity-text" caseConversion="upper")
+   constant(value=" ")
+   constant(value="[")
+   property(name="timereported" dateFormat="rfc3339")
+   constant(value="] ")
+   constant(value="[")
+   property(name="$!gf_code")
+   constant(value="] ")
+   constant(value="[")
+   property(name="$!gf_message")
+   constant(value="] ")
+   property(name="$!msg")
+   constant(value="\n")
+}
+
+if $app-name contains 'glusterfsd' then {
+   action(type="omfile"
+          DynaFile="Glusterfsd_dynLogFile"
+          Template="GLFS_template")
+   stop
+}
+
+if $app-name contains 'gluster' then {
+   action(type="omfile"
+          DynaFile="Gluster_dynLogFile"
+          Template="GLFS_template")
+   stop
+}
+
+#
+## send email for every 50th mmcount
+#$ModLoad ommail
+#if $app-name == 'glusterfsd' and $!mmcount <> 0 and $!mmcount % 50 == 0 then {
+#   $ActionMailSMTPServer smtp.example.com
+#   $ActionMailFrom rsyslog@example.com
+#   $ActionMailTo glusteradmin@example.com
+#   $template mailSubject,"50th message of gf_code=9999 on %hostname%"
+#   $template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'"
+#   $ActionMailSubject mailSubject
+#   $ActionExecOnlyOnceEveryInterval 30
+#   :ommail:;RSYSLOG_SyslogProtocol23Format
+#}
+#
diff --git a/extras/glusterd-sysconfig b/extras/glusterd-sysconfig
new file mode 100644
index 00000000000..8237c571104
--- /dev/null
+++ b/extras/glusterd-sysconfig
@@ -0,0 +1,6 @@
+## Set custom log file and log level (bellow are defaults)
+# LOG_FILE='/var/log/glusterfs/glusterd.log'
+# LOG_LEVEL='INFO'
+
+## Set custom options for glusterd
+# GLUSTERD_OPTIONS=''
diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in
new file mode 100644
index 00000000000..5d7bad0e4c8
--- /dev/null
+++ b/extras/glusterd.vol.in
@@ -0,0 +1,15 @@
+volume management
+    type mgmt/glusterd
+    option working-directory @GLUSTERD_WORKDIR@
+    option transport-type socket
+    option transport.socket.keepalive-time 10
+    option transport.socket.keepalive-interval 2
+    option transport.socket.read-fail-log off
+    option transport.socket.listen-port 24007
+    option ping-timeout 0
+    option event-threads 1
+#   option lock-timer 180
+#   option transport.address-family inet6
+#   option base-port 49152
+    option max-port  60999
+end-volume
diff --git a/extras/glusterfs-defrag.in b/extras/glusterfs-defrag.in
deleted file mode 100644
index 68a393f1c3a..00000000000
--- a/extras/glusterfs-defrag.in
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash
-
-# Please leave 'added_bricks' as empty if you want 100% defrag.
-# If you want to move data to newly added bricks, properly give
-# brick info as "<hostname>:<export-dir>" form (which was given
-# in 'gluster volume create' command)
-# More than one brick can be given with space inbetween.
-
-#
-# (c) 2010 Gluster Inc <http://www.gluster.com/>
-#
-
-set -e;
-
-#
-#added_bricks="node1:/gfs/export1"
-#
-added_bricks=""
-
-CP="cp"
-MV="mv"
-
-scan_dir()
-{
-    path=$1;
-    # run defrag on files first #
-    find "$path" -maxdepth 1 -type f -perm +01000 -exec $0 '{}' \;
-
-    for subdir in $(find "$path" -maxdepth 1 -type d | sed 1d); do
-        $0 "$subdir";
-    done
-}
-
-fix_xattr()
-{
-    path=$1;
-    getfattr -n trusted.distribute.fix.layout "$path" 2>/dev/null;
-}
-
-rsync_filename()
-{
-    path=$1
-    dir=$(dirname "$path");
-    file=$(basename "$path");
-
-    echo "$dir/.$file.zr$$";
-}
-
-relocate_file()
-{
-    path=$1;
-
-    # Make sure we don't 'defrag' valid file.
-    stat_info=$(stat -c '%a' "$path");
-    if [ $stat_info -lt 1000 ] ; then
-        return;
-    fi
-
-    size=$(stat -c '%s' "$path");
-
-    # If there are some entries in added_bricks, then check
-    # if the link file is present on those nodes, if not,
-    # set flag=1, so full defrag happens
-
-    flag=0;
-    for bricks in ${added_bricks}; do
-        linknode=$(getfattr --only-values -n trusted.distribute.linkinfo $path 2>/dev/null);
-        if [ -z $linknode ] ; then
-            return;
-        fi
-        current_brick=${linknode:0:${#bricks}};
-        if [ "${bricks}" == "${current_brick}" ]; then
-            flag=1;
-        fi
-    done
-
-    if [ -z ${added_bricks} ] ; then
-        flag=1;
-    fi
-
-    if [ $flag -ne 1 ]; then
-        return;
-    fi
-
-    tmp_path=$(rsync_filename "$path");
-
-    pre_mtime=$(stat -c '%Y' "$path");
-    $CP -a "$path" "$tmp_path";
-    post_mtime=$(stat -c '%Y' "$path");
-
-    if [ $pre_mtime = $post_mtime ]; then
-	chmod -t "$tmp_path";
-	$MV "$tmp_path" "$path";
-	echo "file '$path' relocated"
-    else
-	echo "file '$path' modified during defrag. skipping"
-	rm -f "$tmp_path";
-    fi
-}
-
-defrag_usage()
-{
-    echo "Usage: $0 <directory>"
-}
-
-main()
-{
-    path="$1";
-
-    if [ -z "$path" ]; then
-        defrag_usage;
-        return;
-    fi
-
-    if [ -d "$path" ]; then
-        fix_xattr "$path";
-	scan_dir "$path";
-    else
-	relocate_file "$@";
-    fi
-}
-
-main "$1"
diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate
new file mode 100644
index 00000000000..3e7ecf373a1
--- /dev/null
+++ b/extras/glusterfs-georep-logrotate
@@ -0,0 +1,61 @@
+/var/log/glusterfs/geo-replication/*/*.log {
+    sharedscripts
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
+    missingok
+    compress
+    delaycompress
+    notifempty
+    postrotate
+    for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do
+        /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true
+    done
+     endscript
+}
+
+
+/var/log/glusterfs/geo-replication-slaves/*.log {
+    sharedscripts
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
+    missingok
+    compress
+    delaycompress
+    notifempty
+    postrotate
+    for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do
+        /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true
+    done
+    endscript
+}
+
+
+/var/log/glusterfs/geo-replication-slaves/*/*.log {
+    sharedscripts
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
+    missingok
+    compress
+    delaycompress
+    notifempty
+    postrotate
+    for pid in `ps -aef | grep glusterfs | egrep "\-\-aux-gfid-mount" | awk '{print $2}'`; do
+        /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true
+    done
+    endscript
+}
diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py
new file mode 100755
index 00000000000..634576058d6
--- /dev/null
+++ b/extras/glusterfs-georep-upgrade.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+"""
+
+Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com>
+This file is part of GlusterFS.
+
+This file is licensed to you under your choice of the GNU Lesser
+General Public License, version 3 or any later version (LGPLv3 or
+later), or the GNU General Public License, version 2 (GPLv2), in all
+cases as published by the Free Software Foundation.
+
+"""
+
+import argparse
+import errno
+import os, sys
+import shutil
+from datetime import datetime
+
+def find_htime_path(brick_path):
+    dirs = []
+    htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime')
+    for file in os.listdir(htime_dir):
+        if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"):
+            dirs.append(os.path.join(htime_dir, file))
+        else:
+            raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file)))
+    return dirs
+
+def modify_htime_file(brick_path):
+    htime_file_path_list = find_htime_path(brick_path)
+
+    for htime_file_path in htime_file_path_list:
+        changelog_path = os.path.join(brick_path, '.glusterfs/changelogs')
+        temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file')
+        with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file:
+            #extract epoch times from htime file
+            paths = htime_file.read().split("\x00")
+
+            for pth in paths:
+                epoch_no = pth.split(".")[-1]
+                changelog = os.path.basename(pth)
+                #convert epoch time to year, month and day
+                if epoch_no != '':
+                    date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d"))
+                    #update paths in temp htime file
+                    temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog))
+                    #create directory in the format year/month/days
+                    path = os.path.join(changelog_path, date)
+
+                if changelog.startswith("CHANGELOG."):
+                    try:
+                        os.makedirs(path, mode = 0o600);
+                    except OSError as exc:
+                        if exc.errno == errno.EEXIST:
+                            pass
+                        else:
+                            raise
+
+                    #copy existing changelogs to new directory structure, delete old changelog files
+                    shutil.copyfile(pth, os.path.join(path, changelog))
+                    os.remove(pth)
+
+        #rename temp_htime_file with htime file
+        os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path))
+        os.rename(temp_htime_path, htime_file_path)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('brick_path', help="This upgrade script, which is to be run on\
+                         server side, takes brick path as the argument, \
+                         updates paths inside htime file and alters the directory structure \
+                         above the changelog files inorder to support new optimised format \
+                         of the directory structure as per \
+                         https://review.gluster.org/#/c/glusterfs/+/23733/")
+    args = parser.parse_args()
+    modify_htime_file(args.brick_path)
diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate
new file mode 100644
index 00000000000..6ba6ef18e9f
--- /dev/null
+++ b/extras/glusterfs-logrotate
@@ -0,0 +1,68 @@
+# Rotate client logs
+/var/log/glusterfs/*.log {
+  sharedscripts
+  weekly
+  maxsize 10M
+  minsize 100k
+
+# 6 months of logs are good enough
+  rotate 26
+
+  missingok
+  compress
+  delaycompress
+  notifempty
+  postrotate
+  /usr/bin/killall -HUP glusterfs > /dev/null 2>&1 || true
+  /usr/bin/killall -HUP glusterd > /dev/null 2>&1 || true
+  endscript
+}
+
+# Rotate server logs
+/var/log/glusterfs/bricks/*.log {
+  sharedscripts
+  weekly
+  maxsize 10M
+  minsize 100k
+
+# 6 months of logs are good enough
+  rotate 26
+
+  missingok
+  compress
+  delaycompress
+  notifempty
+  postrotate
+  /usr/bin/killall -HUP glusterfsd > /dev/null 2>&1 || true
+  endscript
+}
+
+/var/log/glusterfs/samples/*.samp {
+  daily
+  rotate 3
+  sharedscripts
+  missingok
+  compress
+  delaycompress
+}
+
+# Rotate snapd log
+/var/log/glusterfs/snaps/*/*.log {
+    sharedscripts
+    weekly
+    maxsize 10M
+    minsize 100k
+
+    # 6 months of logs are good enough
+    rotate 26
+
+    missingok
+    compress
+    delaycompress
+    notifempty
+    postrotate
+    for pid in `ps -aef | grep glusterfs | egrep "snapd" | awk '{print $2}'`; do
+        /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true
+    done
+    endscript
+}
diff --git a/extras/glusterfs-mode.el b/extras/glusterfs-mode.el
index 2f4b40c4e01..a9ed2335ab3 100644
--- a/extras/glusterfs-mode.el
+++ b/extras/glusterfs-mode.el
@@ -1,112 +1,113 @@
-;;; Copyright (C) 2007-2009 Gluster Inc. <http://www.gluster.com>
-;;;  
-;;; This program is free software; you can redistribute it and/or modify
-;;; it under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 2 of the License, or
-;;; (at your option) any later version.
-;;;  
-;;; This program is distributed in the hope that it will be useful,
-;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;;; GNU General Public License for more details.
-;;;  
-;;; You should have received a copy of the GNU General Public License
-;;; along with this program; if not, write to the Free Software
-;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-;;;  
-
-(defvar glusterfs-mode-hook nil)
-
-;; (defvar glusterfs-mode-map
-;;   (let ((glusterfs-mode-map (make-keymap)))
-;;     (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
-;;     glusterfs-mode-map)
-;;   "Keymap for WPDL major mode")
-
-(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
-
-(defconst glusterfs-font-lock-keywords-1
-  (list
-					; "cluster/{unify,afr,stripe}" 
-					; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
-					; "protocol/{client/server}"
-					; "features/{trash,posix-locks,fixed-id,filter}"
-					; "stroage/posix"
-					; "encryption/rot-13"
-					; "debug/trace"
-    '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
-"Additional Keywords to highlight in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-2
-  (append glusterfs-font-lock-keywords-1
-	  (list
-      ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host" 
-      ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
-      ; "limits.min-free-disk" "directory"
-	; TODO: add all the keys here.
-	   '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
-  "option keys in GlusterFS mode.")
-
-(defconst glusterfs-font-lock-keywords-3
-  (append glusterfs-font-lock-keywords-2
-	  (list
-					; "option" "volume" "end-volume" "subvolumes" "type"
-	   '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
-					;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
-  "Minimal highlighting expressions for GlusterFS mode.")
-
-
-(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
-  "Default highlighting expressions for GlusterFS mode.")
-
-(defvar glusterfs-mode-syntax-table
-  (let ((glusterfs-mode-syntax-table (make-syntax-table)))
-    (modify-syntax-entry ?\# "<"  glusterfs-mode-syntax-table)
-    (modify-syntax-entry ?* ". 23"  glusterfs-mode-syntax-table)
-    (modify-syntax-entry ?\n ">#"  glusterfs-mode-syntax-table)
-    glusterfs-mode-syntax-table)
-  "Syntax table for glusterfs-mode")
-
-;; TODO: add an indentation table
-
-(defun glusterfs-indent-line ()
-  "Indent current line as GlusterFS code"
-  (interactive)
-  (beginning-of-line)
-  (if (bobp)
-      (indent-line-to 0)   ; First line is always non-indented
-    (let ((not-indented t) cur-indent)
-      (if (looking-at "^[ \t]*volume\ ")
-	  (progn
-	    (save-excursion
-	      (forward-line -1)
-	      (setq not-indented nil)
-	      (setq cur-indent 0))))
-      (if (looking-at "^[ \t]*end-volume")
-	  (progn
-	    (save-excursion
-	      (forward-line -1)
-	      (setq cur-indent 0))
-	    (if (< cur-indent 0) ; We can't indent past the left margin
-		(setq cur-indent 0)))
-	(save-excursion
-	  (while not-indented ; Iterate backwards until we find an indentation hint
-	    (progn
-	      (setq cur-indent 2) ; Do the actual indenting
-	      (setq not-indented nil)))))
-      (if cur-indent
-	  (indent-line-to cur-indent)
-	(indent-line-to 0)))))
-
-(defun glusterfs-mode ()
-  (interactive)
-  (kill-all-local-variables)
-  ;; (use-local-map glusterfs-mode-map)
-  (set-syntax-table glusterfs-mode-syntax-table)
-  (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)  
-  (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
-  (setq major-mode 'glusterfs-mode)
-  (setq mode-name "GlusterFS")
-  (run-hooks 'glusterfs-mode-hook))
-
-(provide 'glusterfs-mode)
+;;; Copyright (C) 2007-2017 Red Hat, Inc. <http://www.redhat.com>
+;;; Copyright (C) 2007-2011 Gluster Inc. <http://www.gluster.com>
+;;;
+;;; This program is free software; you can redistribute it and/or
+;;; modify it under the terms of the GNU General Public License
+;;; as published by the Free Software Foundation; either version 2
+;;; of the License, or (at your option) any later version.
+;;;
+;;; This program is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program; if not, write to the Free Software
+;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+;;;
+
+(defvar glusterfs-mode-hook nil)
+
+;; (defvar glusterfs-mode-map
+;;   (let ((glusterfs-mode-map (make-keymap)))
+;;     (define-key glusterfs-mode-map "\C-j" 'newline-and-indent)
+;;     glusterfs-mode-map)
+;;   "Keymap for WPDL major mode")
+
+(add-to-list 'auto-mode-alist '("\\.vol\\'" . glusterfs-mode))
+
+(defconst glusterfs-font-lock-keywords-1
+  (list
+					; "cluster/{unify,afr,stripe}"
+					; "performance/{io-cache,io-threads,write-behind,read-ahead,stat-prefetch}"
+					; "protocol/{client/server}"
+					; "features/{trash,posix-locks,fixed-id,filter}"
+					; "storage/posix"
+					; "encryption/rot-13"
+					; "debug/trace"
+    '("\\<\\(cluster/\\(unify\\|afr\\|replicate\\|stripe\\|ha\\|dht\\|distribute\\)\\|\\performance/\\(io-\\(cache\\|threads\\)\\|write-behind\\|read-ahead\\|symlink-cache\\)\\|protocol/\\(server\\|client\\)\\|features/\\(trash\\|posix-locks\\|locks\\|path-converter\\|filter\\)\\|storage/\\(posix\\|bdb\\)\\|encryption/rot-13\\|debug/trace\\)\\>" . font-lock-keyword-face))
+"Additional Keywords to highlight in GlusterFS mode.")
+
+(defconst glusterfs-font-lock-keywords-2
+  (append glusterfs-font-lock-keywords-1
+	  (list
+      ; "replicate" "namespace" "scheduler" "remote-subvolume" "remote-host"
+      ; "auth.addr" "block-size" "remote-port" "listen-port" "transport-type"
+      ; "limits.min-free-disk" "directory"
+	; TODO: add all the keys here.
+	   '("\\<\\(inode-lru-limit\\|replicate\\|namespace\\|scheduler\\|username\\|password\\|allow\\|reject\\|block-size\\|listen-port\\|transport-type\\|transport-timeout\\|directory\\|page-size\\|page-count\\|aggregate-size\\|non-blocking-io\\|client-volume-filename\\|bind-address\\|self-heal\\|read-only-subvolumes\\|read-subvolume\\|thread-count\\|cache-size\\|window-size\\|force-revalidate-timeout\\|priority\\|include\\|exclude\\|remote-\\(host\\|subvolume\\|port\\)\\|auth.\\(addr\\|login\\)\\|limits.\\(min-disk-free\\|transaction-size\\|ib-verbs-\\(work-request-\\(send-\\|recv-\\(count\\|size\\)\\)\\|port\\|mtu\\|device-name\\)\\)\\)\ \\>" . font-lock-constant-face)))
+  "option keys in GlusterFS mode.")
+
+(defconst glusterfs-font-lock-keywords-3
+  (append glusterfs-font-lock-keywords-2
+	  (list
+					; "option" "volume" "end-volume" "subvolumes" "type"
+	   '("\\<\\(option\ \\|volume\ \\|subvolumes\ \\|type\ \\|end-volume\\)\\>" . font-lock-builtin-face)))
+					;'((regexp-opt (" option " "^volume " "^end-volume" "subvolumes " " type ") t) . font-lock-builtin-face))
+  "Minimal highlighting expressions for GlusterFS mode.")
+
+
+(defvar glusterfs-font-lock-keywords glusterfs-font-lock-keywords-3
+  "Default highlighting expressions for GlusterFS mode.")
+
+(defvar glusterfs-mode-syntax-table
+  (let ((glusterfs-mode-syntax-table (make-syntax-table)))
+    (modify-syntax-entry ?\# "<"  glusterfs-mode-syntax-table)
+    (modify-syntax-entry ?* ". 23"  glusterfs-mode-syntax-table)
+    (modify-syntax-entry ?\n ">#"  glusterfs-mode-syntax-table)
+    glusterfs-mode-syntax-table)
+  "Syntax table for glusterfs-mode")
+
+;; TODO: add an indentation table
+
+(defun glusterfs-indent-line ()
+  "Indent current line as GlusterFS code"
+  (interactive)
+  (beginning-of-line)
+  (if (bobp)
+      (indent-line-to 0)   ; First line is always non-indented
+    (let ((not-indented t) cur-indent)
+      (if (looking-at "^[ \t]*volume\ ")
+	  (progn
+	    (save-excursion
+	      (forward-line -1)
+	      (setq not-indented nil)
+	      (setq cur-indent 0))))
+      (if (looking-at "^[ \t]*end-volume")
+	  (progn
+	    (save-excursion
+	      (forward-line -1)
+	      (setq cur-indent 0))
+	    (if (< cur-indent 0) ; We can't indent past the left margin
+		(setq cur-indent 0)))
+	(save-excursion
+	  (while not-indented ; Iterate backwards until we find an indentation hint
+	    (progn
+	      (setq cur-indent 2) ; Do the actual indenting
+	      (setq not-indented nil)))))
+      (if cur-indent
+	  (indent-line-to cur-indent)
+	(indent-line-to 0)))))
+
+(defun glusterfs-mode ()
+  (interactive)
+  (kill-all-local-variables)
+  ;; (use-local-map glusterfs-mode-map)
+  (set-syntax-table glusterfs-mode-syntax-table)
+  (set (make-local-variable 'indent-line-function) 'glusterfs-indent-line)
+  (set (make-local-variable 'font-lock-defaults) '(glusterfs-font-lock-keywords))
+  (setq major-mode 'glusterfs-mode)
+  (setq mode-name "GlusterFS")
+  (run-hooks 'glusterfs-mode-hook))
+
+(provide 'glusterfs-mode)
diff --git a/extras/glusterfs.vim b/extras/glusterfs.vim
index eb5b2c98b80..899cc65511c 100644
--- a/extras/glusterfs.vim
+++ b/extras/glusterfs.vim
@@ -1,20 +1,11 @@
 " glusterfs.vim: GNU Vim Syntax file for GlusterFS .vol specification
-" Copyright (C) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-" This file is part of GlusterFS.
+"  Copyright (c) 2007777777Red Hat, Inc. <http://www.redhat.com>
+"  This file is part of GlusterFS.
 "
-" GlusterFS is free software; you can redistribute it and/or modify
-" it under the terms of the GNU General Public License as published
-" by the Free Software Foundation; either version 3 of the License,
-" or (at your option) any later version.
-"
-" GlusterFS is distributed in the hope that it will be useful, but
-" WITHOUT ANY WARRANTY; without even the implied warranty of
-" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-" General Public License for more details.
-"
-" You should have received a copy of the GNU General Public License
-" along with this program.  If not, see
-" <http://www.gnu.org/licenses/>.
+"  This file is licensed to you under your choice of the GNU Lesser
+"  General Public License, version 3 or any later version (LGPLv3 or
+"  later), or the GNU General Public License, version 2 (GPLv2), in all
+"  cases as published by the Free Software Foundation.
 "
 " Last Modified: Wed Aug  1 00:47:10 IST 2007
 " Version: 0.8 
diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py
index 1e0fde7f295..6341d007188 100644..100755
--- a/extras/gnfs-loganalyse.py
+++ b/extras/gnfs-loganalyse.py
@@ -1,29 +1,29 @@
 #!/bin/python
 """
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
 """
 
+from __future__ import print_function
 import os
 import string
 import sys
 
 
 class NFSRequest:
+        def requestIsEntryOp (self):
+                op = self.op
+                if op == "CREATE" or op == "LOOKUP" or op == "REMOVE" or op == "LINK" or op == "RENAME" or op == "MKDIR" or op == "RMDIR" or op == "SYMLINK" or op == "MKNOD":
+                        return 1
+                else:
+                        return 0
+
         def __init__ (self, logline, linecount):
                 self.calllinecount = 0
                 self.xid = ""
@@ -31,8 +31,13 @@ class NFSRequest:
                 self.opdata = ""
                 self.replydata = ""
                 self.replylinecount = 0
+                self.timestamp = ""
+                self.entryname = ""
+                self.gfid = ""
+                self.replygfid = ""
 
                 tokens = logline.strip ().split (" ")
+                self.timestamp = tokens[0] + " " + tokens[1]
                 if "XID:" not in tokens:
                         return None
 
@@ -47,6 +52,11 @@ class NFSRequest:
                 opidx = tokens.index ("args:")
                 self.op = tokens [opidx-1].strip (":")
                 self.opdata = " ".join(tokens [opidx+1:])
+                if self.requestIsEntryOp ():
+                        nameidx = tokens.index ("name:")
+                        self.entryname = tokens[nameidx + 1].strip (",")
+                gfididx = tokens.index ("gfid")
+                self.gfid = tokens[gfididx +1].strip(",")
 
 
         def getXID (self):
@@ -54,25 +64,56 @@ class NFSRequest:
 
         def setReply (self, logline, linecount):
                 tokens = logline.strip ().split (" ")
+                timestamp = tokens[0] + " " + tokens[1]
                 statidx = tokens.index ("NFS:")
-                self.replydata = " ".join (tokens [statidx+1:])
+                self.replydata = " TimeStamp: " + timestamp + " " + " ".join (tokens [statidx+1:])
                 self.replylinecount = linecount
+                if "gfid" in tokens:
+                        gfididx = tokens.index ("gfid")
+                        self.replygfid = tokens [gfididx + 1].strip(",")
 
         def dump (self):
-                print "ReqLine: " + str(self.calllinecount) + " XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata
+                print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata)
 
 class NFSLogAnalyzer:
 
-        def __init__ (self):
+        def __init__ (self, optn, trackfilename, tracknamefh, stats):
+                self.stats = stats
                 self.xid_request_map = {}
                 self.orphan_replies = {}
+                self.rqlist = []
                 self.CALL = 1
                 self.REPLY = 2
+                self.optn = optn
+                self.trackfilename = trackfilename
+                self.tracknamefh = tracknamefh
+                self.trackedfilehandles = []
 
         def handle_call_line (self, logline, linecount):
                 newreq = NFSRequest (logline, linecount)
                 xid = newreq.getXID ()
-                self.xid_request_map [xid] = newreq
+                if (self.optn == SYNTHESIZE):
+                        self.xid_request_map [xid] = newreq
+                        self.rqlist.append(newreq)
+                elif self.optn == TRACKFILENAME:
+                        if newreq.requestIsEntryOp():
+                                if newreq.entryname == self.trackfilename:
+                                        self.xid_request_map [xid] = newreq
+                                        self.rqlist.append(newreq)
+                                else:
+                                        del newreq
+                        elif self.tracknamefh == ENABLE_TRACKNAME_FH:
+                                if len (self.trackedfilehandles) > 0:
+                                        if newreq.gfid in self.trackedfilehandles:
+                                                self.xid_request_map [xid] = newreq
+                                                self.rqlist.append(newreq)
+                                        else:
+                                                del newreq
+                                else:
+                                        del newreq
+                        else:
+                                del newreq
+
 
         def handle_reply_line (self, logline, linecount):
                 tokens = logline.strip ().split (" ")
@@ -82,7 +123,10 @@ class NFSLogAnalyzer:
                 if xid not in self.xid_request_map.keys ():
                         self.orphan_replies [xid] = logline
                 else:
-                        self.xid_request_map [xid].setReply (logline, linecount)
+                        rq = self.xid_request_map [xid]
+                        rq.setReply (logline, linecount)
+                        if rq.requestIsEntryOp() and rq.entryname == self.trackfilename:
+                                self.trackedfilehandles.append (rq.replygfid)
 
         def analyzeLine (self, logline, linecount):
                 tokens = logline.strip ().split (" ")
@@ -102,43 +146,116 @@ class NFSLogAnalyzer:
                         self.handle_reply_line (logline, linecount)
 
         def getStats (self):
+                if self.stats == 0:
+                        return
                 rcount = len (self.xid_request_map.keys ())
                 orphancount = len (self.orphan_replies.keys ())
-                print "Requests: " + str(rcount) + ", Orphans: " + str(orphancount)
+                print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount))
 
         def dump (self):
                 self.getStats ()
-                for rq in self.xid_request_map.values ():
+                for rq in self.rqlist:
                         rq.dump ()
+                        del rq
 
-
+                self.rqlist = []
+                self.orphan_replies = {}
+                self.xid_request_map = {}
 
 
 linecount = 0
-la = NFSLogAnalyzer ()
+
+SYNTHESIZE = 1
+TRACKFILENAME = 2
+
+ENABLESTATS = 1
+DISABLESTATS = 0
+
+ENABLE_TRACKNAME_FH = 1
+DISABLE_TRACKNAME_FH = 0
 
 progmsgcount = 1000
-dumpinterval = 2000000
+dumpinterval = 200000
+operation = SYNTHESIZE
+stats = ENABLESTATS
+tracknamefh = DISABLE_TRACKNAME_FH
+trackfilename = ""
+
+"""
+Print the progress of the analysing operations every X number of lines read from
+the logs, where X is the argument provided to this option.
+
+Use this to print a status message every say 10000 lines processed or 100000
+lines processed to know how much longer the processing will go on for.
 
+
+USAGE: --progress <NUMLINES>
+"""
 if "--progress" in sys.argv:
         idx = sys.argv.index ("--progress")
         progmsgcount = int(sys.argv[idx+1])
 
-
+"""
+The replies for a NFS request can be separated by hundreds and even thousands
+of other NFS requests and replies. These can be spread over many hundreds and
+thousands of log lines. This script maintains a memory dict to map each request
+to its reply using the XID. Because this is in-core, there is a limit to the
+number of entries in the dict. At regular intervals, it dumps the mapped
+requests and the replies into the stdout. The requests whose replies were not
+found at the point of dumping are left as orphans, i.e. without info about the
+replies. Use this option to tune the number of lines to maximize the number of
+requests whose replies are found while balancing the dict size with memory
+on the machine. The default works fine for most cases.
+
+USAGE: --dump <NUMLINES>
+"""
 if "--dump" in sys.argv:
         idx = sys.argv.index ("--dump")
         dumpinterval = int(sys.argv[idx+1])
 
+"""
+The default operation of the script is to output all the requests mapped to
+their replies in a single line. This operation mode can be changed by this
+argument. It is used to print only those operations that were performed on the
+filename given as the argument to this option. Only those entry operations are
+printed which contain this filename.
+
+USAGE: --trackfilename <filename>
+"""
+if "--trackfilename" in sys.argv:
+        idx = sys.argv.index ("--trackfilename")
+        trackfilename = sys.argv[idx + 1]
+        operation = TRACKFILENAME
+
+"""
+At every dump interval, some stats are printed about the dumped lines.
+Use this option to disable printing that to avoid cluttering the
+output.
+"""
+if "--nostats" in sys.argv:
+        stats = DISABLESTATS
+
+"""
+While tracking a file using --trackfilename, we're only given those
+operations which contain the filename. This excludes a large number
+of operations which operate on that file using its filehandle instead of
+the filename. This option enables outputting those operations also. It
+tracks every single file handle that was ever seen in the log for a given
+filename.
+
+USAGE: --trackfilename
+"""
+if "--tracknamefh" in sys.argv:
+        tracknamefh = ENABLE_TRACKNAME_FH
+
+la = NFSLogAnalyzer (operation, trackfilename, tracknamefh, stats)
+
 for line in sys.stdin:
         linecount = linecount + 1
         if linecount % dumpinterval == 0:
                 sys.stderr.write ("Dumping data..\n")
                 la.dump ()
-                del la
-                la = NFSLogAnalyzer ()
 
         if linecount % progmsgcount == 0:
                 sys.stderr.write ("Integrating line: "+ str(linecount) + "\n")
         la.analyzeLine (line, linecount)
-
-la.dump ()
diff --git a/extras/group-db-workload b/extras/group-db-workload
new file mode 100644
index 00000000000..9334d6fb942
--- /dev/null
+++ b/extras/group-db-workload
@@ -0,0 +1,12 @@
+performance.open-behind=on
+performance.write-behind=off
+performance.stat-prefetch=off
+performance.quick-read=off
+performance.strict-o-direct=on
+performance.read-ahead=off
+performance.io-cache=off
+performance.readdir-ahead=off
+performance.client-io-threads=on
+server.event-threads=4
+client.event-threads=4
+performance.read-after-open=yes
diff --git a/extras/group-distributed-virt b/extras/group-distributed-virt
new file mode 100644
index 00000000000..a960b76c694
--- /dev/null
+++ b/extras/group-distributed-virt
@@ -0,0 +1,10 @@
+performance.quick-read=off
+performance.read-ahead=off
+performance.io-cache=off
+performance.low-prio-threads=32
+network.remote-dio=enable
+features.shard=on
+user.cifs=off
+client.event-threads=4
+server.event-threads=4
+performance.client-io-threads=on
diff --git a/extras/group-gluster-block b/extras/group-gluster-block
new file mode 100644
index 00000000000..1e398019e6b
--- /dev/null
+++ b/extras/group-gluster-block
@@ -0,0 +1,27 @@
+performance.quick-read=off
+performance.read-ahead=off
+performance.io-cache=off
+performance.stat-prefetch=off
+performance.open-behind=off
+performance.readdir-ahead=off
+performance.strict-o-direct=on
+performance.client-io-threads=on
+performance.io-thread-count=32
+performance.high-prio-threads=32
+performance.normal-prio-threads=32
+performance.low-prio-threads=32
+performance.least-prio-threads=4
+client.event-threads=8
+server.event-threads=8
+network.remote-dio=disable
+cluster.eager-lock=enable
+cluster.quorum-type=auto
+cluster.data-self-heal-algorithm=full
+cluster.locking-scheme=granular
+cluster.shd-max-threads=8
+cluster.shd-wait-qlength=10000
+features.shard=on
+features.shard-block-size=64MB
+user.cifs=off
+server.allow-insecure=on
+cluster.choose-local=off
diff --git a/extras/group-metadata-cache b/extras/group-metadata-cache
new file mode 100644
index 00000000000..b890b288fc7
--- /dev/null
+++ b/extras/group-metadata-cache
@@ -0,0 +1,6 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.stat-prefetch=on
+performance.cache-invalidation=on
+performance.md-cache-timeout=600
+network.inode-lru-limit=200000
diff --git a/extras/group-nl-cache b/extras/group-nl-cache
new file mode 100644
index 00000000000..897807e8933
--- /dev/null
+++ b/extras/group-nl-cache
@@ -0,0 +1,5 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.nl-cache=on
+performance.nl-cache-timeout=600
+network.inode-lru-limit=200000
diff --git a/extras/group-samba b/extras/group-samba
new file mode 100644
index 00000000000..eeee6e06031
--- /dev/null
+++ b/extras/group-samba
@@ -0,0 +1,11 @@
+features.cache-invalidation=on
+features.cache-invalidation-timeout=600
+performance.cache-samba-metadata=on
+performance.stat-prefetch=on
+performance.cache-invalidation=on
+performance.md-cache-timeout=600
+network.inode-lru-limit=200000
+performance.nl-cache=on
+performance.nl-cache-timeout=600
+performance.readdir-ahead=on
+performance.parallel-readdir=on
diff --git a/extras/group-virt.example b/extras/group-virt.example
new file mode 100644
index 00000000000..cc37c98a25c
--- /dev/null
+++ b/extras/group-virt.example
@@ -0,0 +1,24 @@
+performance.quick-read=off
+performance.read-ahead=off
+performance.io-cache=off
+performance.low-prio-threads=32
+network.remote-dio=disable
+performance.strict-o-direct=on
+cluster.eager-lock=enable
+cluster.quorum-type=auto
+cluster.server-quorum-type=server
+cluster.data-self-heal-algorithm=full
+cluster.locking-scheme=granular
+cluster.shd-max-threads=8
+cluster.shd-wait-qlength=10000
+features.shard=on
+user.cifs=off
+cluster.choose-local=off
+client.event-threads=4
+server.event-threads=4
+performance.client-io-threads=on
+network.ping-timeout=20
+server.tcp-user-timeout=20
+server.keepalive-time=10
+server.keepalive-interval=2
+server.keepalive-count=5
diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am
new file mode 100644
index 00000000000..26059d7dbb9
--- /dev/null
+++ b/extras/hook-scripts/Makefile.am
@@ -0,0 +1,7 @@
+EXTRA_DIST = S40ufo-stop.py S56glusterd-geo-rep-create-post.sh
+SUBDIRS = add-brick create delete set start stop reset
+
+scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/gsync-create/post/
+if USE_GEOREP
+scripts_SCRIPTS = S56glusterd-geo-rep-create-post.sh
+endif
diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py
new file mode 100755
index 00000000000..2c79eb1d54a
--- /dev/null
+++ b/extras/hook-scripts/S40ufo-stop.py
@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+import os
+from optparse import OptionParser
+
+if __name__ == '__main__':
+    # check if swift is installed
+    try:
+        from gluster.swift.common.Glusterfs import get_mnt_point, unmount
+    except ImportError:
+        import sys
+        sys.exit("Openstack Swift does not appear to be installed properly");
+
+    op = OptionParser(usage="%prog [options...]")
+    op.add_option('--volname', dest='vol', type=str)
+    op.add_option('--last', dest='last', type=str)
+    (opts, args) = op.parse_args()
+
+
+    mnt_point = get_mnt_point(opts.vol)
+    if mnt_point:
+        unmount(mnt_point)
+    else:
+        sys.exit("get_mnt_point returned none for mount point")
diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
new file mode 100755
index 00000000000..7d6052315bb
--- /dev/null
+++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+#key_val_pair is the arguments passed to the script in the form of
+#key value pair
+
+key_val_pair1=`echo $2 | cut -d ',' -f 1`
+key_val_pair2=`echo $2 | cut -d ',' -f 2`
+key_val_pair3=`echo $2 | cut -d ',' -f 3`
+key_val_pair4=`echo $2 | cut -d ',' -f 4`
+key_val_pair5=`echo $2 | cut -d ',' -f 5`
+key_val_pair6=`echo $2 | cut -d ',' -f 6`
+
+mastervol=`echo $1 | cut -d '=' -f 2`
+if [ "$mastervol" == "" ]; then
+    exit;
+fi
+
+key=`echo $key_val_pair1 | cut -d '=' -f 1`
+val=`echo $key_val_pair1 | cut -d '=' -f 2`
+if [ "$key" != "is_push_pem" ]; then
+    exit;
+fi
+if [ "$val" != '1' ]; then
+    exit;
+fi
+
+key=`echo $key_val_pair2 | cut -d '=' -f 1`
+val=`echo $key_val_pair2 | cut -d '=' -f 2`
+if [ "$key" != "pub_file" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+
+pub_file=`echo $val`
+pub_file_bname="$(basename $pub_file)"
+pub_file_dname="$(dirname $pub_file)"
+pub_file_tmp=`echo $val`_tmp
+
+key=`echo $key_val_pair3 | cut -d '=' -f 1`
+val=`echo $key_val_pair3 | cut -d '=' -f 2`
+if [ "$key" != "slave_user" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+slave_user=`echo $val`
+
+key=`echo $key_val_pair4 | cut -d '=' -f 1`
+val=`echo $key_val_pair4 | cut -d '=' -f 2`
+if [ "$key" != "slave_ip" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+slave_ip=`echo $val`
+
+key=`echo $key_val_pair5 | cut -d '=' -f 1`
+val=`echo $key_val_pair5 | cut -d '=' -f 2`
+if [ "$key" != "slave_vol" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+slavevol=`echo $val`
+
+key=`echo $key_val_pair6 | cut -d '=' -f 1`
+val=`echo $key_val_pair6 | cut -d '=' -f 2`
+if [ "$key" != "ssh_port" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+SSH_PORT=`echo $val`
+SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no"
+
+if [ -f $pub_file ]; then
+    # For a non-root user copy the pub file to the user's home directory
+    # For a root user copy the pub files to priv_dir->geo-rep.
+    if [ "$slave_user" != "root" ]; then
+        slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"`
+        scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp
+        ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub"
+    else
+        if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+            scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on"
+        else
+            scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null"
+            ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on"
+        fi
+    fi
+fi
diff --git a/extras/hook-scripts/add-brick/Makefile.am b/extras/hook-scripts/add-brick/Makefile.am
new file mode 100644
index 00000000000..6e2701e909e
--- /dev/null
+++ b/extras/hook-scripts/add-brick/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = post pre
+CLEANFILES =
diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am
new file mode 100644
index 00000000000..9b236df096d
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh
+
+hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/post/
+if WITH_SERVER
+hook_SCRIPTS = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh
+endif
diff --git a/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh
new file mode 100755
index 00000000000..4a17c993a77
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/add-brick/post
+#
+# Add an SELinux file context for each brick using the glusterd_brick_t type.
+# This ensures that the brick is relabeled correctly on an SELinux restart or
+# restore. Subsequently, run a restore on the brick path to set the selinux
+# labels.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+VOL=
+
+parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+        ;;
+      --gd-workdir)
+          shift
+          GLUSTERD_WORKDIR=$1
+          ;;
+      --version)
+          shift
+          ;;
+      --volume-op)
+          shift
+          ;;
+      *)
+          shift
+          break
+          ;;
+    esac
+    shift
+  done
+}
+
+set_brick_labels()
+{
+  local volname="${1}"
+  local fctx
+  local list=()
+
+  fctx="$(semanage fcontext --list -C)"
+
+  # wait for new brick path to be updated under
+  # ${GLUSTERD_WORKDIR}/vols/${volname}/bricks/
+  sleep 5
+
+  # grab the path for each local brick
+  brickpath="${GLUSTERD_WORKDIR}/vols/${volname}/bricks/"
+  brickdirs=$(
+    find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | \
+    sort -u
+  )
+
+  # create a list of bricks for which custom SELinux
+  # label doesn't exist
+  for b in ${brickdirs}; do
+    pattern="${b}(/.*)?"
+    echo "${fctx}" | grep "^${pattern}\s" >/dev/null
+    if [[ $? -ne 0 ]]; then
+      list+=("${pattern}")
+    fi
+  done
+
+  # Add a file context for each brick path in the list and associate with the
+  # glusterd_brick_t SELinux type.
+  for p in ${list[@]}
+  do
+    semanage fcontext --add -t glusterd_brick_t -r s0 "${p}"
+  done
+
+  # Set the labels for which SELinux label was added above
+  for b in ${brickdirs}
+  do
+    echo "${list[@]}" | grep "${b}" >/dev/null
+    if [[ $? -eq 0 ]]; then
+      restorecon -R "${b}"
+    fi
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+set_brick_labels "${VOL}"
+
+exit 0
diff --git a/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh
new file mode 100755
index 00000000000..1a6923ee7aa
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+##---------------------------------------------------------------------------
+## This script runs the self-heal of the directories which are expected to
+## be present as they are mounted as subdirectory mounts.
+##---------------------------------------------------------------------------
+
+MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`;
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="add-brick-create-subdir"
+VOL_NAME=test
+GLUSTERD_WORKDIR="/var/lib/glusterd"
+
+cleanup_mountpoint ()
+{
+        umount -f $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+
+        rmdir $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+eval set -- "$ARGS"
+
+while true;
+do
+    case $1 in
+        --volname)
+            shift
+            VOL_NAME=$1
+            ;;
+        --gd-workdir)
+            shift
+            GLUSTERD_WORKDIR=$1
+            ;;
+	--version)
+	    shift
+	    ;;
+	--volume-op)
+	    shift
+	    ;;
+	*)
+	    shift
+	    break
+	    ;;
+    esac
+    shift
+done
+
+## See if we have any subdirs to be healed before going further
+subdirs=$(grep 'auth.allow' ${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info | cut -f2 -d'=' | tr ',' '\n' | cut -f1 -d'(');
+
+if [ -z ${subdirs} ]; then
+    rmdir $MOUNT_DIR;
+    exit 0;
+fi
+
+##----------------------------------------
+## Mount the volume in temp directory.
+## -----------------------------------
+glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-50 $MOUNT_DIR;
+if [ 0 -ne $? ]
+then
+    exit $?;
+fi
+
+## -----------------------------------
+# Do the 'stat' on all the directory for now. Ideal fix is to look at subdir
+# list from 'auth.allow' option and only stat them.
+for subdir in ${subdirs}
+do
+    stat ${MOUNT_DIR}/${subdir} > /dev/null;
+done
+
+## Clean up and exit
+cleanup_mountpoint;
diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
new file mode 100755
index 00000000000..ca17a903549
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
@@ -0,0 +1,145 @@
+#!/bin/sh
+
+##---------------------------------------------------------------------------
+## This script updates the 'limit-set' xattr on the newly added node. Please
+## refer hook-scripts/add-brick/pre/S28Quota-root-xattr-heal.sh for the complete
+## description.
+## Do the following only if limit configured on root.
+## 1. Do an auxiliary mount.
+## 2. Get 'limit-set' xattr on root
+## 3. Set xattrs with the same value on the root.
+## 4. Disable itself
+##---------------------------------------------------------------------------
+
+QUOTA_LIMIT_XATTR="trusted.glusterfs.quota.limit-set"
+QUOTA_OBJECT_LIMIT_XATTR="trusted.glusterfs.quota.limit-objects"
+MOUNT_DIR=$(mktemp -d -t "${0##*/}.XXXXXX");
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="Quota-xattr-heal-add-brick"
+VOL_NAME=
+VERSION=
+VOLUME_OP=
+GLUSTERD_WORKDIR=
+ENABLED_NAME_PREFIX="S28"
+ENABLED_NAME="Quota-root-xattr-heal.sh"
+
+THIS_SCRIPT=$(echo "${0}" | awk -F'/' '{print $NF}')
+
+cleanup_mountpoint ()
+{
+
+  if umount -f "${MOUNT_DIR}"; then
+    return $?
+  fi
+
+  if rmdir "${MOUNT_DIR}"; then
+    return $?
+  fi
+}
+
+disable_and_exit ()
+{
+  if [ -e "${ENABLED_STATE}" ]
+  then
+    unlink "${ENABLED_STATE}";
+    exit $?
+  fi
+
+  exit 0
+}
+
+get_and_set_xattr ()
+{
+  XATTR=$1
+
+  VALUE=$(getfattr -n "${XATTR}" -e hex --absolute-names "${MOUNT_DIR}" 2>&1)
+  RET=$?
+  if [ 0 -eq ${RET} ]; then
+    VALUE=$(echo "${VALUE}" | grep "${XATTR}" | awk -F'=' '{print $NF}')
+    setfattr -n "${XATTR}" -v "${VALUE}" "${MOUNT_DIR}";
+    RET=$?
+  else
+    if echo "${VALUE}" | grep -iq "No such attribute" ; then
+      RET=0
+    fi
+  fi
+
+  return ${RET};
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+eval set -- "$ARGS"
+
+while true;
+do
+  case $1 in
+    --volname)
+      shift
+      VOL_NAME=$1
+    ;;
+    --version)
+      shift
+      VERSION=$1
+    ;;
+    --gd-workdir)
+      shift
+      GLUSTERD_WORKDIR=$1
+    ;;
+    --volume-op)
+      shift
+      VOLUME_OP=$1
+    ;;
+    *)
+      shift
+      break
+    ;;
+  esac
+  shift
+done
+##----------------------------------------
+
+# Avoid long lines
+ENABLED_STATE_1="${GLUSTERD_WORKDIR}/hooks/${VERSION}/${VOLUME_OP}/"
+ENABLED_STATE_2="post/${ENABLED_NAME_PREFIX}${VOL_NAME}-${ENABLED_NAME}"
+ENABLED_STATE="${ENABLED_STATE_1}${ENABLED_STATE_2}"
+
+if [ "${THIS_SCRIPT}" != *"${VOL_NAME}"* ]; then
+  exit 0
+fi
+
+## Is quota enabled?
+FLAG=$(grep "^features.quota=" "${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info" \
+| awk -F'=' '{print $NF}');
+if [ "${FLAG}" != "on" ]
+then
+  disable_and_exit
+fi
+
+## -----------------------------------
+## Mount the volume in temp directory.
+## -----------------------------------
+# Avoid long lines
+CMD_1="glusterfs -s localhost"
+CMD_2="--volfile-id=${VOL_NAME} client-pid=-42 ${MOUNT_DIR}"
+CMD="${CMD_1}${CMD_2}"
+
+if ${CMD}
+then
+  exit $?;
+fi
+## -----------------------------------
+
+RET1=$(get_and_set_xattr "${QUOTA_LIMIT_XATTR}")
+RET2=$(get_and_set_xattr "${QUOTA_OBJECT_LIMIT_XATTR}")
+
+## Clean up and exit
+cleanup_mountpoint;
+
+if [ "${RET1}" -ne 0 ] || [ "${RET2}" -ne 0 ]; then
+  exit 1
+fi
+
+disable_and_exit;
diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am
new file mode 100644
index 00000000000..3288581aa57
--- /dev/null
+++ b/extras/hook-scripts/add-brick/pre/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh
+
+hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/pre/
+if WITH_SERVER
+hook_SCRIPTS = S28Quota-enable-root-xattr-heal.sh
+endif
diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
new file mode 100755
index 00000000000..27e85231f45
--- /dev/null
+++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
@@ -0,0 +1,101 @@
+#!/bin/sh
+
+###############################################################################
+## ----------------------------------------------------------------------------
+## The scripts
+## I.   add-brick/pre/S28Quota-root-xattr-heal.sh (itself)
+## II.  add-brick/post/disabled-root-xattr-heal.sh AND
+## collectively archieves the job of healing the 'limit-set' xattr upon
+## add-brick to the gluster volume.
+##
+## This script is the 'controlling' script. Upon add-brick this script enables
+## the corresponding script based on the status of the volume.
+## If volume is started - enable add-brick/post script
+## else                 - enable start/post script.
+##
+## The enabling and disabling of a script is based on the glusterd's logic,
+## that it only runs the scripts which starts its name with 'S'. So,
+## Enable - symlink the file to 'S'*.
+## Disable- unlink symlink
+## ----------------------------------------------------------------------------
+###############################################################################
+
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="Quota-xattr-heal-add-brick-pre"
+VOL_NAME=
+GLUSTERD_WORKDIR=
+VOLUME_OP=
+VERSION=
+ENABLED_NAME_PREFIX="S28"
+ENABLED_NAME="Quota-root-xattr-heal.sh"
+DISABLED_NAME="disabled-quota-root-xattr-heal.sh"
+
+activate ()
+{
+        ln -sf $DISABLED_STATE $1;
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
+eval set -- "$ARGS"
+
+while true;
+do
+    case $1 in
+        --volname)
+            shift
+            VOL_NAME=$1
+            ;;
+        --gd-workdir)
+            shift
+            GLUSTERD_WORKDIR=$1
+            ;;
+        --volume-op)
+            shift
+            VOLUME_OP=$1
+            ;;
+        --version)
+            shift
+            VERSION=$1
+            ;;
+        *)
+            shift
+            break
+            ;;
+    esac
+    shift
+done
+##----------------------------------------
+
+DISABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME"
+ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/""$ENABLED_NAME_PREFIX$VOL_NAME""-""$ENABLED_NAME"
+ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/""$ENABLED_NAME_PREFIX""$VOL_NAME""-""$ENABLED_NAME";
+
+## Why to proceed if the required script itself is not present?
+ls $DISABLED_STATE;
+if [ 0 -ne $? ]
+then
+        exit $?;
+fi
+
+## Is quota enabled?
+FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^features.quota=" \
+      | awk -F'=' '{print $NF}'`;
+if [ "$FLAG" != "on" ]
+then
+        exit $EXIT_SUCCESS;
+fi
+
+## Is volume started?
+FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^status=" \
+      | awk -F'=' '{print $NF}'`;
+if [ "$FLAG" != "1" ]
+then
+        activate $ENABLED_STATE_START;
+        exit $?
+fi
+
+activate $ENABLED_STATE_ADD_BRICK;
+exit $?
diff --git a/extras/hook-scripts/create/Makefile.am b/extras/hook-scripts/create/Makefile.am
new file mode 100644
index 00000000000..b083a9145d6
--- /dev/null
+++ b/extras/hook-scripts/create/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = post
diff --git a/extras/hook-scripts/create/post/Makefile.am b/extras/hook-scripts/create/post/Makefile.am
new file mode 100644
index 00000000000..fd1892e9589
--- /dev/null
+++ b/extras/hook-scripts/create/post/Makefile.am
@@ -0,0 +1,8 @@
+EXTRA_DIST = S10selinux-label-brick.sh
+
+scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/create/post/
+if WITH_SERVER
+if USE_SELINUX
+scripts_SCRIPTS = S10selinux-label-brick.sh
+endif
+endif
diff --git a/extras/hook-scripts/create/post/S10selinux-label-brick.sh b/extras/hook-scripts/create/post/S10selinux-label-brick.sh
new file mode 100755
index 00000000000..f9b4b1a57e3
--- /dev/null
+++ b/extras/hook-scripts/create/post/S10selinux-label-brick.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/create/post
+#
+# Add an SELinux file context for each brick using the glusterd_brick_t type.
+# This ensures that the brick is relabeled correctly on an SELinux restart or
+# restore. Subsequently, run a restore on the brick path to set the selinux
+# labels.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:"
+VOL=
+
+parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+      ;;
+      *)
+        shift
+        break
+      ;;
+    esac
+    shift
+  done
+}
+
+set_brick_labels()
+{
+  volname="${1}"
+
+  # grab the path for each local brick
+  brickpath="/var/lib/glusterd/vols/${volname}/bricks/"
+  brickdirs=$(
+    find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | \
+    sort -u
+  )
+
+  for b in ${brickdirs}; do
+    # Add a file context for each brick path and associate with the
+    # glusterd_brick_t SELinux type.
+    pattern="${b}(/.*)?"
+    semanage fcontext --add -t glusterd_brick_t -r s0 "${pattern}"
+    # Set the labels on the new brick path.
+    restorecon -R "${b}"
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+set_brick_labels "${VOL}"
+
+exit 0
diff --git a/extras/hook-scripts/delete/Makefile.am b/extras/hook-scripts/delete/Makefile.am
new file mode 100644
index 00000000000..c98a05d9205
--- /dev/null
+++ b/extras/hook-scripts/delete/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = pre
diff --git a/extras/hook-scripts/delete/pre/Makefile.am b/extras/hook-scripts/delete/pre/Makefile.am
new file mode 100644
index 00000000000..4fbfbe7311f
--- /dev/null
+++ b/extras/hook-scripts/delete/pre/Makefile.am
@@ -0,0 +1,8 @@
+EXTRA_DIST = S10selinux-del-fcontext.sh
+
+scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/delete/pre/
+if WITH_SERVER
+if USE_SELINUX
+scripts_SCRIPTS = S10selinux-del-fcontext.sh
+endif
+endif
diff --git a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh
new file mode 100755
index 00000000000..056b52afe76
--- /dev/null
+++ b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Install to hooks/<HOOKS_VER>/delete/pre
+#
+# Delete the file context associated with the brick path on volume deletion. The
+# associated file context was added during volume creation.
+#
+# We do not explicitly relabel the brick, as this could be time consuming and
+# unnecessary.
+#
+###
+
+PROGNAME="Sselinux"
+OPTSPEC="volname:"
+VOL=
+
+function parse_args () {
+  ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@")
+  eval set -- "${ARGS}"
+
+  while true; do
+    case ${1} in
+      --volname)
+        shift
+        VOL=${1}
+      ;;
+      *)
+        shift
+        break
+      ;;
+    esac
+    shift
+  done
+}
+
+function delete_brick_fcontext()
+{
+  local volname=$1
+  local fctx
+  local list=()
+
+  fctx="$(semanage fcontext --list -C)"
+  # grab the path for each local brick
+  brickpath="/var/lib/glusterd/vols/${volname}/bricks/"
+  brickdirs=$(find "${brickpath}" -type f -exec grep '^path=' {} \; | \
+    cut -d= -f 2 | sort -u)
+  for b in ${brickdirs}
+  do
+    pattern="${b}(/.*)?"
+    echo "${fctx}" | grep "^${pattern}\s" >/dev/null
+    if [[ $? -eq 0 ]]; then
+      list+=("${pattern}")
+    fi
+  done
+  if [[ ${#list[@]} -gt 0 ]]; then
+    printf 'fcontext --delete %s\n' "${list[@]}" | semanage -i -
+  fi
+  for b in ${brickdirs}
+  do
+    restorecon -R "${b}"
+  done
+}
+
+SELINUX_STATE=$(which getenforce && getenforce)
+[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0
+
+parse_args "$@"
+[ -z "${VOL}" ] && exit 1
+
+delete_brick_fcontext "${VOL}"
+
+# failure to delete the fcontext is not fatal
+exit 0
diff --git a/extras/hook-scripts/reset/Makefile.am b/extras/hook-scripts/reset/Makefile.am
new file mode 100644
index 00000000000..6e2701e909e
--- /dev/null
+++ b/extras/hook-scripts/reset/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = post pre
+CLEANFILES =
diff --git a/extras/hook-scripts/reset/post/Makefile.am b/extras/hook-scripts/reset/post/Makefile.am
new file mode 100644
index 00000000000..1b336ac1a85
--- /dev/null
+++ b/extras/hook-scripts/reset/post/Makefile.am
@@ -0,0 +1 @@
+EXTRA_DIST =
diff --git a/extras/hook-scripts/reset/pre/Makefile.am b/extras/hook-scripts/reset/pre/Makefile.am
new file mode 100644
index 00000000000..1b336ac1a85
--- /dev/null
+++ b/extras/hook-scripts/reset/pre/Makefile.am
@@ -0,0 +1 @@
+EXTRA_DIST =
diff --git a/extras/hook-scripts/set/Makefile.am b/extras/hook-scripts/set/Makefile.am
new file mode 100644
index 00000000000..1fcade4b07f
--- /dev/null
+++ b/extras/hook-scripts/set/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = post
+CLEANFILES =
diff --git a/extras/hook-scripts/set/post/Makefile.am b/extras/hook-scripts/set/post/Makefile.am
new file mode 100644
index 00000000000..506a25a8666
--- /dev/null
+++ b/extras/hook-scripts/set/post/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST = S30samba-set.sh S32gluster_enable_shared_storage.sh
+
+hookdir = $(GLUSTERD_WORKDIR)/hooks/1/set/post/
+if WITH_SERVER
+hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/set/post/S30samba-set.sh b/extras/hook-scripts/set/post/S30samba-set.sh
new file mode 100755
index 00000000000..854f131f6c8
--- /dev/null
+++ b/extras/hook-scripts/set/post/S30samba-set.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/set/post/
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script enables user to enable or disable smb share by volume set
+#option. Keys "user.cifs" and "user.smb" both are valid, but user.smb is
+#preferred.
+
+
+PROGNAME="Ssamba-set"
+OPTSPEC="volname:,gd-workdir:"
+VOL=
+CONFIGFILE=
+LOGFILEBASE=
+PIDDIR=
+GLUSTERD_WORKDIR=
+USERSMB_SET=""
+USERCIFS_SET=""
+
+function parse_args () {
+        ARGS=$(getopt -o 'o:' -l $OPTSPEC -n $PROGNAME -- "$@")
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --gd-workdir)
+                    shift
+                    GLUSTERD_WORKDIR=$1
+                    ;;
+                --)
+                    shift
+                    break
+                    ;;
+                -o)
+                    shift
+                        read key value < <(echo "$1" | tr "=" " ")
+                        case "$key" in
+                            "user.cifs")
+                                USERCIFS_SET="YES"
+                                ;;
+                            "user.smb")
+                                USERSMB_SET="YES"
+                                ;;
+                            *)
+                                ;;
+                        esac
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+            shift
+        done
+}
+
+function find_config_info () {
+        cmdout=`smbd -b | grep smb.conf`
+        if [ $? -ne 0 ]; then
+                echo "Samba is not installed"
+                exit 1
+        fi
+        CONFIGFILE=`echo $cmdout | awk '{print $2}'`
+        PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'`
+        LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'`
+}
+
+function add_samba_share () {
+        volname=$1
+        STRING="\n[gluster-$volname]\n"
+        STRING+="comment = For samba share of volume $volname\n"
+        STRING+="vfs objects = glusterfs\n"
+        STRING+="glusterfs:volume = $volname\n"
+        STRING+="glusterfs:logfile = $LOGFILEBASE/glusterfs-$volname.%%M.log\n"
+        STRING+="glusterfs:loglevel = 7\n"
+        STRING+="path = /\n"
+        STRING+="read only = no\n"
+        STRING+="kernel share modes = no\n"
+        printf "$STRING"  >> ${CONFIGFILE}
+}
+
+function sighup_samba () {
+        pid=`cat ${PIDDIR}/smbd.pid`
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP "$pid";
+        else
+                service smb condrestart
+        fi
+}
+
+function deactivate_samba_share () {
+        volname=$1
+        sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE}
+}
+
+function is_volume_started () {
+        volname=$1
+        echo "$(grep status $GLUSTERD_WORKDIR/vols/"$volname"/info |\
+                cut -d"=" -f2)"
+}
+
+function get_smb () {
+        volname=$1
+        uservalue=
+
+        usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\
+                        cut -d"=" -f2)
+        usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\
+                       cut -d"=" -f2)
+
+        if [ -n "$usercifsvalue" ]; then
+                if [ "$usercifsvalue" = "disable" ] || [ "$usercifsvalue" = "off" ]; then
+                        uservalue="disable"
+                fi
+        fi
+
+        if [ -n "$usersmbvalue" ]; then
+                if [ "$usersmbvalue" = "disable" ] || [ "$usersmbvalue" = "off" ]; then
+                        uservalue="disable"
+                fi
+        fi
+
+        echo "$uservalue"
+}
+
+parse_args "$@"
+if [ "0" = "$(is_volume_started "$VOL")" ]; then
+    exit 0
+fi
+
+
+if [ "$USERCIFS_SET" = "YES" ] || [ "$USERSMB_SET" = "YES" ]; then
+    #Find smb.conf, smbd pid directory and smbd logfile path
+    find_config_info
+
+    if [ "$(get_smb "$VOL")" = "disable" ]; then
+        deactivate_samba_share $VOL
+    else
+        if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then
+            add_samba_share $VOL
+        else
+            sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' ${CONFIGFILE}
+        fi
+    fi
+    sighup_samba
+fi
diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh
new file mode 100755
index 00000000000..1f2564b44ff
--- /dev/null
+++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+key=`echo $3 | cut -d '=' -f 1`
+val=`echo $3 | cut -d '=' -f 2`
+if [ "$key" != "cluster.enable-shared-storage" ] && [ "$key" != "enable-shared-storage" ]; then
+    exit;
+fi
+if [ "$val" != 'enable' ]; then
+    if [ "$val" != 'disable' ]; then
+        exit;
+    fi
+fi
+
+option=$val
+
+key_val_pair1=`echo $4 | cut -d ',' -f 1`
+key_val_pair2=`echo $4 | cut -d ',' -f 2`
+
+key=`echo $key_val_pair1 | cut -d '=' -f 1`
+val=`echo $key_val_pair1 | cut -d '=' -f 2`
+if [ "$key" != "is_originator" ]; then
+    exit;
+fi
+is_originator=$val;
+
+key=`echo $key_val_pair2 | cut -d '=' -f 1`
+val=`echo $key_val_pair2 | cut -d '=' -f 2`
+if [ "$key" != "local_node_hostname" ]; then
+    exit;
+fi
+local_node_hostname=$val;
+
+# Read gluster peer status to find the peers
+# which are in 'Peer in Cluster' mode and
+# are connected.
+
+number_of_connected_peers=0
+while read -r line
+do
+    # Already got two connected peers. Including the current node
+    # we have 3 peers which is enough to create a shared storage
+    # with replica 3
+    if [ "$number_of_connected_peers" == "2" ]; then
+        break;
+    fi
+
+    key=`echo $line | cut -d ':' -f 1`
+    if [ "$key" == "Hostname" ]; then
+        hostname=`echo $line | cut -d ':' -f 2 | xargs`
+    fi
+
+    if [ "$key" == "State" ]; then
+        peer_state=`echo $line | cut -d ':' -f 2 | cut -d '(' -f 1 | xargs`
+        conn_state=`echo $line | cut -d '(' -f 2 | cut -d ')' -f 1 | xargs`
+
+        if [ "$peer_state" == "Peer in Cluster" ]; then
+            if [ "$conn_state" == "Connected" ]; then
+                ((number_of_connected_peers++))
+                connected_peer[$number_of_connected_peers]=$hostname
+            fi
+        fi
+    fi
+
+done < <(gluster peer status)
+
+# Include current node in connected peer list
+((number_of_connected_peers++))
+connected_peer[$number_of_connected_peers]=$local_node_hostname
+
+# forming the create vol command
+create_cmd="gluster --mode=script --wignore volume create \
+            gluster_shared_storage replica $number_of_connected_peers"
+
+# Adding the brick names in the command
+for i in "${connected_peer[@]}"
+do
+    create_cmd=$create_cmd" "$i:"$GLUSTERD_WORKDIR"/ss_brick
+done
+
+if [ "$option" == "disable" ]; then
+    # Unmount the volume on all the nodes
+    umount /run/gluster/shared_storage
+    cat /etc/fstab  | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp
+    mv /run/gluster/fstab.tmp /etc/fstab
+fi
+
+if [ "$is_originator" == 1 ]; then
+    if [ "$option" == "enable" ]; then
+        # Create and start the volume
+        $create_cmd
+        gluster --mode=script --wignore volume start gluster_shared_storage
+    fi
+
+    if [ "$option" == "disable" ]; then
+        # Stop and delete the volume
+        gluster --mode=script --wignore volume stop gluster_shared_storage
+        gluster --mode=script --wignore volume delete gluster_shared_storage
+    fi
+fi
+
+function check_volume_status()
+{
+    status=`gluster volume info gluster_shared_storage  | grep Status | cut -d ':' -f 2 | xargs`
+    echo $status
+}
+
+key=`echo $5 | cut -d '=' -f 1`
+val=`echo $5 | cut -d '=' -f 2`
+if [ "$key" == "transport.address-family" ]; then
+    mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \
+               $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage"
+else
+    mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \
+           /run/gluster/shared_storage"
+fi
+
+if [ "$option" == "enable" ]; then
+    retry=0;
+    # Wait for volume to start before mounting
+    status=$(check_volume_status)
+    while [ "$status" != "Started" ]; do
+        sleep 5;
+        ((retry++))
+        if [ "$retry" == 3 ]; then
+            break;
+        fi
+        status=$(check_volume_status)
+    done
+    # Mount the volume on all the nodes
+    umount /run/gluster/shared_storage
+    mkdir -p /run/gluster/shared_storage
+    $mount_cmd
+    cp /etc/fstab /run/gluster/fstab.tmp
+    echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults        0 0" >> /run/gluster/fstab.tmp
+    mv /run/gluster/fstab.tmp /etc/fstab
+fi
diff --git a/extras/hook-scripts/start/Makefile.am b/extras/hook-scripts/start/Makefile.am
new file mode 100644
index 00000000000..1fcade4b07f
--- /dev/null
+++ b/extras/hook-scripts/start/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = post
+CLEANFILES =
diff --git a/extras/hook-scripts/start/post/Makefile.am b/extras/hook-scripts/start/post/Makefile.am
new file mode 100644
index 00000000000..792019d3c9f
--- /dev/null
+++ b/extras/hook-scripts/start/post/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S31ganesha-start.sh
+
+hookdir = $(GLUSTERD_WORKDIR)/hooks/1/start/post/
+if WITH_SERVER
+hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/start/post/S29CTDBsetup.sh b/extras/hook-scripts/start/post/S29CTDBsetup.sh
new file mode 100755
index 00000000000..69a0d89a3eb
--- /dev/null
+++ b/extras/hook-scripts/start/post/S29CTDBsetup.sh
@@ -0,0 +1,84 @@
+#! /bin/bash
+#
+# - The script mounts the 'meta-vol' on start 'event' on a known
+#   directory (eg. /gluster/lock)
+# - P.S: There are other 'tasks' that need to be done outside this script
+#   to get CTDB based failover up and running.
+
+CTDB_MNT=/gluster/lock
+# Make sure ping-timeout is not default for CTDB volume
+PING_TIMEOUT_SECS=10
+PROGNAME="ctdb"
+OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:"
+HOSTNAME=`hostname`
+MNTOPTS="_netdev,transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}"
+VOL=
+GLUSTERD_WORKDIR=
+VERSION=
+VOLUME_OP=
+FIRST=
+# $META is the volume that will be used by CTDB as a shared filesystem.
+# It is not desirable to use this volume for storing 'data' as well.
+# META is set to 'all' (viz. a keyword and hence not a legal volume name)
+# to prevent the script from running for volumes it was not intended.
+# User needs to set META to the volume that serves CTDB lockfile.
+META="all"
+
+function parse_args () {
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --gd-workdir)
+                    shift
+                    GLUSTERD_WORKDIR=$1
+                    ;;
+                --version)
+                    shift
+                    VERSION=$1
+                    ;;
+                --volume-op)
+                    shift
+                    VOLUME_OP=$1
+                    ;;
+                --first)
+                    shift
+                    FIRST=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+
+            shift
+        done
+}
+
+function add_fstab_entry () {
+        volname=$1
+        mntpt=$2
+        mntopts="${MNTOPTS}"
+
+        mntent="${HOSTNAME}:/${volname} ${mntpt} glusterfs ${mntopts} 0 0"
+        exists=`grep "${mntpt}" /etc/fstab`
+        if [ "$exists" == "" ]
+        then
+            echo "${mntent}" >> /etc/fstab
+        fi
+}
+
+parse_args "$@"
+if [ "$META" = "$VOL" ]
+then
+        mkdir -p $CTDB_MNT
+        sleep 5
+        mount -t glusterfs -o${MNTOPTS} ${HOSTNAME}:/$VOL "$CTDB_MNT" && \
+            add_fstab_entry $VOL $CTDB_MNT
+        chkconfig ctdb on
+fi
diff --git a/extras/hook-scripts/start/post/S30samba-start.sh b/extras/hook-scripts/start/post/S30samba-start.sh
new file mode 100755
index 00000000000..cac0cbf1464
--- /dev/null
+++ b/extras/hook-scripts/start/post/S30samba-start.sh
@@ -0,0 +1,145 @@
+#!/bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/start/post
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script automagically creates shares for volume on every volume start
+#event by adding the entries in smb.conf file and sending SIGHUP to samba.
+#
+#In smb.conf:
+#glusterfs vfs plugin has to be specified as required vfs object.
+#Path value is relative to the root of gluster volume;"/" signifies complete
+#volume.
+
+PROGNAME="Ssamba-start"
+OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:"
+VOL=
+CONFIGFILE=
+LOGFILEBASE=
+PIDDIR=
+GLUSTERD_WORKDIR=
+VERSION=
+VOLUME_OP=
+FIRST=
+
+function parse_args () {
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --gd-workdir)
+                    shift
+                    GLUSTERD_WORKDIR=$1
+                    ;;
+                --version)
+                    shift
+                    VERSION=$1
+                    ;;
+                --volume-op)
+                    shift
+                    VOLUME_OP=$1
+                    ;;
+                --first)
+                    shift
+                    FIRST=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+
+            shift
+        done
+}
+
+function find_config_info () {
+        cmdout=$(smbd -b 2> /dev/null)
+        CONFIGFILE=$(echo "$cmdout" | grep CONFIGFILE | awk '{print $2}')
+        if [ -z "$CONFIGFILE" ]; then
+                echo "Samba is not installed"
+                exit 1
+        fi
+        PIDDIR=$(echo "$cmdout" | grep PIDDIR | awk '{print $2}')
+        LOGFILEBASE=$(echo "$cmdout" | grep 'LOGFILEBASE' | awk '{print $2}')
+}
+
+function add_samba_share () {
+        volname=$1
+        STRING="\n[gluster-$volname]\n"
+        STRING+="comment = For samba share of volume $volname\n"
+        STRING+="vfs objects = glusterfs\n"
+        STRING+="glusterfs:volume = $volname\n"
+        STRING+="glusterfs:logfile = $LOGFILEBASE/glusterfs-$volname.%%M.log\n"
+        STRING+="glusterfs:loglevel = 7\n"
+        STRING+="path = /\n"
+        STRING+="read only = no\n"
+        STRING+="kernel share modes = no\n"
+        printf "$STRING"  >> "${CONFIGFILE}"
+}
+
+function sighup_samba () {
+        pid=$(cat "${PIDDIR}/smbd.pid" 2> /dev/null)
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP "$pid";
+        else
+                service smb condrestart
+        fi
+}
+
+function get_smb () {
+        volname=$1
+        uservalue=
+
+        usercifsvalue=$(grep user.cifs "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\
+                        cut -d"=" -f2)
+        usersmbvalue=$(grep user.smb "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\
+                       cut -d"=" -f2)
+
+        if [ -n "$usercifsvalue" ]; then
+                if [ "$usercifsvalue" = "enable" ] || [ "$usercifsvalue" = "on" ]; then
+                        uservalue="enable"
+                fi
+        fi
+
+        if [ -n "$usersmbvalue" ]; then
+                if [ "$usersmbvalue" = "enable" ] || [ "$usersmbvalue" = "on" ]; then
+                        uservalue="enable"
+                fi
+        fi
+
+        echo "$uservalue"
+}
+
+parse_args "$@"
+
+value=$(get_smb "$VOL")
+
+if [ -z "$value" ] || [ "$value" != "enable" ]; then
+        exit 0
+fi
+
+#Find smb.conf, smbd pid directory and smbd logfile path
+find_config_info
+
+if ! grep --quiet "\[gluster-$VOL\]" "${CONFIGFILE}" ; then
+        add_samba_share "$VOL"
+else
+        sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' "${CONFIGFILE}"
+fi
+sighup_samba
diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh
new file mode 100755
index 00000000000..7ad6f23ad06
--- /dev/null
+++ b/extras/hook-scripts/start/post/S31ganesha-start.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+PROGNAME="Sganesha-start"
+OPTSPEC="volname:,gd-workdir:"
+VOL=
+declare -i EXPORT_ID
+ganesha_key="ganesha.enable"
+GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha"
+CONF1="$GANESHA_DIR/ganesha.conf"
+GLUSTERD_WORKDIR=
+
+function parse_args ()
+{
+        ARGS=$(getopt -l $OPTSPEC  -o "o" -name $PROGNAME $@)
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --gd-workdir)
+                    shift
+                    GLUSTERD_WORKDIR=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+            shift
+        done
+}
+
+
+
+#This function generates a new export entry as export.volume_name.conf
+function write_conf()
+{
+echo -e "# WARNING : Using Gluster CLI will overwrite manual
+# changes made to this file. To avoid it, edit the
+# file, copy it over to all the NFS-Ganesha nodes
+# and run ganesha-ha.sh --refresh-config."
+
+echo "EXPORT{"
+echo "      Export_Id = 2;"
+echo "      Path = \"/$VOL\";"
+echo "      FSAL {"
+echo "           name = \"GLUSTER\";"
+echo "           hostname=\"localhost\";"
+echo "           volume=\"$VOL\";"
+echo "           }"
+echo "      Access_type = RW;"
+echo "      Disable_ACL = true;"
+echo "      Squash=\"No_root_squash\";"
+echo "      Pseudo=\"/$VOL\";"
+echo "      Protocols = \"3\", \"4\" ;"
+echo "      Transports = \"UDP\",\"TCP\";"
+echo "      SecType = \"sys\";"
+echo "}"
+}
+
+#It adds the export dynamically by sending dbus signals
+function export_add()
+{
+        dbus-send --print-reply --system --dest=org.ganesha.nfsd \
+/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
+string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Export_Id=$EXPORT_ID)"
+
+}
+
+# based on src/scripts/ganeshactl/Ganesha/export_mgr.py
+function is_exported()
+{
+        local volume="${1}"
+
+        dbus-send --type=method_call --print-reply --system \
+                  --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
+                  org.ganesha.nfsd.exportmgr.ShowExports \
+            | grep -w -q "/${volume}"
+
+        return $?
+}
+
+# Check the info file (contains the volume options) to see if Ganesha is
+# enabled for this volume.
+function ganesha_enabled()
+{
+        local volume="${1}"
+        local info_file="${GLUSTERD_WORKDIR}/vols/${VOL}/info"
+        local enabled="off"
+
+        enabled=$(grep -w ${ganesha_key} ${info_file} | cut -d"=" -f2)
+
+        [ "${enabled}" == "on" ]
+
+        return $?
+}
+
+parse_args $@
+
+if ganesha_enabled ${VOL} && ! is_exported ${VOL}
+then
+        if [ ! -e ${GANESHA_DIR}/exports/export.${VOL}.conf ]
+        then
+                #Remove export entry from nfs-ganesha.conf
+                sed -i /$VOL.conf/d  $CONF1
+                write_conf ${VOL} > ${GANESHA_DIR}/exports/export.${VOL}.conf
+                EXPORT_ID=`cat $GANESHA_DIR/.export_added`
+                EXPORT_ID=EXPORT_ID+1
+                echo $EXPORT_ID > $GANESHA_DIR/.export_added
+                sed -i s/Export_Id.*/"Export_Id=$EXPORT_ID;"/ \
+                        $GANESHA_DIR/exports/export.$VOL.conf
+                echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1
+        else
+                EXPORT_ID=$(grep ^[[:space:]]*Export_Id $GANESHA_DIR/exports/export.$VOL.conf |\
+                          awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]')
+        fi
+        export_add $VOL
+fi
+
+exit 0
diff --git a/extras/hook-scripts/stop/Makefile.am b/extras/hook-scripts/stop/Makefile.am
new file mode 100644
index 00000000000..e2ac8e2740b
--- /dev/null
+++ b/extras/hook-scripts/stop/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = pre
+CLEANFILES =
diff --git a/extras/hook-scripts/stop/pre/Makefile.am b/extras/hook-scripts/stop/pre/Makefile.am
new file mode 100644
index 00000000000..9e8d1565e93
--- /dev/null
+++ b/extras/hook-scripts/stop/pre/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_DIST =  S29CTDB-teardown.sh S30samba-stop.sh
+
+hookdir = $(GLUSTERD_WORKDIR)/hooks/1/stop/pre/
+if WITH_SERVER
+hook_SCRIPTS = $(EXTRA_DIST)
+endif
diff --git a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh
new file mode 100755
index 00000000000..0975a00f18d
--- /dev/null
+++ b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh
@@ -0,0 +1,62 @@
+#! /bin/bash
+
+CTDB_MNT=/gluster/lock
+PROGNAME="ctdb"
+OPTSPEC="volname:,last:"
+VOL=
+LAST=
+# $META is the volume that will be used by CTDB as a shared filesystem.
+# It is not desirable to use this volume for storing 'data' as well.
+# META is set to 'all' (viz. a keyword and hence not a legal volume name)
+# to prevent the script from running for volumes it was not intended.
+# User needs to set META to the volume that serves CTDB lockfile.
+META="all"
+
+function parse_args () {
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --last)
+                    shift
+                    LAST=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+            shift
+        done
+}
+
+
+function remove_fstab_entry () {
+        mntpt=$1
+        fstab="/etc/fstab"
+        exists=`grep "$mntpt" ${fstab}`
+        esc_mntpt=$(echo -e $mntpt | sed 's/\//\\\//g')
+        if [ "$exists" != " " ]
+        then
+            sed -i /"$esc_mntpt"/d $fstab
+            exists=`grep "$mntpt" ${fstab}`
+            if [ "$exists" != " " ]
+            then
+                echo "fstab entry cannot be removed for unknown reason"
+                exit 1
+            fi
+        fi
+}
+
+parse_args "$@"
+if [ "$META" = "$VOL" ]
+then
+        umount "$CTDB_MNT"
+        chkconfig ctdb off
+        remove_fstab_entry $CTDB_MNT
+fi
diff --git a/extras/hook-scripts/stop/pre/S30samba-stop.sh b/extras/hook-scripts/stop/pre/S30samba-stop.sh
new file mode 100755
index 00000000000..ea799381d62
--- /dev/null
+++ b/extras/hook-scripts/stop/pre/S30samba-stop.sh
@@ -0,0 +1,77 @@
+#! /bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/stop/pre
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script automagically removes shares for volume on every volume stop
+#event by removing the volume related entries(if any) in smb.conf file.
+
+PROGNAME="Ssamba-stop"
+OPTSPEC="volname:,last:"
+VOL=
+CONFIGFILE=
+PIDDIR=
+LAST=
+
+function parse_args () {
+        ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@")
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                    ;;
+                --last)
+                    shift
+                    LAST=$1
+                    ;;
+                *)
+                    shift
+                    break
+                    ;;
+            esac
+
+            shift
+        done
+}
+
+function find_config_info () {
+        cmdout=`smbd -b | grep smb.conf`
+        if [ $? -ne 0 ];then
+                echo "Samba is not installed"
+                exit 1
+        fi
+        CONFIGFILE=`echo $cmdout | awk '{print $2}'`
+        PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'`
+}
+
+function deactivate_samba_share () {
+        volname=$1
+        sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE}
+}
+
+function sighup_samba () {
+        pid=`cat ${PIDDIR}/smbd.pid`
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP $pid;
+        else
+                service smb condrestart
+        fi
+}
+
+parse_args "$@"
+find_config_info
+deactivate_samba_share $VOL
+sighup_samba
diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh
new file mode 100755
index 00000000000..ebc6bf144aa
--- /dev/null
+++ b/extras/identify-hangs.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+function get_statedump_fnames_without_timestamps
+{
+    ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u
+}
+
+function get_non_uniq_fields
+{
+    local statedump_fname_prefix=$1
+    print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}'
+}
+
+function print_stack_lkowner_unique_in_one_line
+{
+    local statedump_fname_prefix=$1
+    sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - -
+}
+
+function get_stacks_that_appear_in_multiple_statedumps
+{
+    #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps
+    #print the stack
+    local statedump_fname_prefix=$1
+    while read -r non_uniq_stack;
+    do
+        if [ -z "$printed" ];
+        then
+            printed="1"
+        fi
+        echo "$statedump_fname_prefix" "$non_uniq_stack"
+    done < <(get_non_uniq_fields "$statedump_fname_prefix")
+}
+
+statedumpdir=${1}
+if [ -z "$statedumpdir" ];
+then
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+if [ ! -d "$statedumpdir" ];
+then
+    echo "$statedumpdir: Is not a directory"
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+cd "$statedumpdir" || exit 1
+for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps);
+do
+    get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix"
+done | column -t
+echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2
diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am
index 66715f4314a..8d8cc69571a 100644
--- a/extras/init.d/Makefile.am
+++ b/extras/init.d/Makefile.am
@@ -1,19 +1,32 @@
 
-EXTRA_DIST = glusterd-Debian glusterd-Redhat glusterd-SuSE glusterd.plist
+EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat \
+	glusterd-SuSE glusterd.plist glustereventsd-FreeBSD \
+	glustereventsd-Redhat glustereventsd-Debian
 
-CLEANFILES = 
+CLEANFILES =
 
-initdir = @initdir@
-launchddir = @launchddir@
+INIT_DIR = @initdir@
+SYSTEMD_DIR = @systemddir@
+LAUNCHD_DIR = @launchddir@
 
-$(GF_DISTRIBUTION): 
-	$(mkdir_p) $(DESTDIR)$(initdir)	
-	$(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(initdir)/glusterd
+$(GF_DISTRIBUTION):
+if WITH_SERVER
+	@if [ ! -d $(SYSTEMD_DIR) ]; then \
+		$(mkdir_p) $(DESTDIR)$(INIT_DIR); \
+		$(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \
+	fi
+endif
+if BUILD_EVENTS
+	@if [ ! -d $(SYSTEMD_DIR) ]; then \
+		$(mkdir_p) $(DESTDIR)$(INIT_DIR); \
+		$(INSTALL_PROGRAM) glustereventsd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glustereventsd; \
+	fi
+endif
 
 install-exec-local: $(GF_DISTRIBUTION)
 
-install-data-local: 
+install-data-local:
 if GF_DARWIN_HOST_OS
-	$(mkdir_p) $(DESTDIR)$(launchddir)
-	$(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(launchddir)/com.gluster.glusterd.plist
+	$(mkdir_p) $(DESTDIR)$(LAUNCHD_DIR)
+	$(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(LAUNCHD_DIR)/org.gluster.glusterd.plist
 endif
diff --git a/extras/init.d/glusterd-FreeBSD.in b/extras/init.d/glusterd-FreeBSD.in
new file mode 100644
index 00000000000..21c3da72624
--- /dev/null
+++ b/extras/init.d/glusterd-FreeBSD.in
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# PROVIDE: glusterd
+
+. /etc/rc.subr
+
+name="glusterd"
+rcvar=`set_rcvar`
+command=@prefix@/sbin/${name}
+pidfile="/var/run/${name}.pid"
+glusterd_flags="-p /var/run/${name}.pid"
+start_precmd="glusterd_prestart"
+
+glusterd_prestart()
+{
+    mkdir -p @GLUSTERD_WORKDIR@ /var/log/glusterfs
+    return 0
+}
+
+load_rc_config $name
+run_rc_command "$1"
diff --git a/extras/init.d/glusterd-Redhat.in b/extras/init.d/glusterd-Redhat.in
index 5ee3ab90832..94801fe31a5 100755
--- a/extras/init.d/glusterd-Redhat.in
+++ b/extras/init.d/glusterd-Redhat.in
@@ -1,55 +1,143 @@
 #!/bin/bash
 #
-# chkconfig: 35 90 12
-# description: Gluster File System service for volume management
+# glusterd   Startup script for the glusterfs server
 #
+# chkconfig:   - 20 80
+# description: Clustered file-system server
 
-# Get function from functions library
+### BEGIN INIT INFO
+# Provides: glusterd
+# Required-Start: $local_fs $network
+# Required-Stop: $local_fs $network
+# Should-Start: 
+# Should-Stop: 
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: glusterfs server
+# Description:       Clustered file-system server
+### END INIT INFO
+#
+
+# Source function library.
 . /etc/rc.d/init.d/functions
 
 BASE=glusterd
+
+# Fedora File System Layout dictates /run
+[ -e /run ] && RUNDIR="/run"
+PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid"
+
+PID=`test -f $PIDFILE && cat $PIDFILE`
+
+# Overwriteable from sysconfig
+LOG_LEVEL=''
+LOG_FILE=''
+GLUSTERD_OPTIONS=''
+GLUSTERD_NOFILE='65536'
+
+[ -f /etc/sysconfig/${BASE} ] && . /etc/sysconfig/${BASE}
+
+[ ! -z $LOG_LEVEL ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-level ${LOG_LEVEL}"
+[ ! -z $LOG_FILE ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-file ${LOG_FILE}"
+
+GLUSTERFSD=glusterfsd
+GLUSTERFS=glusterfs
 GLUSTERD_BIN=@prefix@/sbin/$BASE
-GLUSTERD_OPTS=""
+GLUSTERD_OPTS="--pid-file=$PIDFILE ${GLUSTERD_OPTIONS}"
 GLUSTERD="$GLUSTERD_BIN $GLUSTERD_OPTS"
 RETVAL=0
 
+LOCKFILE=/var/lock/subsys/${BASE}
+
 # Start the service $BASE
 start()
 {
-       echo -n $"Starting $BASE:"
-       daemon $GLUSTERD
-       RETVAL=$?
-       echo
-       [ $RETVAL -ne 0 ] && exit $RETVAL
+       if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then
+           echo "glusterd service is already running with pid $PID"
+           return 0
+       else
+           ulimit -n $GLUSTERD_NOFILE
+           echo -n $"Starting $BASE:"
+           daemon $GLUSTERD
+           RETVAL=$?
+           echo
+           [ $RETVAL -eq 0 ] && touch $LOCKFILE
+           return $RETVAL
+       fi
 }
 
 # Stop the service $BASE
 stop()
 {
-       echo -n $"Stopping $BASE:"
-       killproc $BASE
-       echo
+    echo -n $"Stopping $BASE:"
+    if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then
+        killproc -p $PIDFILE $BASE
+    else
+        killproc $BASE
+    fi
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
+    return $RETVAL
+}
+
+restart()
+{
+    stop
+    start
+}
+
+reload()
+{
+    restart
+}
+
+force_reload()
+{
+    restart
+}
+
+rh_status()
+{
+    status $BASE
+}
+
+rh_status_q()
+{
+    rh_status &>/dev/null
 }
 
 
 ### service arguments ###
 case $1 in
- start)
-       start
-       ;;
- stop)
-       stop
-       ;;
- status)
-       status $BASE
-       ;;
- restart)
-       $0 stop
-       $0 start
-       ;;
- *)
-       echo $"Usage: $0 {start|stop|status|restart}."
-       exit 1
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 1
 esac
 
-exit 0
+exit $?
diff --git a/extras/init.d/glusterd-SuSE.in b/extras/init.d/glusterd-SuSE.in
index 16cf8de6a13..6259bab00b6 100755
--- a/extras/init.d/glusterd-SuSE.in
+++ b/extras/init.d/glusterd-SuSE.in
@@ -2,8 +2,8 @@
 #
 ### BEGIN INIT INFO
 # Provides:       glusterd
-# Required-Start: $local_fs $network
-# Required-Stop:
+# Required-Start: $remote_fs $network
+# Required-Stop: $remote_fs $network
 # Default-Start:  3 5
 # Default-Stop:
 # Short-Description: Gluster File System service for volume management
@@ -61,13 +61,17 @@ case $1 in
 	fi
 	rc_status -v
        ;;
+ reload)
+       rc_failed 3
+       rc_status -v
+       ;;
  restart)
        $0 stop
        $0 start
        rc_status
        ;;
  *)
-       echo $"Usage: $0 {start|stop|status|restart}."
+       echo $"Usage: $0 {start|stop|status|reload|restart}."
        exit 1
 esac
 
diff --git a/extras/init.d/glusterd.plist.in b/extras/init.d/glusterd.plist.in
index 7385fa486df..3ee9f60c5c5 100644
--- a/extras/init.d/glusterd.plist.in
+++ b/extras/init.d/glusterd.plist.in
@@ -3,7 +3,7 @@
 <plist version="1.0">
 <dict>
         <key>Label</key>
-        <string>com.gluster.glusterd</string>
+        <string>org.gluster.glusterd</string>
         <key>ProgramArguments</key>
         <array>
                 <string>@prefix@/sbin/glusterd</string>
diff --git a/extras/init.d/glustereventsd-Debian.in b/extras/init.d/glustereventsd-Debian.in
new file mode 100644
index 00000000000..6eebdb2b8d8
--- /dev/null
+++ b/extras/init.d/glustereventsd-Debian.in
@@ -0,0 +1,91 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          glustereventsd
+# Required-Start:    $local_fs $network
+# Required-Stop:     $local_fs $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Gluster Events Server
+# Description:       Gluster Events Server
+### END INIT INFO
+
+# Author: Chris AtLee <chris@atlee.ca>
+# Patched by: Matthias Albert < matthias@linux4experts.de>
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+NAME=glustereventsd
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON=@prefix@/sbin/$NAME
+PIDFILE=/var/run/$NAME.pid
+GLUSTEREVENTSD_OPTS=""
+PID=`test -f $PIDFILE && cat $PIDFILE`
+
+
+# Gracefully exit if the package has been removed.
+test -x $DAEMON || exit 0
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+. /lib/lsb/init-functions
+
+
+do_start()
+{
+    pidofproc -p $PIDFILE $DAEMON >/dev/null
+    status=$?
+    if [ $status -eq 0 ]; then
+      log_success_msg "glustereventsd service is already running with pid $PID"
+    else
+      log_daemon_msg "Starting glustereventsd service" "glustereventsd"
+      start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $GLUSTEREVENTSD_OPTS
+      log_end_msg $?
+      start_daemon -p $PIDFILE $DAEMON -f $CONFIGFILE
+      return $?
+    fi
+}
+
+do_stop()
+{
+    log_daemon_msg "Stopping glustereventsd service" "glustereventsd"
+    start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+    log_end_msg $?
+    rm -f $PIDFILE
+    killproc -p $PIDFILE $DAEMON
+    return $?
+}
+
+do_status()
+{
+     pidofproc -p $PIDFILE $DAEMON >/dev/null
+     status=$?
+     if [ $status -eq 0 ]; then
+       log_success_msg "glustereventsd service is running with pid $PID"
+     else
+       log_failure_msg "glustereventsd service is not running."
+     fi
+     exit $status
+}
+
+case "$1" in
+  start)
+        do_start
+        ;;
+  stop)
+        do_stop
+        ;;
+  status)
+        do_status;
+        ;;
+  restart|force-reload)
+        do_stop
+        sleep 2
+        do_start
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac
+
diff --git a/extras/init.d/glustereventsd-FreeBSD.in b/extras/init.d/glustereventsd-FreeBSD.in
new file mode 100644
index 00000000000..2e8303ec6c6
--- /dev/null
+++ b/extras/init.d/glustereventsd-FreeBSD.in
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# PROVIDE: glustereventsd
+
+. /etc/rc.subr
+
+name="glustereventsd"
+rcvar=`set_rcvar`
+command=@prefix@/sbin/${name}
+command_interpreter=/usr/local/bin/python
+pidfile="/var/run/${name}.pid"
+glustereventsd_flags="-p /var/run/${name}.pid"
+start_cmd="/usr/sbin/daemon $command ${glustereventsd_flags}"
+
+load_rc_config $name
+run_rc_command "$1"
diff --git a/extras/init.d/glustereventsd-Redhat.in b/extras/init.d/glustereventsd-Redhat.in
new file mode 100644
index 00000000000..d23ce4c244f
--- /dev/null
+++ b/extras/init.d/glustereventsd-Redhat.in
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# glustereventsd   Startup script for the glusterfs Events server
+#
+# chkconfig:   - 20 80
+# description: Gluster Events Server
+
+### BEGIN INIT INFO
+# Provides: glustereventsd
+# Required-Start: $local_fs $network
+# Required-Stop: $local_fs $network
+# Should-Start:
+# Should-Stop:
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: glusterfs Events server
+# Description:       GlusterFS Events Server
+### END INIT INFO
+#
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+BASE=glustereventsd
+
+# Fedora File System Layout dictates /run
+[ -e /run ] && RUNDIR="/run"
+PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid"
+
+PID=`test -f $PIDFILE && cat $PIDFILE`
+
+GLUSTEREVENTSD_BIN=@prefix@/sbin/$BASE
+GLUSTEREVENTSD_OPTS="--pid-file=$PIDFILE"
+GLUSTEREVENTSD="$GLUSTEREVENTSD_BIN $GLUSTEREVENTSD_OPTS"
+RETVAL=0
+
+LOCKFILE=/var/lock/subsys/${BASE}
+
+# Start the service $BASE
+start()
+{
+       if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then
+           echo "glustereventsd service is already running with pid $PID"
+           return 0
+       else
+           echo -n $"Starting $BASE:"
+           daemon $GLUSTEREVENTSD &
+           RETVAL=$?
+           echo
+           [ $RETVAL -eq 0 ] && touch $LOCKFILE
+           return $RETVAL
+       fi
+}
+
+# Stop the service $BASE
+stop()
+{
+    echo -n $"Stopping $BASE:"
+    if pidofproc -p $PIDFILE $GLUSTEREVENTSD_BIN &> /dev/null; then
+        killproc -p $PIDFILE $BASE
+    else
+        killproc $BASE
+    fi
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
+    return $RETVAL
+}
+
+restart()
+{
+    stop
+    start
+}
+
+reload()
+{
+    restart
+}
+
+force_reload()
+{
+    restart
+}
+
+rh_status()
+{
+    status $BASE
+}
+
+rh_status_q()
+{
+    rh_status &>/dev/null
+}
+
+
+### service arguments ###
+case $1 in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 1
+esac
+
+exit $?
diff --git a/extras/logger.conf.example b/extras/logger.conf.example
new file mode 100644
index 00000000000..248be5bda69
--- /dev/null
+++ b/extras/logger.conf.example
@@ -0,0 +1,13 @@
+#
+# Sample logger.conf file to configure enhanced Logging in GlusterFS
+#
+# To enable enhanced logging capabilities,
+#
+# 1. rename this file to /etc/glusterfs/logger.conf
+#
+# 2. rename /etc/rsyslog.d/gluster.conf.example to
+#    /etc/rsyslog.d/gluster.conf
+#
+# This change requires restart of all gluster services/volumes and
+# rsyslog.
+#
diff --git a/extras/mount-shared-storage.sh b/extras/mount-shared-storage.sh
new file mode 100755
index 00000000000..cc40e13c3e3
--- /dev/null
+++ b/extras/mount-shared-storage.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#Post reboot there is a chance in which mounting of shared storage will fail
+#This will impact starting of features like NFS-Ganesha. So this script will
+#try to mount the shared storage if it fails
+
+exitStatus=0
+
+while IFS= read -r glm
+do
+	IFS=$' \t' read -r -a arr <<< "$glm"
+
+	#Validate storage type is glusterfs
+	if [ "${arr[2]}" == "glusterfs" ]
+	then
+
+		#check whether shared storage is mounted
+		#if it is mounted then mountpoint -q will return a 0 success code
+		if mountpoint -q "${arr[1]}"
+		then
+			echo "${arr[1]} is already mounted"
+			continue
+		fi
+
+		mount -t glusterfs -o "${arr[3]}" "${arr[0]}" "${arr[1]}"
+		#wait for few seconds
+		sleep 10
+
+		#recheck mount got succeed
+		if mountpoint -q "${arr[1]}"
+		then
+			echo "${arr[1]} has been mounted"
+			continue
+		else
+			echo "${arr[1]} failed to mount"
+			exitStatus=1
+		fi
+	fi
+done <<< "$(sed '/^#/ d' </etc/fstab | grep 'glusterfs')"
+exit $exitStatus
diff --git a/extras/ocf/Makefile.am b/extras/ocf/Makefile.am
new file mode 100644
index 00000000000..c49a835fbca
--- /dev/null
+++ b/extras/ocf/Makefile.am
@@ -0,0 +1,11 @@
+EXTRA_DIST = glusterd.in volume.in
+
+# The root of the OCF resource agent hierarchy
+# Per the OCF standard, it's always "lib",
+# not "lib64" (even on 64-bit platforms).
+ocfdir = $(prefix)/lib/ocf
+
+# The ceph provider directory
+radir = $(ocfdir)/resource.d/$(PACKAGE_NAME)
+
+ra_SCRIPTS = glusterd volume
diff --git a/extras/ocf/glusterd.in b/extras/ocf/glusterd.in
new file mode 100755
index 00000000000..c119a285d32
--- /dev/null
+++ b/extras/ocf/glusterd.in
@@ -0,0 +1,212 @@
+#!/bin/sh
+#
+# glusterd
+#
+# Description:  Manages a glusterd server as a (typically cloned)
+#               HA resource
+#
+# Authors:      Florian Haas (hastexo Professional Services GmbH)
+#
+# License:      GNU General Public License (GPL)
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir and localstatedir aren't passed in as
+# configure flags, they're defined in terms of prefix
+prefix=@prefix@
+#######################################################################
+
+
+OCF_RESKEY_binary_default="glusterd"
+OCF_RESKEY_pid_default="@localstatedir@/run/glusterd.pid"
+OCF_RESKEY_socket_default=""
+OCF_RESKEY_additional_parameters_default=""
+
+: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
+: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}}
+
+glusterd_meta_data() {
+    cat <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="glusterd" version="0.1">
+  <version>0.1</version>
+  <longdesc lang="en">
+  </longdesc>
+  <shortdesc lang="en">Manages a Gluster server</shortdesc>
+  <parameters>
+    <parameter name="binary">
+      <longdesc lang="en">
+      Name of the glusterd executable. Specify a full absolute
+      path if the binary is not in your \$PATH.
+      </longdesc>
+      <shortdesc lang="en">glusterd executable</shortdesc>
+      <content type="string" default="$OCF_RESKEY_binary_default"/>
+    </parameter>
+    <parameter name="pid">
+      <longdesc lang="en">
+      Path to the glusterd PID file.
+      </longdesc>
+      <shortdesc lang="en">PID file</shortdesc>
+      <content type="string" default="$OCF_RESKEY_pid_default"/>
+    </parameter>
+    <parameter name="socket">
+      <longdesc lang="en">
+      Path to the glusterd UNIX socket file. If unspecified,
+      glusterd will not listen on any socket.
+      </longdesc>
+      <shortdesc lang="en">Socket file</shortdesc>
+      <content type="string"/>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="start"        timeout="20" />
+    <action name="stop"         timeout="20" />
+    <action name="monitor"      timeout="20" interval="10" />
+    <action name="reload"       timeout="20" />
+    <action name="meta-data"    timeout="5" />
+    <action name="validate-all"   timeout="20" />
+  </actions>
+</resource-agent>
+EOF
+
+}
+
+glusterd_start() {
+    local glusterd_options
+    # exit immediately if configuration is not valid
+    glusterd_validate_all || exit $?
+
+    # if resource is already running, bail out early
+    if glusterd_monitor; then
+        ocf_log info "Resource is already running"
+        return $OCF_SUCCESS
+    fi
+
+    # actually start up the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    glusterd_options="-p $OCF_RESKEY_pid"
+    if [ -n "$OCF_RESKEY_socket" ]; then
+	glusterd_options="$glusterd_options -S $OCF_RESKEY_socket"
+    fi
+    if [ -n "$OCF_RESKEY_additional_parameters" ]; then
+	glusterd_options="$glusterd_options $OCF_RESKEY_additional_parameters"
+    fi
+
+    ocf_run $OCF_RESKEY_binary $glusterd_options || exit $OCF_ERR_GENERIC
+
+    # After the resource has been started, check whether it started up
+    # correctly. If the resource starts asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # start up within the defined timeout, the cluster manager will
+    # consider the start action failed
+    while ! glusterd_monitor; do
+        ocf_log debug "Resource has not started yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+}
+
+glusterd_stop() {
+    local rc
+    local pid
+
+    # exit immediately if configuration is not valid
+    glusterd_validate_all || exit $?
+
+    glusterd_monitor
+    rc=$?
+    case "$rc" in
+        "$OCF_SUCCESS")
+            # Currently running. Normal, expected behavior.
+            ocf_log debug "Resource is currently running"
+            ;;
+        "$OCF_NOT_RUNNING")
+            # Currently not running. Nothing to do.
+            ocf_log info "Resource is already stopped"
+            return $OCF_SUCCESS
+            ;;
+    esac
+
+    # actually shut down the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    pid=`cat $OCF_RESKEY_pid`
+    ocf_run kill -s TERM $pid || exit OCF_ERR_GENERIC
+
+    # After the resource has been stopped, check whether it shut down
+    # correctly. If the resource stops asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # shut down within the defined timeout, the cluster manager will
+    # consider the stop action failed
+    while glusterd_monitor; do
+        ocf_log debug "Resource has not stopped yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+
+}
+
+glusterd_monitor() {
+    local pid
+
+    [ -e $OCF_RESKEY_pid ] || return $OCF_NOT_RUNNING
+
+    pid=`cat $OCF_RESKEY_pid`
+    ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING
+
+    ocf_log debug "$OCF_RESKEY_binary running with PID $pid"
+    return $OCF_SUCCESS
+}
+
+glusterd_validate_all() {
+    # Test for required binaries
+    check_binary $OCF_RESKEY_binary
+
+    return $OCF_SUCCESS
+}
+
+
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+meta-data)      glusterd_meta_data
+                exit $OCF_SUCCESS
+                ;;
+usage|help)     glusterd_usage
+                exit $OCF_SUCCESS
+                ;;
+esac
+
+# Anything other than meta-data and usage must pass validation
+glusterd_validate_all || exit $?
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          glusterd_start;;
+stop)           glusterd_stop;;
+status|monitor) glusterd_monitor;;
+reload)         ocf_log info "Reloading..."
+                glusterd_start
+                ;;
+validate-all)   ;;
+notify)		exit $OCF_SUCCESS;;
+*)              glusterd_usage
+                exit $OCF_ERR_UNIMPLEMENTED
+                ;;
+esac
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc"
+exit $rc
diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in
new file mode 100755
index 00000000000..76cc649e55f
--- /dev/null
+++ b/extras/ocf/volume.in
@@ -0,0 +1,276 @@
+#!/bin/sh
+#
+# glusterd
+#
+# Description:  Manages a glusterd server as a (typically cloned)
+#               HA resource
+#
+# Authors:      Florian Haas (hastexo Professional Services GmbH)
+#               Jiri Lunacek (Hosting90 Systems s.r.o.)
+#
+# License:      GNU General Public License (GPL)
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir and localstatedir aren't passed in as
+# configure flags, they're defined in terms of prefix
+prefix=@prefix@
+SHORTHOSTNAME=`hostname -s`
+#######################################################################
+
+OCF_RESKEY_binary_default="gluster"
+
+: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
+
+volume_meta_data() {
+    cat <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="volume" version="0.1">
+  <version>0.1</version>
+  <longdesc lang="en">
+Manages a GlusterFS volume and monitors its bricks. When a resource of
+this type is configured as a clone (as is commonly the case), then it
+must have clone ordering enabled.
+  </longdesc>
+  <shortdesc lang="en">Manages a GlusterFS volume</shortdesc>
+  <parameters>
+    <parameter name="volname" required="1">
+      <longdesc lang="en">
+      The name of the volume to manage.
+      </longdesc>
+      <shortdesc lang="en">volume name</shortdesc>
+      <content type="string"/>
+    </parameter>
+    <parameter name="binary">
+      <longdesc lang="en">
+      Name of the gluster executable. Specify a full absolute
+      path if the binary is not in your \$PATH.
+      </longdesc>
+      <shortdesc lang="en">gluster executable</shortdesc>
+      <content type="string" default="$OCF_RESKEY_binary_default"/>
+    </parameter>
+    <parameter name="peer_map">
+      <longdesc lang="en">
+      Mapping of hostname - peer name in the gluster cluster
+      in format hostname1:peername1,hostname2:peername2,...
+      </longdesc>
+      <shortdesc lang="en">gluster peer map</shortdesc>
+      <content type="string" default=""/>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="start"        timeout="20" />
+    <action name="stop"         timeout="20" />
+    <action name="monitor"      timeout="20" interval="10" />
+    <action name="reload"       timeout="20" />
+    <action name="meta-data"    timeout="5" />
+    <action name="validate-all"   timeout="20" />
+  </actions>
+</resource-agent>
+EOF
+
+}
+
+if [ -n "${OCF_RESKEY_peer_map}" ]; then
+    SHORTHOSTNAME=`echo "${OCF_RESKEY_peer_map}" | egrep -o "$SHORTHOSTNAME\:[^,]+" | awk -F: '{print $2}'`
+fi
+
+volume_getdir() {
+    local voldir
+    voldir="@GLUSTERD_WORKDIR@/vols/${OCF_RESKEY_volname}"
+
+    [ -d ${voldir} ] || return 1
+
+    echo "${voldir}"
+    return 0
+}
+
+volume_getpid_dir() {
+    local volpid_dir
+    volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}"
+
+    [ -d ${volpid_dir} ] || return 1
+
+    echo "${volpid_dir}"
+    return 0
+}
+
+volume_getbricks() {
+    local infofile
+    local voldir
+    voldir=`volume_getdir`
+    infofile="${voldir}/info"
+
+    [ -e ${infofile} ] || return 1
+
+    echo "`sed -n -e "s/^brick-.\+=${SHORTHOSTNAME}://p" < ${infofile}`"
+    return 0
+}
+
+volume_getpids() {
+    local bricks
+    local pidfile
+    local infofile
+    local volpid_dir
+
+    volpid_dir=`volume_getpid_dir`
+    bricks=`volume_getbricks`
+    
+    if [ -z "$bricks" ]; then
+        return 1
+    fi
+
+    for brick in ${bricks}; do
+	pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid"
+	[ -e $pidfile ] || return 1
+	cat $pidfile
+    done
+
+    return 0
+}
+
+volume_start() {
+    local volume_options
+
+    # exit immediately if configuration is not valid
+    volume_validate_all || exit $?
+
+    # if resource is already running, bail out early
+    if volume_monitor; then
+        ocf_log info "Resource is already running"
+        return $OCF_SUCCESS
+    fi
+
+    # actually start up the resource here
+    ocf_run "$OCF_RESKEY_binary" \
+	volume start "$OCF_RESKEY_volname" force || exit $OCF_ERR_GENERIC
+
+    # After the resource has been started, check whether it started up
+    # correctly. If the resource starts asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # start up within the defined timeout, the cluster manager will
+    # consider the start action failed
+    while ! volume_monitor; do
+        ocf_log debug "Resource has not started yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+}
+
+volume_stop() {
+    local rc
+    local pid
+
+    # exit immediately if configuration is not valid
+    volume_validate_all || exit $?
+
+    volume_monitor
+    rc=$?
+    case "$rc" in
+        "$OCF_SUCCESS")
+            # Currently running. Normal, expected behavior.
+            ocf_log debug "Resource is currently running"
+            ;;
+        "$OCF_NOT_RUNNING")
+            # Currently not running. Nothing to do.
+            ocf_log info "Resource is already stopped"
+            return $OCF_SUCCESS
+            ;;
+    esac
+
+    # actually shut down the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    pids=`volume_getpids`
+    for pid in $pids; do
+	ocf_run kill -s TERM $pid
+    done
+
+    # After the resource has been stopped, check whether it shut down
+    # correctly. If the resource stops asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # shut down within the defined timeout, the cluster manager will
+    # consider the stop action failed
+    while volume_monitor; do
+        ocf_log debug "Resource has not stopped yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+
+}
+
+volume_monitor() {
+    local pid
+
+    pids=`volume_getpids` || return $OCF_NOT_RUNNING
+
+    for pid in $pids; do
+	ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING
+    done
+
+    ocf_log debug "Local bricks for volume ${OCF_RESKEY_volname} running with PIDs $pids"
+    return $OCF_SUCCESS
+}
+
+volume_validate_all() {
+    # Test for configuration errors first
+    if [ -z "${OCF_RESKEY_volname}" ]; then
+	ocf_log err 'Missing required parameter "volname"'
+	return $OCF_ERR_CONFIGURED
+    fi
+
+    # Test for required binaries
+    check_binary $OCF_RESKEY_binary
+	
+	if [ -z "$SHORTHOSTNAME" ]; then
+		ocf_log err 'Unable to get host in node map'
+		return $OCF_ERR_CONFIGURED
+	fi
+
+    return $OCF_SUCCESS
+}
+
+
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+meta-data)      volume_meta_data
+                exit $OCF_SUCCESS
+                ;;
+usage|help)     volume_usage
+                exit $OCF_SUCCESS
+                ;;
+esac
+
+# Anything other than meta-data and usage must pass validation
+volume_validate_all || exit $?
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          volume_start;;
+stop)           volume_stop;;
+status|monitor) volume_monitor;;
+reload)         ocf_log info "Reloading..."
+                volume_start
+                ;;
+validate-all)   ;;
+notify)		exit $OCF_SUCCESS;;
+*)              volume_usage
+                exit $OCF_ERR_UNIMPLEMENTED
+                ;;
+esac
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc"
+exit $rc
diff --git a/extras/peer_add_secret_pub.in b/extras/peer_add_secret_pub.in
new file mode 100644
index 00000000000..c9674af353d
--- /dev/null
+++ b/extras/peer_add_secret_pub.in
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+user=$1
+pub_file=$2
+
+if [ "$user" == "" ]; then
+    echo "Invalid User";
+    exit 1;
+fi
+
+if [ "$pub_file" == "" ]; then
+    echo "Invalid pub file";
+    exit 1;
+fi
+
+home_dir=`getent passwd $user | cut -d ':' -f 6`;
+
+if [ "$home_dir" == "" ]; then
+    echo "Invalid home dir";
+    exit 1;
+fi
+
+authorized_keys_file=$(cat /etc/ssh/sshd_config | \
+                              grep -e "^AuthorizedKeysFile" | \
+                              awk '{print $2}' | tail -1);
+
+# If not set, use default location
+if [ "x$authorized_keys_file" == "x" ]; then
+    authorized_keys_file="%h/.ssh/authorized_keys"
+fi
+
+# If default location
+if [ "$authorized_keys_file" == ".ssh/authorized_keys" ]; then
+    authorized_keys_file="%h/$authorized_keys_file"
+fi
+
+# Replace %u with user name (ex: /etc/ssh/keys/%u/authorized_keys)
+authorized_keys_file="${authorized_keys_file//%u/$user}";
+
+# Replace %h with home dir (ex: %h/.ssh/authorized_keys)
+authorized_keys_file="${authorized_keys_file//%h/$home_dir}";
+ssh_dir=$(dirname $authorized_keys_file);
+
+if [ ! -d $ssh_dir ]; then
+    mkdir $ssh_dir;
+    chmod 700 $ssh_dir;
+    chown $user: $ssh_dir;
+fi
+
+if [ ! -d $authorized_keys_file ]; then
+    touch $authorized_keys_file;
+    chmod 600 $authorized_keys_file;
+    chown $user: $authorized_keys_file;
+fi
+
+# Restore SELinux security contexts. This is required
+# for passwdless SSH to work.
+
+if type restorecon >/dev/null 2>&1; then
+    restorecon -F $ssh_dir $authorized_keys_file;
+fi
+
+# Add to authorized_keys file only if not exists already
+while read line
+do
+    grep -Fxq "$line" $authorized_keys_file;
+    [ $? -ne 0 ] && echo "$line" >> $authorized_keys_file;
+done < "$GLUSTERD_WORKDIR"/$pub_file;
+
+exit 0;
diff --git a/extras/post-upgrade-script-for-quota.sh b/extras/post-upgrade-script-for-quota.sh
new file mode 100755
index 00000000000..15652429056
--- /dev/null
+++ b/extras/post-upgrade-script-for-quota.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+#The post-upgrade script must be executed after all the nodes in the cluster
+#have upgraded.
+#Also, all the clients accessing the given volume must also be upgraded
+#before the script is run.
+#Make sure glusterd and the brick processes are running on all nodes in the
+#cluster post upgrade.
+#Execute this script on the node where the pre-upgrade script for quota was run.
+
+VOL=$1;
+
+BACKUP_DIR=/var/tmp/glusterfs/quota-config-backup
+
+function set_limits {
+        local var=$(gluster volume info $1 | grep 'features.quota'| cut -d" " -f2);
+
+        if  [ -z "$var" ] || [ "$var" = "off" ]; then
+                if [ $2 -eq '0' ]; then
+                        echo "Volume $1 does not have quota enabled. " \
+                             "Exiting ..."
+                        exit 1
+                fi
+        else
+                gluster volume set $1 default-soft-limit 80%
+                if [ $? -ne '0' ]; then
+                        echo "Post-upgrade process failed." \
+                             "Please address the error and run " \
+                             "post-upgrade-script.sh on volume $VOL again."
+                        exit 1
+                fi
+
+                gluster volume start $1 force
+                sleep 3;
+
+                local path_array=( $(cat $BACKUP_DIR/vol_$1 | tail -n +3 | awk '{print $1}') )
+                local limit_array=( $(cat $BACKUP_DIR/vol_$1 | tail -n +3 | awk '{print $2}') )
+                local len=${#path_array[@]}
+
+                for ((j=0; j<$len; j++))
+                    do
+                            gluster volume quota $1 limit-usage ${path_array[$j]} ${limit_array[j]};
+                            if [ $? -eq 0 ]; then
+                                    echo "Setting limit (${limit_array[j]}) on " \
+                                         "path ${path_array[$j]} has been " \
+                                         "successful"
+                            fi
+                    done
+        fi;
+}
+
+if [ -z $1 ]; then
+        echo "Please provide volume name or 'all'";
+        exit 1;
+fi
+
+if [ "$1" = "all" ]; then
+        for VOL in `gluster volume list`;
+            do
+                    set_limits $VOL '1';
+            done
+
+else
+        set_limits $1 '0';
+fi
diff --git a/extras/pre-upgrade-script-for-quota.sh b/extras/pre-upgrade-script-for-quota.sh
new file mode 100755
index 00000000000..9ff15f3b307
--- /dev/null
+++ b/extras/pre-upgrade-script-for-quota.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+#Make sure glusterd and the brick processes are running on all nodes in the
+#cluster.
+#This script must be run prior to upgradation, that too on
+#only one of the nodes in the cluster.
+
+BACKUP_DIR=/var/tmp/glusterfs/quota-config-backup
+
+mkdir -p $BACKUP_DIR
+for i in `gluster volume list`; do
+        var=$(gluster volume info $i | grep 'features.quota'| cut -d" " -f2);
+        if  [ -z "$var" ] || [ "$var" = "off" ]; then
+                continue
+        else
+                gluster volume quota $i list > $BACKUP_DIR/vol_$i;
+        fi;
+done
diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler
index f70db52286a..aaafd088648 100755
--- a/extras/profiler/glusterfs-profiler
+++ b/extras/profiler/glusterfs-profiler
@@ -1,22 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 
-# texttable - module for creating simple ASCII tables
-# Copyright (C) 2003-2010 Gerome Fournier <jefke(at)free.fr>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
+
+# texttable - module for creating simple ASCII tables
 # Incorporated from texttable.py downloaded from
 # http://jefke.free.fr/stuff/python/texttable/texttable-0.7.0.tar.gz
 
@@ -298,7 +291,7 @@ class Texttable:
         s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()],
             horiz)
         # build the line
-        l = string.join([horiz*n for n in self._width], s)
+        l = s.join([horiz*n for n in self._width])
         # add border if needed
         if self._has_border():
             l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz,
@@ -420,7 +413,7 @@ class Texttable:
         return line_wrapped
 
 
-#    Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+#    Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
 #    This file is part of GlusterFS.
 
 #    GlusterFS is free software; you can redistribute it and/or modify
diff --git a/extras/python/Makefile.am b/extras/python/Makefile.am
new file mode 100644
index 00000000000..7d81fa0319b
--- /dev/null
+++ b/extras/python/Makefile.am
@@ -0,0 +1,7 @@
+if HAVE_PYTHON
+# Install __init__.py into the Python site-packages area
+pypkgdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster
+pypkg_PYTHON = __init__.py
+endif
+
+EXTRA_DIST = __init__.py
diff --git a/extras/python/__init__.py b/extras/python/__init__.py
new file mode 100644
index 00000000000..3ad9513f40e
--- /dev/null
+++ b/extras/python/__init__.py
@@ -0,0 +1,2 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
diff --git a/extras/quota/contri-add.sh b/extras/quota/contri-add.sh
new file mode 100755
index 00000000000..7db5edd5d20
--- /dev/null
+++ b/extras/quota/contri-add.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# This script adds contributions of files/directories in backend to volume
+# size.
+# It can also be used to debug by passing dir as first argument, in which case
+# it will just add contributions from immediate children of a directory and
+# displays only if added contributions from immediate children is different
+# from size stored in directory.
+# For Eg., find <backend-directory> -type d -exec ./contri-add.sh dir \{} \;
+# will list all the directories which have descrepancies in their
+# size/contributions.
+
+usage ()
+{
+    echo >&2 "usage: $0 <file|dir> <list-of-backend-directories>"
+}
+
+add_contributions ()
+{
+    local var=0
+    local count=0
+
+    SIZE=`getfattr -h -e hex -n trusted.glusterfs.quota.size $2 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2`
+    CONTRI=`getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri $2 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2`
+
+    if [ $1 == "file" ]; then
+        PATHS=`find  $2 ! -type d | sed -e "\|^$2$|d" | sed -e '/^[ \t]*$/d'`
+    else
+        PATHS=`find  $2 -maxdepth 1 | sed -e "\|^$2$|d" | sed -e '/^[ \t]*$/d'`
+    fi
+
+    if [ -z "$PATHS" ]; then
+        return 0
+    fi
+
+    CONTRIBUTIONS=`echo $PATHS | xargs getfattr -h -e hex -d -m trusted.glusterfs.quota.*.contri 2>&1 | sed -e '/^#/d' | sed -e '/^getfattr/d' | sed -e '/^$/d' | cut -d'=' -f 2 | sed -e 's/^[ \t]*\([^ \t]*\)/\1/g'`
+
+    if [ -n "$CONTRIBUTIONS" ]; then
+        for i in $CONTRIBUTIONS; do
+            count=$(($count + 1))
+            var=$(($var + $i))
+        done
+    fi
+
+    if [ $1 == "file" ] || [ $var -ne $(($SIZE)) ] || [ $(($SIZE)) -ne $(($CONTRI)) ]; then
+        if [ $1 == "dir" ]; then
+            TMP_PATH=`echo $2 | sed -e "s/\/home\/export\/[0-9]*/\/mnt\/raghu/g"`
+            stat $TMP_PATH > /dev/null
+        fi
+
+        echo "file count $count"
+        echo "added contribution of $2=$var"
+        echo "size stored in xattrs on $2=$(($SIZE))"
+        echo "contribution of $2 to its parent directory=$(($CONTRI))"
+        echo "=============================================================="
+    fi
+}
+
+
+main ()
+{
+    [ $# -lt 1 ] && usage
+
+    TYPE=$1
+
+    shift 1
+
+    for i in $@; do
+        add_contributions $TYPE $i
+    done
+}
+
+main $@
+\ No newline at end of file
diff --git a/extras/quota/log_accounting.sh b/extras/quota/log_accounting.sh
new file mode 100755
index 00000000000..e2dd87b84d7
--- /dev/null
+++ b/extras/quota/log_accounting.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# The script does an accounting of all directories using command 'du' and
+# using gluster. We can then compare the two to identify accounting mismatch
+# THere can be minor mismatch because gluster only accounts for the size of
+# files. Direcotries can take up upto 4kB space on FS per directory. THis
+# size is accounted by du and not by gluster. However the difference would
+# not be significant.
+
+mountpoint=$1
+volname=$2
+
+usage ()
+{
+    echo >&2 "usage: $0 <mountpoint> <volume name>"
+    exit
+}
+
+[ $# -lt 2 ] && usage
+
+cd $mountpoint
+du -h | head -n -1 | tr -d '.' |awk  '{ for (i = 2; i <= NF; i++) { printf("%s ", $i);}  print "" }' > /tmp/gluster_quota_1
+cat /tmp/gluster_quota_1 | sed 's/ $//' | sed 's/ /\\ /g' | sed 's/(/\\(/g' | sed 's/)/\\)/g' |xargs gluster v quota $volname list > /tmp/gluster_quota_2
+du -h | head -n -1 |awk  '{ for (i = 2; i <= NF; i++) { printf("%s %s", $i, $1);}  print "" }' | tr -d '.' >  /tmp/gluster_quota_3
+cat /tmp/gluster_quota_2 /tmp/gluster_quota_3 | sort > /tmp/gluster_quota_4
+find . -type d > /tmp/gluster_quota_5
+tar -cvf /tmp/gluster_quota_files.tar /tmp/gluster_quota_*
diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py
new file mode 100755
index 00000000000..e62f7fc52a3
--- /dev/null
+++ b/extras/quota/quota_fsck.py
@@ -0,0 +1,377 @@
+#!/usr/bin/python3
+# The following script enables, Detecting, Reporting and Fixing
+# anomalies in quota accounting. Run this script with -h option
+# for further details.
+
+'''
+   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+'''
+from __future__ import print_function
+import os, sys, re
+from stat import *
+import subprocess
+import argparse
+import xattr
+
+aggr_size = {}
+verbose_mode = False
+mnt_path = None
+brick_path = None
+obj_fix_count = 0
+file_count = 0
+dir_count = 0
+
+#CONSTANTS
+KB = 1024
+MB = 1048576
+GB = 1048576 * 1024
+TB = 1048576 * 1048576
+
+QUOTA_VERBOSE = 0
+QUOTA_META_ABSENT = 1
+QUOTA_SIZE_MISMATCH = 2
+
+IS_DIRTY ='0x3100'
+IS_CLEAN ='0x3000'
+
+
+epilog_msg='''
+            The script attempts to find any gluster accounting issues in the
+            filesystem at the given subtree. The script crawls the given
+            subdirectory tree doing a stat for all files and compares the
+            size reported by gluster quota with the size reported by stat
+            calls. Any mismatch is reported. In addition integrity of marker
+            xattrs are verified.
+            '''
+
+def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None):
+    if log_type == QUOTA_VERBOSE:
+        print('%-24s %-60s\nxattr_values: %s\n%s\n' % ("Verbose", path, xattr_dict, stbuf))
+    elif log_type == QUOTA_META_ABSENT:
+        print('%-24s %-60s\n%s\n' % ("Quota-Meta Absent", path, xattr_dict))
+    elif log_type == QUOTA_SIZE_MISMATCH:
+        print("mismatch")
+        if dir_size is not None:
+            print('%24s %60s %12s %12s' % ("Size Mismatch", path,
+                xattr_dict, dir_size))
+        else:
+            print('%-24s %-60s %-12s %-12s' % ("Size Mismatch", path, xattr_dict,
+                   stbuf.st_size))
+
+def size_differs_lot(s1, s2):
+    '''
+    There could be minor accounting differences between the stat based
+    accounting and gluster accounting. To avoid these from throwing lot
+    of false positives in our logs. using a threshold of 1M for now.
+    TODO: For a deeply nested directory, at higher levels in hierarchy
+    differences may not be significant, hence this check needs to be improved.
+    '''
+    if abs(s1-s2) > 0:
+        return True
+    else:
+        return False
+
+def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size):
+    '''
+            Hard links are messy.. we have to account them for their parent
+            directory. But, stop accounting at the most common ancestor.
+            Eg:
+                say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3
+
+            suppose we encounter the hard links h1 first , then h2 and then h3.
+            while accounting for h1, we account the size until root(d2->d1->/)
+            while accounting for h2, we need to account only till d3. (as d1
+            and / are accounted for this inode).
+            while accounting for h3 we should not account at all.. as all
+            its ancestors are already accounted for same inode.
+
+            curr_dict                : dict of hardlinks that were seen and
+                                       accounted by the current iteration.
+            accounted_dict           : dict of hardlinks that has already been
+                                       accounted for.
+
+            size                     : size of the object as accounted by the
+                                       curr_iteration.
+
+            Return vale:
+            curr_size                : size reduced by hardlink sizes for those
+                                       hardlinks that has already been accounted
+                                       in current subtree.
+            Also delete the duplicate link from curr_dict.
+    '''
+
+    dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys())
+    for link in dual_accounted_links:
+        curr_size = curr_size - curr_dict[link]
+        del curr_dict[link]
+    return curr_size
+
+
+def fix_xattr(file_name, mark_dirty):
+    global obj_fix_count
+    global mnt_path
+
+    if mnt_path is None:
+        return
+    if mark_dirty:
+        print("MARKING DIRTY: " + file_name)
+        out = subprocess.check_output (["/usr/bin/setfattr", "-n",
+                                       "trusted.glusterfs.quota.dirty",
+                                       "-v", IS_DIRTY, file_name])
+    rel_path = os.path.relpath(file_name, brick_path)
+    print("stat on "  + mnt_path + "/" + rel_path)
+    stbuf = os.lstat(mnt_path + "/" + rel_path)
+
+    obj_fix_count += 1
+
+def get_quota_xattr_brick(dpath):
+    out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference",
+                                    "-d", "-m.", "-e", "hex", dpath])
+    pairs = out.splitlines()
+
+    '''
+    Sample output to be parsed:
+    [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/
+    # file: export/b1/B0/d14/d13/
+    security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000
+    trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8
+    trusted.glusterfs.dht=0x000000010000000000000000ffffffff
+    trusted.glusterfs.dht.mds=0x00000000
+    trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e
+    trusted.glusterfs.quota.dirty=0x3000
+    trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e
+    '''
+
+    '''
+    xattr_dict dictionary holds quota related xattrs
+    eg:
+    '''
+
+    xattr_dict = {}
+    xattr_dict['parents'] = {}
+
+    for xattr in pairs[1:]:
+        xattr = xattr.decode("utf-8")
+        xattr_key = xattr.split("=")[0]
+        if xattr_key == "":
+            # skip any empty lines
+            continue
+        elif not re.search("quota", xattr_key):
+            # skip all non quota xattr.
+            continue
+
+        xattr_value = xattr.split("=")[1]
+        if re.search("contri", xattr_key):
+
+            xattr_version = xattr_key.split(".")[5]
+            if 'version' not in xattr_dict:
+                xattr_dict['version'] = xattr_version
+            else:
+                if xattr_version != xattr_dict['version']:
+                   print("Multiple xattr version found")
+
+
+            cur_parent = xattr_key.split(".")[3]
+            if cur_parent not in xattr_dict['parents']:
+                xattr_dict['parents'][cur_parent] = {}
+
+            contri_dict = xattr_dict['parents'][cur_parent]
+            if len(xattr_value) == 34:
+                # 34 bytes implies file contri xattr
+                # contri format =0x< 16bytes file size><16bytes file count>
+                # size is obtained in iatt, file count = 1, dir count=0
+                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
+                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
+                contri_dict['contri_dir_count'] = 0
+            else:
+                # This is a directory contri.
+                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
+                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
+                contri_dict['contri_dir_count'] = int(xattr_value[34:], 16)
+
+        elif re.search("size", xattr_key):
+            xattr_dict['size'] = int(xattr_value[2:18], 16)
+            xattr_dict['file_count'] = int(xattr_value[18:34], 16)
+            xattr_dict['dir_count'] = int(xattr_value[34:], 16)
+        elif re.search("dirty", xattr_key):
+            if xattr_value == IS_CLEAN:
+                xattr_dict['dirty'] = False
+            elif xattr_value == IS_DIRTY:
+                xattr_dict['dirty'] = True
+        elif re.search("limit_objects", xattr_key):
+            xattr_dict['limit_objects'] = int(xattr_value[2:18], 16)
+        elif re.search("limit_set", xattr_key):
+            xattr_dict['limit_set'] = int(xattr_value[2:18], 16)
+
+    return xattr_dict
+
+def verify_file_xattr(path, stbuf = None):
+
+    global file_count
+    file_count += 1
+
+    if stbuf is None:
+        stbuf = os.lstat(path)
+
+    xattr_dict = get_quota_xattr_brick(path)
+
+    for parent in xattr_dict['parents']:
+        contri_dict = xattr_dict['parents'][parent]
+
+        if 'contri_size' not in contri_dict or \
+           'contri_file_count' not in contri_dict or \
+           'contri_dir_count' not in contri_dict:
+            print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf)
+            fix_xattr(path, False)
+            return
+        elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size):
+            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf)
+            fix_xattr(path, False)
+            return
+
+    if verbose_mode is True:
+        print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf)
+
+
+def verify_dir_xattr(path, dir_size):
+
+    global dir_count
+    dir_count += 1
+    xattr_dict = get_quota_xattr_brick(path)
+
+    stbuf = os.lstat(path)
+
+    for parent in xattr_dict['parents']:
+        contri_dict = xattr_dict['parents'][parent]
+
+        if 'size' not in xattr_dict or 'contri_size' not in contri_dict:
+            print_msg(QUOTA_META_ABSENT, path)
+            fix_xattr(path, True)
+            return
+        elif size_differs_lot(dir_size, xattr_dict['size']) or \
+             size_differs_lot(contri_dict['contri_size'], xattr_dict['size']):
+            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size)
+            fix_xattr(path, True)
+            return
+
+    if verbose_mode is True:
+        print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size)
+
+
+def walktree(t_dir, hard_link_dict):
+    '''recursively descend the directory tree rooted at dir,
+       aggregating the size
+       t_dir            : directory to walk over.
+       hard_link_dict   : dict of inodes with multiple hard_links under t_dir
+    '''
+    global aggr_size
+    aggr_size[t_dir] = 0
+
+    for entry in os.listdir(t_dir):
+        pathname = os.path.join(t_dir, entry)
+        stbuf = os.lstat(pathname)
+        if S_ISDIR(stbuf.st_mode):
+            # It's a directory, recurse into it
+            if entry == '.glusterfs':
+                print("skipping " + pathname)
+                continue
+            descendent_hardlinks = {}
+            subtree_size = walktree(pathname, descendent_hardlinks)
+
+            subtree_size = fix_hardlink_accounting(descendent_hardlinks,
+                                                   hard_link_dict,
+                                                   subtree_size)
+
+            aggr_size[t_dir] = aggr_size[t_dir] + subtree_size
+
+        elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode):
+            # Even a symbolic link file may have multiple hardlinks.
+
+            file_size = stbuf.st_size
+            if stbuf.st_nlink > 2:
+                # send a single element dict to check if file is accounted.
+                file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size},
+                                                    hard_link_dict,
+                                                    stbuf.st_size)
+
+                if file_size == 0:
+                    print_msg("HARD_LINK (skipped)", pathname, "",
+                                stbuf)
+                else:
+                    print_msg("HARD_LINK (accounted)", pathname, "",
+                               stbuf)
+                    hard_link_dict[stbuf.st_ino] = stbuf.st_size
+
+            if t_dir in aggr_size:
+                aggr_size[t_dir] = aggr_size[t_dir] + file_size
+            else:
+                aggr_size[t_dir] = file_size
+            verify_file_xattr(pathname, stbuf)
+
+        else:
+            # Unknown file type, print a message
+            print('Skipping %s, due to file mode' % (pathname))
+
+    if t_dir not in aggr_size:
+        aggr_size[t_dir] = 0
+
+    verify_dir_xattr(t_dir, aggr_size[t_dir])
+    # du also accounts for t_directory sizes
+    # aggr_size[t_dir] += 4096
+
+    #cleanup
+    ret = aggr_size[t_dir]
+    del aggr_size[t_dir]
+    return ret
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg)
+    parser.add_argument('brick_path', nargs=1,
+                        help='The brick path (or any descendent sub-directory of brick path)',
+                        )
+    parser.add_argument('--full-logs', dest='verbose', action='store_true',
+                   help='''
+                         log all the xattr values and stat values reported
+                         for analysis. [CAUTION: This can give lot of output
+                         depending on FS depth. So one has to make sure enough
+                         disk space exists if redirecting to file]
+                        '''
+                        )
+    parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store',
+                   help='''
+                         fix accounting issues where the xattr values disagree
+                         with stat sizes reported by gluster. A mount is also
+                         required for this option to be used.
+                         [CAUTION: This will directly modify backend xattr]
+                        '''
+                        )
+    parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store',
+                   help='''
+                         limit the crawling and accounting verification/correction
+                         to a specific subdirectory.
+                        '''
+                        )
+
+    args = parser.parse_args()
+    verbose_mode = args.verbose
+    brick_path = args.brick_path[0]
+    sub_dir = args.sub_dir
+    mnt_path = args.mnt
+    hard_link_dict = {}
+    if sub_dir is not None:
+        walktree(os.path.join(brick_path, sub_dir), hard_link_dict)
+    else:
+        walktree(brick_path, hard_link_dict)
+
+    print("Files verified : " + str(file_count))
+    print("Directories verified : " + str(dir_count))
+    if mnt_path is not None:
+        print("Objects Fixed : " + str(obj_fix_count))
diff --git a/extras/quota/xattr_analysis.py b/extras/quota/xattr_analysis.py
new file mode 100755
index 00000000000..7bd7d96374c
--- /dev/null
+++ b/extras/quota/xattr_analysis.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# Below script has two purposes
+#  1. Display xattr of entire FS tree in a human readable form
+#  2. Display all the directory where contri and size mismatch.
+#      (If there are any directory with contri and size mismatch that are not dirty
+#       then that highlights a propagation issue)
+#  The script takes only one input LOG _FILE generated from the command,
+#  find <brick_path> | xargs  getfattr -d -m. -e hex  > log_gluster_xattr
+
+from __future__ import print_function
+import re
+import subprocess
+import sys
+from hurry.filesize import size
+
+if len(sys.argv) < 2:
+    sys.exit('Usage: %s log_gluster_xattr \n'
+              'to generate log_gluster_xattr use: \n'
+              'find <brick_path> | xargs  getfattr -d -m. -e hex  > log_gluster_xattr'
+              % sys.argv[0])
+LOG_FILE=sys.argv[1]
+
+def get_quota_xattr_brick():
+    out = subprocess.check_output (["/usr/bin/cat", LOG_FILE])
+    pairs = out.splitlines()
+
+    xdict = {}
+    mismatch_size = [('====contri_size===', '====size====')]
+    for xattr in pairs:
+        k = xattr.split("=")[0]
+        if re.search("# file:", k):
+            print(xdict)
+            filename=k
+            print("=====" + filename + "=======")
+            xdict = {}
+        elif k is "":
+            pass
+        else:
+            print(xattr)
+            v = xattr.split("=")[1]
+            if re.search("contri", k):
+                if len(v) == 34:
+                    # for files size is obtained in iatt, file count should be 1, dir count=0
+                    xdict['contri_file_count'] = int(v[18:34], 16)
+                    xdict['contri_dir_count'] = 0
+                else:
+                    xdict['contri_size'] = size(int(v[2:18], 16))
+                    xdict['contri_file_count'] = int(v[18:34], 16)
+                    xdict['contri_dir_count'] = int(v[34:], 16)
+            elif re.search("size", k):
+                xdict['size'] = size(int(v[2:18], 16))
+                xdict['file_count'] = int(v[18:34], 16)
+                xdict['dir_count'] = int(v[34:], 16)
+            elif re.search("dirty", k):
+                if v == '0x3000':
+                    xdict['dirty'] = False
+                elif v == '0x3100':
+                    xdict['dirty'] = True
+            elif re.search("limit_objects", k):
+                xdict['limit_objects'] = int(v[2:18], 16)
+            elif re.search("limit_set", k):
+                xdict['limit_set'] = size(int(v[2:18], 16))
+
+            if 'size' in xdict and 'contri_size' in xdict and xdict['size'] != xdict['contri_size']:
+                mismatch_size.append((xdict['contri_size'], xdict['size'], filename))
+
+    for values in mismatch_size:
+        print(values)
+
+
+if __name__ == '__main__':
+    get_quota_xattr_brick()
+
diff --git a/extras/rebalance.py b/extras/rebalance.py
new file mode 100755
index 00000000000..37c68ebbb42
--- /dev/null
+++ b/extras/rebalance.py
@@ -0,0 +1,309 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import atexit
+import copy
+import optparse
+import os
+import pipes
+import shutil
+import string
+import subprocess
+import sys
+import tempfile
+import volfilter
+import platform
+
+# It's just more convenient to have named fields.
+class Brick:
+        def __init__ (self, path, name):
+                self.path = path
+                self.sv_name = name
+                self.size = 0
+                self.curr_size = 0
+                self.good_size = 0
+        def set_size (self, size):
+                self.size = size
+        def set_range (self, rs, re):
+                self.r_start = rs
+                self.r_end = re
+                self.curr_size = self.r_end - self.r_start + 1
+        def __repr__ (self):
+                value = self.path[:]
+                value += "(%d," % self.size
+                if self.curr_size:
+                        value += "0x%x,0x%x)" % (self.r_start, self.r_end)
+                else:
+                        value += "-)"
+                return value
+
+def get_bricks (host, vol):
+        t = pipes.Template()
+        t.prepend("gluster --remote-host=%s system getspec %s"%(host, vol), ".-")
+        return t.open(None, "r")
+
+def generate_stanza (vf, all_xlators, cur_subvol):
+        sv_list = []
+        for sv in cur_subvol.subvols:
+                generate_stanza(vf, all_xlators, sv)
+                sv_list.append(sv.name)
+        vf.write("volume %s\n" % cur_subvol.name)
+        vf.write("  type %s\n" % cur_subvol.type)
+        for kvpair in cur_subvol.opts.items():
+                vf.write("  option %s %s\n" % kvpair)
+        if sv_list:
+                vf.write("  subvolumes %s\n" % ''.join(sv_list))
+        vf.write("end-volume\n\n")
+
+
+def mount_brick (localpath, all_xlators, dht_subvol):
+
+        # Generate a volfile.
+        vf_name = localpath + ".vol"
+        vf = open(vf_name, "w")
+        generate_stanza(vf, all_xlators, dht_subvol)
+        vf.flush()
+        vf.close()
+
+        # Create a brick directory and mount the brick there.
+        os.mkdir(localpath)
+        subprocess.call(["glusterfs", "-f", vf_name, localpath])
+
+# We use the command-line tools because there's no getxattr support in the
+# Python standard library (which is ridiculous IMO).  Adding the xattr package
+# from PyPI would create a new and difficult dependency because the bits to
+# satisfy it don't seem to exist in Fedora.  We already expect the command-line
+# tools to be there, so it's safer just to rely on them.
+#
+# We might have to revisit this if we get as far as actually issuing millions
+# of setxattr requests.  Even then, it might be better to do that part with a C
+# program which has only a build-time dependency.
+def get_range (brick):
+        t = pipes.Template()
+        cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null"
+        t.prepend(cmd%brick, ".-")
+        t.append("grep ^trusted.glusterfs.dht=", "--")
+        f = t.open(None, "r")
+        try:
+                value = f.readline().rstrip().split('=')[1][2:]
+        except:
+                print("could not get layout for %s (might be OK)" % brick)
+                return None
+        v_start = int("0x"+value[16:24], 16)
+        v_end = int("0x"+value[24:32], 16)
+        return (v_start, v_end)
+
+def calc_sizes (bricks, total):
+        leftover = 1 << 32
+        for b in bricks:
+               if b.size:
+                        b.good_size = (b.size << 32) / total
+                        leftover -= b.good_size
+               else:
+                        b.good_size = 0
+        if leftover:
+                # Add the leftover to an old brick if we can.
+                for b in bricks:
+                        if b.good_size:
+                                b.good_size += leftover
+                                break
+                else:
+                        # Fine, just add it wherever.
+                        bricks[0].good_size += leftover
+
+# Normalization means sorting the bricks by r_start and (b) ensuring that there
+# are no gaps.
+def normalize (in_bricks):
+        out_bricks = []
+        curr_hash = 0
+        used = 0
+        while curr_hash < (1<<32):
+                curr_best = None
+                for b in in_bricks:
+                        if b.r_start == curr_hash:
+                                used += 1
+                                out_bricks.append(b)
+                                in_bricks.remove(b)
+                                curr_hash = b.r_end + 1
+                                break
+                else:
+                        print("gap found at 0x%08x" % curr_hash)
+                        sys.exit(1)
+        return out_bricks + in_bricks, used
+
+def get_score (bricks):
+        score = 0
+        curr_hash = 0
+        for b in bricks:
+                if not b.curr_size:
+                        curr_hash += b.good_size
+                        continue
+                new_start = curr_hash
+                curr_hash += b.good_size
+                new_end = curr_hash - 1
+                if new_start > b.r_start:
+                        max_start = new_start
+                else:
+                        max_start = b.r_start
+                if new_end < b.r_end:
+                        min_end = new_end
+                else:
+                        min_end = b.r_end
+                if max_start <= min_end:
+                        score += (min_end - max_start + 1)
+        return score
+
+if __name__ == "__main__":
+
+        my_usage = "%prog [options] server volume [directory]"
+        parser = optparse.OptionParser(usage=my_usage)
+        parser.add_option("-f", "--free-space", dest="free_space",
+                          default=False, action="store_true",
+                          help="use free space instead of total space")
+        parser.add_option("-l", "--leave-mounted", dest="leave_mounted",
+                          default=False, action="store_true",
+                          help="leave subvolumes mounted")
+        parser.add_option("-v", "--verbose", dest="verbose",
+                          default=False, action="store_true",
+                          help="verbose output")
+        options, args = parser.parse_args()
+
+        if len(args) == 3:
+                fix_dir = args[2]
+        else:
+                if len(args) != 2:
+                        parser.print_help()
+                        sys.exit(1)
+                fix_dir = None
+        hostname, volname = args[:2]
+
+        # Make sure stuff gets cleaned up, even if there are exceptions.
+        orig_dir = os.getcwd()
+        work_dir = tempfile.mkdtemp()
+        bricks = []
+        def cleanup_workdir ():
+                os.chdir(orig_dir)
+                if options.verbose:
+                        print("Cleaning up %s" % work_dir)
+                for b in bricks:
+                        subprocess.call(["umount", b.path])
+                shutil.rmtree(work_dir)
+        if not options.leave_mounted:
+                atexit.register(cleanup_workdir)
+        os.chdir(work_dir)
+
+        # Mount each brick individually, so we can issue brick-specific calls.
+        if options.verbose:
+                print("Mounting subvolumes...")
+        index = 0
+        volfile_pipe = get_bricks(hostname, volname)
+        all_xlators, last_xlator = volfilter.load(volfile_pipe)
+        for dht_vol in all_xlators.itervalues():
+                if dht_vol.type == "cluster/distribute":
+                        break
+        else:
+                print("no DHT volume found")
+                sys.exit(1)
+        for sv in dht_vol.subvols:
+                #print "found subvol %s" % sv.name
+                lpath = "%s/brick%s" % (work_dir, index)
+                index += 1
+                mount_brick(lpath, all_xlators, sv)
+                bricks.append(Brick(lpath, sv.name))
+        if index == 0:
+                print("no bricks")
+                sys.exit(1)
+
+        # Collect all of the sizes.
+        if options.verbose:
+                print("Collecting information...")
+        total = 0
+        for b in bricks:
+                info = os.statvfs(b.path)
+                # On FreeBSD f_bsize (info[0]) contains the optimal I/O size,
+                # not the block size as it's found on Linux. In this case we
+                # use f_frsize (info[1]).
+                if platform.system() == 'FreeBSD':
+                        bsize = info[1]
+                else:
+                        bsize = info[0]
+                # We want a standard unit even if different bricks use
+                # different block sizes.  The size is chosen to avoid overflows
+                # for very large bricks with very small block sizes, but also
+                # accommodate filesystems which use very large block sizes to
+                # cheat on benchmarks.
+                blocksper100mb = 104857600 / bsize
+                if options.free_space:
+                        size = info[3] / blocksper100mb
+                else:
+                        size = info[2] / blocksper100mb
+                if size <= 0:
+                        print("brick %s has invalid size %d" % (b.path, size))
+                        sys.exit(1)
+                b.set_size(size)
+                total += size
+
+        # Collect all of the layout information.
+        for b in bricks:
+                hash_range = get_range(b.path)
+                if hash_range is not None:
+                        rs, re = hash_range
+                        if rs > re:
+                                print("%s has backwards hash range" % b.path)
+                                sys.exit(1)
+                        b.set_range(hash_range[0], hash_range[1])
+
+        if options.verbose:
+                print("Calculating new layouts...")
+        calc_sizes(bricks, total)
+        bricks, used = normalize(bricks)
+
+        # We can't afford O(n!) here, but O(n^2) should be OK and the result
+        # should be almost as good.
+        while used < len(bricks):
+                best_place = used
+                best_score = get_score(bricks)
+                for i in range(used):
+                        new_bricks = bricks[:]
+                        del new_bricks[used]
+                        new_bricks.insert(i, bricks[used])
+                        new_score = get_score(new_bricks)
+                        if new_score > best_score:
+                                best_place = i
+                                best_score = new_score
+                if best_place != used:
+                        nb = bricks[used]
+                        del bricks[used]
+                        bricks.insert(best_place, nb)
+                used += 1
+
+        # Finalize whatever we decided on.
+        curr_hash = 0
+        for b in bricks:
+                b.r_start = curr_hash
+                curr_hash += b.good_size
+                b.r_end = curr_hash - 1
+
+        print("Here are the xattr values for your size-weighted layout:")
+        for b in bricks:
+                print("  %s: 0x0000000200000000%08x%08x" % (
+                        b.sv_name, b.r_start, b.r_end))
+
+        if fix_dir:
+                if options.verbose:
+                        print("Fixing layout for %s" % fix_dir)
+                for b in bricks:
+                        value = "0x0000000200000000%08x%08x" % (
+                                b.r_start, b.r_end)
+                        path = "%s/%s" % (b.path, fix_dir)
+                        cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % (
+                                value, path)
+                        print(cmd)
+
+        if options.leave_mounted:
+                print("The following subvolumes are still mounted:")
+                for b in bricks:
+                        print("%s on %s" % (b.sv_name, b.path))
+                print("Don't forget to clean up when you're done.")
+
diff --git a/extras/run-gluster.tmpfiles.in b/extras/run-gluster.tmpfiles.in
new file mode 100644
index 00000000000..329f2dde6db
--- /dev/null
+++ b/extras/run-gluster.tmpfiles.in
@@ -0,0 +1,2 @@
+# hardcoding /run for now, should be detected while building from source?
+d /run/gluster 0775 gluster gluster -
diff --git a/extras/snap_scheduler/Makefile.am b/extras/snap_scheduler/Makefile.am
new file mode 100644
index 00000000000..782f139016f
--- /dev/null
+++ b/extras/snap_scheduler/Makefile.am
@@ -0,0 +1,9 @@
+snap_schedulerdir = $(sbindir)/
+
+if WITH_SERVER
+snap_scheduler_SCRIPTS = gcron.py snap_scheduler.py conf.py
+endif
+
+EXTRA_DIST = gcron.py snap_scheduler.py conf.py
+
+CLEANFILES =
diff --git a/extras/snap_scheduler/README.md b/extras/snap_scheduler/README.md
new file mode 100644
index 00000000000..1316bb76469
--- /dev/null
+++ b/extras/snap_scheduler/README.md
@@ -0,0 +1,125 @@
+Snapshot Scheduler
+==============================
+
+SUMMARY
+-------
+
+GlusterFS volume snapshot provides point-in-time copy of a GlusterFS volume. Currently, GlusterFS volume snapshots can be easily scheduled by setting up cron jobs on one of the nodes in the GlusterFS trusted storage pool. This has a single point failure (SPOF), as scheduled jobs can be missed if the node running the cron jobs dies.
+
+We can avoid the SPOF by distributing the cron jobs to all nodes of the trusted storage pool.
+
+DETAILED DESCRIPTION
+--------------------
+
+The solution to the above problems involves the usage of:
+
+* A shared storage - This can be any shared storage (another gluster volume, a NFS mount, etc.) that will be used to share the schedule configuration and will help in the coordination of the jobs.
+* An agent - This agent will perform the actual snapshot commands, instead of cron. It will contain the logic to perform coordinated snapshots.
+* A helper script - This script will allow the user to initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules.
+* cronie - It is the default cron daemon shipped with RHEL. It invokes the agent at the appropriate intervals as mentioned by the user to perform the snapshot operation on the volume as mentioned by the user in the schedule.
+
+INITIAL SETUP
+-------------
+
+The administrator needs to create a shared storage that can be available to nodes across the cluster. A GlusterFS volume can also be used for the same. It is preferable that the *shared volume* be a replicate volume to avoid SPOF.
+
+Once the shared storage is created, it should be mounted on all nodes in the trusted storage pool which will be participating in the scheduling. The location where the shared_storage should be mounted (/var/run/gluster/snaps/shared_storage) in these nodes is fixed and is not configurable. Each node participating in the scheduling then needs to perform an initialisation of the snapshot scheduler by invoking the following:
+
+snap_scheduler.py init
+
+NOTE: This command needs to be run on all the nodes participating in the scheduling
+
+HELPER SCRIPT
+-------------
+
+The helper script(snap_scheduler.py) will initialise the scheduler on the local node, enable/disable scheduling, add/edit/list/delete snapshot schedules.
+
+a) snap_scheduler.py init
+
+This command initialises the snap_scheduler and interfaces it with the crond running on the local node. This is the first step, before executing any scheduling related commands from a node.
+
+NOTE: The helper script needs to be run with this option on all the nodes participating in the scheduling. Other options of the helper script can be run independently from any node, where initialisation has been successfully completed.
+
+b) snap_scheduler.py enable
+
+The snap scheduler is disabled by default after initialisation. This command enables the snap scheduler.
+
+c) snap_scheduler.py disable
+
+This command disables the snap scheduler.
+
+d) snap_scheduler.py status
+
+This command displays the current status(Enabled/Disabled) of the snap scheduler.
+
+e) snap_scheduler.py add "Job Name" "Schedule" "Volume Name"
+
+This command adds a new snapshot schedule. All the arguments must be provided within double-quotes(""). It takes three arguments:
+
+-> Job Name: This name uniquely identifies this particular schedule, and can be used to reference this schedule for future events like edit/delete. If a schedule already exists for the specified Job Name, the add command will fail.
+
+-> Schedule: The schedules are accepted in the format crond understands:-
+
+Example of job definition:
+.---------------- minute (0 - 59)
+| .------------- hour (0 - 23)
+| | .---------- day of month (1 - 31)
+| | | .------- month (1 - 12) OR jan,feb,mar,apr ...
+| | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
+| | | | |
+* * * * * user-name command to be executed
+Although we accept all valid cron schedules, currently we support granularity of snapshot schedules to a maximum of half-hourly snapshots.
+
+-> Volume Name: The name of the volume on which the scheduled snapshot operation will be performed.
+
+f) snap_scheduler.py edit "Job Name" "Schedule" "Volume Name"
+
+This command edits an existing snapshot schedule. It takes the same three arguments that the add option takes. All the arguments must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the edit command will fail.
+
+g) snap_scheduler.py delete "Job Name"
+
+This command deletes an existing snapshot schedule. It takes the job name of the schedule as argument. The argument must be provided within double-quotes(""). If a schedule does not exists for the specified Job Name, the delete command will fail.
+
+h) snap_scheduler.py list
+
+This command lists the existing snapshot schedules in the following manner: Pseudocode:
+
+# snap_scheduler.py list
+JOB_NAME         SCHEDULE         OPERATION        VOLUME NAME
+--------------------------------------------------------------------
+Job0             * * * * *        Snapshot Create  test_vol
+
+THE AGENT
+---------
+
+The snapshots scheduled with the help of the helper script, are read by crond which then invokes the agent(gcron.py) at the scheduled intervals to perform the snapshot operations on the specified volumes. It then performs the scheduled snapshots using the following algorithm to coordinate.
+
+start_time = get current time
+lock_file = job_name passed as an argument
+vol_name = volume name psased as an argument
+try POSIX locking the $lock_file
+    if lock is obtained, then
+        mod_time = Get modification time of $entry
+        if $mod_time < $start_time, then
+            Take snapshot of $entry.name (Volume name)
+            if snapshot failed, then
+                log the failure
+            Update modification time of $entry to current time
+        unlock the $entry
+
+The coordination with other scripts running on other nodes, is handled by the use of POSIX locks. All the instances of the script will attempt to lock the lock_file which is essentialy an empty file with the job name, and one which gets the lock will take the snapshot.
+
+To prevent redoing a done task, the script will make use of the mtime attribute of the entry. At the beginning execution, the script would have saved its start time. Once the script obtains the lock on the lock_file, before taking the snapshot, it compares the mtime of the entry with the start time. The snapshot will only be taken if the mtime is smaller than start time. Once the snapshot command completes, the script will update the mtime of the lock_file to the current time before unlocking.
+
+If a snapshot command fails, the script will log the failure (in syslog) and continue with its operation. It will not attempt to retry the failed snapshot in the current schedule, but will attempt it again in the next schedules. It is left to the administrator to monitor the logs and decide what to do after a failure.
+
+ASSUMPTIONS AND LIMITATIONS
+---------------------------
+
+It is assumed that all nodes in the have their times synced using NTP or any other mechanism. This is a hard requirement for this feature to work.
+
+The administrator needs to have python2.7 or higher installed, as well as the argparse module installed, to be able to use the helper script(snap_scheduler.py).
+
+There is a latency of one minute, between providing a command by the helper script and that command taking effect. Hence, currently we do not support snapshot schedules with per minute granularity.
+
+The administrator can however leverage the scheduler to schedule snapshots with granularity of half-hourly/hourly/daily/weekly/monthly/yearly periodic intervals. They can also schedule snapshots, which are customised mentioning which minute of the hour, which day of the week, which week of the month, and which month of the year, they want to schedule the snapshot operation.
diff --git a/extras/snap_scheduler/conf.py.in b/extras/snap_scheduler/conf.py.in
new file mode 100644
index 00000000000..6dcca0534a7
--- /dev/null
+++ b/extras/snap_scheduler/conf.py.in
@@ -0,0 +1,11 @@
+#
+# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+GLUSTERFS_LIBEXECDIR = '@GLUSTERFS_LIBEXECDIR@'
diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py
new file mode 100755
index 00000000000..0e4df77d481
--- /dev/null
+++ b/extras/snap_scheduler/gcron.py
@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+#
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+
+from __future__ import print_function
+import subprocess
+import os
+import os.path
+import sys
+import time
+import logging
+import logging.handlers
+import fcntl
+
+
+GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks"
+GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks"
+GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag"
+LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/"
+log = logging.getLogger("gcron-logger")
+start_time = 0.0
+
+
+def initLogger(script_name):
+    log.setLevel(logging.DEBUG)
+    logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\
+        "%(levelname)s %(message)s"
+    formatter = logging.Formatter(logFormat)
+
+    sh = logging.handlers.SysLogHandler()
+    sh.setLevel(logging.ERROR)
+    sh.setFormatter(formatter)
+
+    process = subprocess.Popen(["gluster", "--print-logdir"],
+                               stdout=subprocess.PIPE,
+                               universal_newlines=True)
+    out, err = process.communicate()
+    if process.returncode == 0:
+        logfile = os.path.join(out.strip(), script_name[:-3]+".log")
+
+    fh = logging.FileHandler(logfile)
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(formatter)
+
+    log.addHandler(sh)
+    log.addHandler(fh)
+
+
+def takeSnap(volname="", snapname=""):
+    success = True
+    if volname == "":
+        log.debug("No volname given")
+        return False
+    if snapname == "":
+        log.debug("No snapname given")
+        return False
+
+    cli = ["gluster",
+           "snapshot",
+           "create",
+           snapname,
+           volname]
+    log.debug("Running command '%s'", " ".join(cli))
+
+    p = subprocess.Popen(cli, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    rv = p.returncode
+
+    log.debug("Command '%s' returned '%d'", " ".join(cli), rv)
+
+    if rv:
+        log.error("Snapshot of %s failed", volname)
+        log.error("Command output:")
+        log.error(err)
+        success = False
+    else:
+        log.info("Snapshot of %s successful", volname)
+
+    return success
+
+
+def doJob(name, lockFile, jobFunc, volname):
+    success = True
+    try:
+        f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK)
+        try:
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            mtime = os.path.getmtime(lockFile)
+            global start_time
+            log.debug("%s last modified at %s", lockFile, time.ctime(mtime))
+            if mtime < start_time:
+                log.debug("Processing job %s", name)
+                if jobFunc(volname, name):
+                    log.info("Job %s succeeded", name)
+                else:
+                    log.error("Job %s failed", name)
+                    success = False
+                os.utime(lockFile, None)
+            else:
+                log.info("Job %s has been processed already", name)
+            fcntl.flock(f, fcntl.LOCK_UN)
+        except (OSError, IOError):
+            log.info("Job %s is being processed by another agent", name)
+        os.close(f)
+    except (OSError, IOError) as e:
+        log.debug("Failed to open lock file %s : %s", lockFile, e)
+        log.error("Failed to process job %s", name)
+        success = False
+
+    return success
+
+
+def main():
+    script_name = os.path.basename(__file__)
+    initLogger(script_name)
+    global start_time
+    if sys.argv[1] == "--update":
+        if not os.path.exists(GCRON_TASKS):
+            # Create a flag in /var/run/gluster which indicates that this
+            # node doesn't have access to GCRON_TASKS right now, so that
+            # when the mount is available and GCRON_TASKS is available
+            # the flag will tell this routine to reload GCRON_CROND_TASK
+            try:
+                f = os.open(GCRON_RELOAD_FLAG,
+                            os.O_CREAT | os.O_NONBLOCK, 0o644)
+                os.close(f)
+            except OSError as e:
+                if errno != EEXIST:
+                    log.error("Failed to create %s : %s",
+                              GCRON_RELOAD_FLAG, e)
+                    output("Failed to create %s. Error: %s"
+                           % (GCRON_RELOAD_FLAG, e))
+            return
+
+        if not os.path.exists(GCRON_CROND_TASK):
+            return
+
+        # As GCRON_TASKS exists now, we should check if GCRON_RELOAD_FLAG
+        # also exists. If so we should touch GCRON_CROND_TASK and remove
+        # the GCRON_RELOAD_FLAG
+        if os.path.exists(GCRON_RELOAD_FLAG):
+            try:
+                os.remove(GCRON_RELOAD_FLAG);
+                process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK],
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.PIPE)
+                out, err = process.communicate()
+                if process.returncode != 0:
+                    log.error("Failed to touch %s. Error: %s.",
+                              GCRON_CROND_TASK, err)
+            except (IOError, OSError) as e:
+                log.error("Failed to touch %s. Error: %s.",
+                          GCRON_CROND_TASK, e)
+            return
+        if os.lstat(GCRON_TASKS).st_mtime > \
+           os.lstat(GCRON_CROND_TASK).st_mtime:
+            try:
+                process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK],
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.PIPE)
+                out, err = process.communicate()
+                if process.returncode != 0:
+                    log.error("Failed to touch %s. Error: %s.",
+                              GCRON_CROND_TASK, err)
+            except IOError as e:
+                log.error("Failed to touch %s. Error: %s.",
+                          GCRON_CROND_TASK, e)
+        return
+
+    volname = sys.argv[1]
+    jobname = sys.argv[2]
+    locking_file = os.path.join(LOCK_FILE_DIR, jobname)
+    log.debug("locking_file = %s", locking_file)
+    log.debug("volname = %s", volname)
+    log.debug("jobname = %s", jobname)
+
+    start_time = int(time.time())
+
+    doJob("Scheduled-" + jobname + "-" + volname, locking_file, takeSnap, volname)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py
new file mode 100755
index 00000000000..e8fcc449a9b
--- /dev/null
+++ b/extras/snap_scheduler/snap_scheduler.py
@@ -0,0 +1,941 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+#
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+
+from __future__ import print_function
+import subprocess
+import os
+import os.path
+import logging
+import argparse
+import fcntl
+import logging.handlers
+import sys
+import shutil
+from errno import EEXIST
+from conf import GLUSTERFS_LIBEXECDIR
+sys.path.insert(1, GLUSTERFS_LIBEXECDIR)
+
+EVENTS_ENABLED = True
+try:
+    from events.eventtypes import SNAPSHOT_SCHEDULER_INITIALISED \
+                         as EVENT_SNAPSHOT_SCHEDULER_INITIALISED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_INIT_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLED \
+                         as EVENT_SNAPSHOT_SCHEDULER_DISABLED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_DISABLE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLED \
+                         as EVENT_SNAPSHOT_SCHEDULER_ENABLED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_ENABLE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADDED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDITED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED
+    from events.eventtypes import SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED \
+                         as EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED
+except ImportError:
+    # Events APIs not installed, dummy eventtypes with None
+    EVENTS_ENABLED = False
+    EVENT_SNAPSHOT_SCHEDULER_INITIALISED = None
+    EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_DISABLED = None
+    EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_ENABLED = None
+    EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED = None
+    EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED = None
+
+SCRIPT_NAME = "snap_scheduler"
+scheduler_enabled = False
+log = logging.getLogger(SCRIPT_NAME)
+SHARED_STORAGE_DIR="/run/gluster/shared_storage"
+GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled"
+GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled"
+GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks"
+GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks"
+LOCK_FILE_DIR = SHARED_STORAGE_DIR+"/snaps/lock_files/"
+LOCK_FILE = LOCK_FILE_DIR+"lock_file"
+TMP_FILE = SHARED_STORAGE_DIR+"/snaps/tmp_file"
+GCRON_UPDATE_TASK = "/etc/cron.d/gcron_update_task"
+CURRENT_SCHEDULER = SHARED_STORAGE_DIR+"/snaps/current_scheduler"
+tasks = {}
+longest_field = 12
+current_scheduler = ""
+
+INTERNAL_ERROR = 2
+SHARED_STORAGE_DIR_DOESNT_EXIST = 3
+SHARED_STORAGE_NOT_MOUNTED = 4
+ANOTHER_TRANSACTION_IN_PROGRESS = 5
+INIT_FAILED = 6
+SCHEDULING_ALREADY_DISABLED = 7
+SCHEDULING_ALREADY_ENABLED = 8
+NODE_NOT_INITIALISED = 9
+ANOTHER_SCHEDULER_ACTIVE = 10
+JOB_ALREADY_EXISTS = 11
+JOB_NOT_FOUND = 12
+INVALID_JOBNAME = 13
+INVALID_VOLNAME = 14
+INVALID_SCHEDULE = 15
+INVALID_ARG = 16
+VOLUME_DOES_NOT_EXIST = 17
+
+def print_error (error_num):
+    if error_num == INTERNAL_ERROR:
+        return "Internal Error"
+    elif error_num == SHARED_STORAGE_DIR_DOESNT_EXIST:
+        return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \
+               " does not exist."
+    elif error_num == SHARED_STORAGE_NOT_MOUNTED:
+        return "The shared storage directory ("+SHARED_STORAGE_DIR+")" \
+               " is not mounted."
+    elif error_num == ANOTHER_TRANSACTION_IN_PROGRESS:
+        return "Another transaction is in progress."
+    elif error_num == INIT_FAILED:
+        return "Initialisation failed."
+    elif error_num == SCHEDULING_ALREADY_DISABLED:
+        return "Snapshot scheduler is already disabled."
+    elif error_num == SCHEDULING_ALREADY_ENABLED:
+        return "Snapshot scheduler is already enabled."
+    elif error_num == NODE_NOT_INITIALISED:
+        return "The node is not initialised."
+    elif error_num == ANOTHER_SCHEDULER_ACTIVE:
+        return "Another scheduler is active."
+    elif error_num == JOB_ALREADY_EXISTS:
+        return "The job already exists."
+    elif error_num == JOB_NOT_FOUND:
+        return "The job cannot be found."
+    elif error_num == INVALID_JOBNAME:
+        return "The job name is invalid."
+    elif error_num == INVALID_VOLNAME:
+        return "The volume name is invalid."
+    elif error_num == INVALID_SCHEDULE:
+        return "The schedule is invalid."
+    elif error_num == INVALID_ARG:
+        return "The argument is invalid."
+    elif error_num == VOLUME_DOES_NOT_EXIST:
+        return "The volume does not exist."
+
+def output(msg):
+    print("%s: %s" % (SCRIPT_NAME, msg))
+
+
+def initLogger():
+    log.setLevel(logging.DEBUG)
+    logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\
+        "%(levelname)s %(message)s"
+    formatter = logging.Formatter(logFormat)
+
+    sh = logging.handlers.SysLogHandler()
+    sh.setLevel(logging.ERROR)
+    sh.setFormatter(formatter)
+
+    process = subprocess.Popen(["gluster", "--print-logdir"],
+                               stdout=subprocess.PIPE, universal_newlines=True)
+    logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log")
+
+    fh = logging.FileHandler(logfile)
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(formatter)
+
+    log.addHandler(sh)
+    log.addHandler(fh)
+
+
+def scheduler_status():
+    ret = INTERNAL_ERROR
+    global scheduler_enabled
+    try:
+        f = os.path.realpath(GCRON_TASKS)
+        if f != os.path.realpath(GCRON_ENABLED) or not os.path.exists(GCRON_ENABLED):
+            log.info("Snapshot scheduler is currently disabled.")
+            scheduler_enabled = False
+        else:
+            log.info("Snapshot scheduler is currently enabled.")
+            scheduler_enabled = True
+        ret = 0
+    except:
+        log.error("Failed to enable snapshot scheduling. Error: "
+                  "Failed to check the status of %s.", GCRON_DISABLED)
+
+    return ret
+
+def enable_scheduler():
+    ret = scheduler_status()
+    if ret == 0:
+        if not scheduler_enabled:
+
+            # Check if another scheduler is active.
+            ret = get_current_scheduler()
+            if ret == 0:
+                if (current_scheduler != "none"):
+                    print_str = "Failed to enable snapshot scheduling. " \
+                                "Error: Another scheduler is active."
+                    log.error(print_str)
+                    output(print_str)
+                    ret = ANOTHER_SCHEDULER_ACTIVE
+                    return ret
+            else:
+                print_str = "Failed to get current scheduler info."
+                log.error(print_str)
+                output(print_str)
+                return ret
+
+            log.info("Enabling snapshot scheduler.")
+            try:
+                if os.path.exists(GCRON_DISABLED):
+                    os.remove(GCRON_DISABLED)
+                if os.path.lexists(GCRON_TASKS):
+                    os.remove(GCRON_TASKS)
+                try:
+                    f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK,
+                                0o644)
+                    os.close(f)
+                except OSError as e:
+                    log.error("Failed to open %s. Error: %s.",
+                              GCRON_ENABLED, e)
+                    ret = INTERNAL_ERROR
+                    return ret
+                os.symlink(GCRON_ENABLED, GCRON_TASKS)
+                update_current_scheduler("cli")
+                log.info("Snapshot scheduling is enabled")
+                output("Snapshot scheduling is enabled")
+                ret = 0
+            except OSError as e:
+                print_str = ("Failed to enable snapshot scheduling."
+                             "Error: {{}}" + e)
+                log.error(print_str)
+                output(print_str)
+                ret = INTERNAL_ERROR
+        else:
+            print_str = "Failed to enable snapshot scheduling. " \
+                        "Error: Snapshot scheduling is already enabled."
+            log.error(print_str)
+            output(print_str)
+            ret = SCHEDULING_ALREADY_ENABLED
+    else:
+        print_str = "Failed to enable snapshot scheduling. " \
+                    "Error: Failed to check scheduler status."
+        log.error(print_str)
+        output(print_str)
+
+    return ret
+
+
+def disable_scheduler():
+    ret = scheduler_status()
+    if ret == 0:
+        if scheduler_enabled:
+            log.info("Disabling snapshot scheduler.")
+            try:
+                # Check if another scheduler is active. If not, then
+                # update current scheduler to "none". Else do nothing.
+                ret = get_current_scheduler()
+                if ret == 0:
+                    if (current_scheduler == "cli"):
+                        update_current_scheduler("none")
+                else:
+                    print_str = "Failed to disable snapshot scheduling. " \
+                                "Error: Failed to get current scheduler info."
+                    log.error(print_str)
+                    output(print_str)
+                    return ret
+
+                if os.path.exists(GCRON_DISABLED):
+                    os.remove(GCRON_DISABLED)
+                if os.path.lexists(GCRON_TASKS):
+                    os.remove(GCRON_TASKS)
+                f = os.open(GCRON_DISABLED, os.O_CREAT, 0o644)
+                os.close(f)
+                os.symlink(GCRON_DISABLED, GCRON_TASKS)
+                log.info("Snapshot scheduling is disabled")
+                output("Snapshot scheduling is disabled")
+                ret = 0
+            except OSError as e:
+                print_str = ("Failed to disable snapshot scheduling. Error: "
+                             + e)
+                log.error(print_str)
+                output(print_str)
+                ret = INTERNAL_ERROR
+        else:
+            print_str = "Failed to disable scheduling. " \
+                        "Error: Snapshot scheduling is already disabled."
+            log.error(print_str)
+            output(print_str)
+            ret = SCHEDULING_ALREADY_DISABLED
+    else:
+        print_str = "Failed to disable snapshot scheduling. " \
+                    "Error: Failed to check scheduler status."
+        log.error(print_str)
+        output(print_str)
+        ret = INTERNAL_ERROR
+
+    return ret
+
+
+def load_tasks_from_file():
+    global tasks
+    global longest_field
+    try:
+        with open(GCRON_ENABLED, 'r') as f:
+            for line in f:
+                line = line.rstrip('\n')
+                if not line:
+                    break
+                line = line.split("gcron.py")
+                schedule = line[0].split("root")[0].rstrip(' ')
+                line = line[1].split(" ")
+                volname = line[1]
+                jobname = line[2]
+                longest_field = max(longest_field, len(jobname), len(volname),
+                                    len(schedule))
+                tasks[jobname] = schedule+":"+volname
+            f.close()
+        ret = 0
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, e)
+        ret = INTERNAL_ERROR
+
+    return ret
+
+
+def get_current_scheduler():
+    global current_scheduler
+    try:
+        with open(CURRENT_SCHEDULER, 'r') as f:
+            current_scheduler = f.readline().rstrip('\n')
+            f.close()
+        ret = 0
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, e)
+        ret = INTERNAL_ERROR
+
+    return ret
+
+
+def list_schedules():
+    log.info("Listing snapshot schedules.")
+    ret = load_tasks_from_file()
+    if ret == 0:
+        if len(tasks) == 0:
+            output("No snapshots scheduled")
+        else:
+            jobname = "JOB_NAME".ljust(longest_field+5)
+            schedule = "SCHEDULE".ljust(longest_field+5)
+            operation = "OPERATION".ljust(longest_field+5)
+            volname = "VOLUME NAME".ljust(longest_field+5)
+            hyphens = "".ljust((longest_field+5) * 4, '-')
+            print(jobname+schedule+operation+volname)
+            print(hyphens)
+            for key in sorted(tasks):
+                jobname = key.ljust(longest_field+5)
+                schedule = tasks[key].split(":")[0].ljust(
+                           longest_field + 5)
+                volname = tasks[key].split(":")[1].ljust(
+                          longest_field + 5)
+                operation = "Snapshot Create".ljust(longest_field+5)
+                print(jobname+schedule+operation+volname)
+            ret = 0
+    else:
+        print_str = "Failed to list snapshot schedules. " \
+                    "Error: Failed to load tasks from "+GCRON_ENABLED
+        log.error(print_str)
+        output(print_str)
+
+    return ret
+
+
+def write_tasks_to_file():
+    try:
+        with open(TMP_FILE, "w", 0o644) as f:
+            # If tasks is empty, just create an empty tmp file
+            if len(tasks) != 0:
+                for key in sorted(tasks):
+                    jobname = key
+                    schedule = tasks[key].split(":")[0]
+                    volname = tasks[key].split(":")[1]
+                    f.write("%s root PATH=$PATH:/usr/local/sbin:/usr/sbin "
+                            "gcron.py %s %s\n" % (schedule, volname, jobname))
+                f.write("\n")
+                f.flush()
+                os.fsync(f.fileno())
+            f.close()
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
+        ret = INTERNAL_ERROR
+        return ret
+
+    shutil.move(TMP_FILE, GCRON_ENABLED)
+    ret = 0
+
+    return ret
+
+def update_current_scheduler(data):
+    try:
+        with open(TMP_FILE, "w", 0o644) as f:
+            f.write("%s" % data)
+            f.flush()
+            os.fsync(f.fileno())
+            f.close()
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
+        ret = INTERNAL_ERROR
+        return ret
+
+    shutil.move(TMP_FILE, CURRENT_SCHEDULER)
+    ret = 0
+
+    return ret
+
+
+def isVolumePresent(volname):
+    success = False
+    if volname == "":
+        log.debug("No volname given")
+        return success
+
+    cli = ["gluster",
+           "volume",
+           "info",
+           volname]
+    log.debug("Running command '%s'", " ".join(cli))
+
+    p = subprocess.Popen(cli, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    rv = p.returncode
+
+    log.debug("Command '%s' returned '%d'", " ".join(cli), rv)
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+    else:
+        success = True;
+
+    return success
+
+
+def add_schedules(jobname, schedule, volname):
+    log.info("Adding snapshot schedules.")
+    ret = load_tasks_from_file()
+    if ret == 0:
+        if jobname in tasks:
+            print_str = ("%s already exists in schedule. Use "
+                         "'edit' to modify %s" % (jobname, jobname))
+            log.error(print_str)
+            output(print_str)
+            ret = JOB_ALREADY_EXISTS
+        else:
+            if not isVolumePresent(volname):
+                print_str = ("Volume %s does not exist. Create %s and retry." %
+                             (volname, volname))
+                log.error(print_str)
+                output(print_str)
+                ret = VOLUME_DOES_NOT_EXIST
+            else:
+                tasks[jobname] = schedule + ":" + volname
+                ret = write_tasks_to_file()
+                if ret == 0:
+                    # Create a LOCK_FILE for the job
+                    job_lockfile = LOCK_FILE_DIR + jobname
+                    try:
+                        f = os.open(job_lockfile, os.O_CREAT | os.O_NONBLOCK,
+                                    0o644)
+                        os.close(f)
+                    except OSError as e:
+                        log.error("Failed to open %s. Error: %s.",
+                                  job_lockfile, e)
+                        ret = INTERNAL_ERROR
+                        return ret
+                    log.info("Successfully added snapshot schedule %s" %
+                             jobname)
+                    output("Successfully added snapshot schedule")
+                    ret = 0
+    else:
+        print_str = "Failed to add snapshot schedule. " \
+                    "Error: Failed to load tasks from "+GCRON_ENABLED
+        log.error(print_str)
+        output(print_str)
+
+    return ret
+
+
+def delete_schedules(jobname):
+    log.info("Delete snapshot schedules.")
+    ret = load_tasks_from_file()
+    if ret == 0:
+        if jobname in tasks:
+            del tasks[jobname]
+            ret = write_tasks_to_file()
+            if ret == 0:
+                # Delete the LOCK_FILE for the job
+                job_lockfile = LOCK_FILE_DIR+jobname
+                try:
+                    os.remove(job_lockfile)
+                except OSError as e:
+                    log.error("Failed to open %s. Error: %s.",
+                              job_lockfile, e)
+                    ret = INTERNAL_ERROR
+                    return ret
+                log.info("Successfully deleted snapshot schedule %s"
+                         % jobname)
+                output("Successfully deleted snapshot schedule")
+                ret = 0
+        else:
+            print_str = ("Failed to delete %s. Error: No such "
+                         "job scheduled" % jobname)
+            log.error(print_str)
+            output(print_str)
+            ret = JOB_NOT_FOUND
+    else:
+        print_str = "Failed to delete snapshot schedule. " \
+                    "Error: Failed to load tasks from "+GCRON_ENABLED
+        log.error(print_str)
+        output(print_str)
+
+    return ret
+
+
+def edit_schedules(jobname, schedule, volname):
+    log.info("Editing snapshot schedules.")
+    ret = load_tasks_from_file()
+    if ret == 0:
+        if jobname in tasks:
+            if not isVolumePresent(volname):
+                print_str = ("Volume %s does not exist. Create %s and retry." %
+                             (volname, volname))
+                log.error(print_str)
+                output(print_str)
+                ret = VOLUME_DOES_NOT_EXIST
+            else:
+                tasks[jobname] = schedule+":"+volname
+                ret = write_tasks_to_file()
+                if ret == 0:
+                    log.info("Successfully edited snapshot schedule %s" %
+                             jobname)
+                    output("Successfully edited snapshot schedule")
+        else:
+            print_str = ("Failed to edit %s. Error: No such "
+                         "job scheduled" % jobname)
+            log.error(print_str)
+            output(print_str)
+            ret = JOB_NOT_FOUND
+    else:
+        print_str = "Failed to edit snapshot schedule. " \
+                    "Error: Failed to load tasks from "+GCRON_ENABLED
+        log.error(print_str)
+        output(print_str)
+
+    return ret
+
+def get_bool_val():
+    getsebool_cli = ["getsebool",
+                     "-a"]
+    p1 = subprocess.Popen(getsebool_cli, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    grep_cmd = ["grep",
+                "cron_system_cronjob_use_shares"]
+    p2 = subprocess.Popen(grep_cmd, stdin=p1.stdout,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    p1.stdout.close()
+    output, err = p2.communicate()
+    rv = p2.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return -1
+
+    bool_val = output.split()[2]
+    log.debug("Bool value = '%s'", bool_val)
+
+    return bool_val
+
+def get_selinux_status():
+    getenforce_cli = ["getenforce"]
+    log.debug("Running command '%s'", " ".join(getenforce_cli))
+
+    try:
+        p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+    except OSError as oserr:
+        log.error("Failed to run the command \"getenforce\". Error: %s" %\
+                  oserr)
+        return -1
+
+    output, err = p1.communicate()
+    rv = p1.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return -1
+    else:
+        selinux_status=output.rstrip()
+        log.debug("selinux status: %s", selinux_status)
+
+    return selinux_status
+
+def set_cronjob_user_share():
+    selinux_status = get_selinux_status()
+    if (selinux_status == -1):
+        log.error("Failed to get selinux status")
+        return -1
+    elif (selinux_status == "Disabled"):
+        return 0
+
+    bool_val = get_bool_val()
+    # In case of a failure (where the boolean value is not)
+    # present in the system, we should not proceed further
+    # We should only proceed when the value is "off"
+    if (bool_val == -1 or bool_val != "off"):
+        return 0
+
+    setsebool_cli = ["setsebool", "-P",
+                     "cron_system_cronjob_use_shares",
+                     "on"]
+    log.debug("Running command '%s'", " ".join(setsebool_cli))
+
+    p1 = subprocess.Popen(setsebool_cli, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+
+    output, err = p1.communicate()
+    rv = p1.returncode
+
+    if rv:
+        log.error("Command output:")
+        log.error(err)
+        return rv
+
+    bool_val = get_bool_val()
+    if (bool_val == "on"):
+        return 0
+    else:
+        # In case of an error or if boolean is not on
+        # we return a failure here
+        return -1
+
+def initialise_scheduler():
+    ret = set_cronjob_user_share()
+    if ret:
+        log.error("Failed to set selinux boolean "
+                  "cron_system_cronjob_use_shares to 'on'")
+        return ret
+
+    try:
+        with open(TMP_FILE, "w+", 0o644) as f:
+            updater = ("* * * * * root PATH=$PATH:/usr/local/sbin:"
+                       "/usr/sbin gcron.py --update\n")
+            f.write("%s\n" % updater)
+            f.flush()
+            os.fsync(f.fileno())
+            f.close()
+    except IOError as e:
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, e)
+        ret = INIT_FAILED
+        return ret
+
+    shutil.move(TMP_FILE, GCRON_UPDATE_TASK)
+
+    if not os.path.lexists(GCRON_TASKS):
+        try:
+            f = open(GCRON_TASKS, "w", 0o644)
+            f.close()
+        except IOError as e:
+            log.error("Failed to open %s. Error: %s.", GCRON_TASKS, e)
+            ret = INIT_FAILED
+            return ret
+
+    if os.path.lexists(GCRON_CROND_TASK):
+        os.remove(GCRON_CROND_TASK)
+
+    os.symlink(GCRON_TASKS, GCRON_CROND_TASK)
+
+    log.info("Successfully initialised snapshot scheduler for this node")
+    output("Successfully initialised snapshot scheduler for this node")
+    gf_event (EVENT_SNAPSHOT_SCHEDULER_INITIALISED, status="Success")
+
+    ret = 0
+    return ret
+
+
+def syntax_checker(args):
+    if hasattr(args, 'jobname'):
+        if (len(args.jobname.split()) != 1):
+            output("Invalid Jobname. Jobname should not be empty and should not contain \" \" character.")
+            ret = INVALID_JOBNAME
+            return ret
+        args.jobname=args.jobname.strip()
+
+    if hasattr(args, 'volname'):
+        if (len(args.volname.split()) != 1):
+            output("Invalid Volname. Volname should not be empty and should not contain \" \" character.")
+            ret = INVALID_VOLNAME
+            return ret
+        args.volname=args.volname.strip()
+
+    if hasattr(args, 'schedule'):
+        if (len(args.schedule.split()) != 5):
+            output("Invalid Schedule. Please refer to the following for adding a valid cron schedule")
+            print ("* * * * *")
+            print ("| | | | |")
+            print ("| | | | +---- Day of the Week   (range: 1-7, 1 standing for Monday)")
+            print ("| | | +------ Month of the Year (range: 1-12)")
+            print ("| | +-------- Day of the Month  (range: 1-31)")
+            print ("| +---------- Hour              (range: 0-23)")
+            print ("+------------ Minute            (range: 0-59)")
+            ret = INVALID_SCHEDULE
+            return ret
+
+    ret = 0
+    return ret
+
+
+def perform_operation(args):
+    if not os.path.exists(CURRENT_SCHEDULER):
+        update_current_scheduler("none")
+
+    # Initialise snapshot scheduler on local node
+    if args.action == "init":
+        ret = initialise_scheduler()
+        if ret != 0:
+            output("Failed to initialise snapshot scheduling")
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_INIT_FAILED,
+                      error=print_error(ret))
+        return ret
+
+    # Disable snapshot scheduler
+    if args.action == "disable_force":
+        ret = disable_scheduler()
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED,
+                      status="Successfully Disabled")
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED,
+                      error=print_error(ret))
+        return ret
+
+    # Check if the symlink to GCRON_TASKS is properly set in the shared storage
+    if (not os.path.lexists(GCRON_UPDATE_TASK) or
+        not os.path.lexists(GCRON_CROND_TASK) or
+        os.readlink(GCRON_CROND_TASK) != GCRON_TASKS):
+        print_str = ("Please run 'snap_scheduler.py' init to initialise "
+                     "the snap scheduler for the local node.")
+        log.error(print_str)
+        output(print_str)
+        ret = NODE_NOT_INITIALISED
+        return ret
+
+    # Check status of snapshot scheduler.
+    if args.action == "status":
+        ret = scheduler_status()
+        if ret == 0:
+            if scheduler_enabled:
+                output("Snapshot scheduling status: Enabled")
+            else:
+                output("Snapshot scheduling status: Disabled")
+        else:
+            output("Failed to check status of snapshot scheduler")
+        return ret
+
+    # Enable snapshot scheduler
+    if args.action == "enable":
+        ret = enable_scheduler()
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLED,
+                      status="Successfully Enabled")
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED,
+                      error=print_error(ret))
+        return ret
+
+    # Disable snapshot scheduler
+    if args.action == "disable":
+        ret = disable_scheduler()
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED,
+                      status="Successfully Disabled")
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED,
+                      error=print_error(ret))
+        return ret
+
+    # List snapshot schedules
+    if args.action == "list":
+        ret = list_schedules()
+        return ret
+
+    # Add snapshot schedules
+    if args.action == "add":
+        ret = syntax_checker(args)
+        if ret != 0:
+            return ret
+        ret = add_schedules(args.jobname, args.schedule, args.volname)
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED,
+                      status="Successfully added job "+args.jobname)
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED,
+                      status="Failed to add job "+args.jobname,
+                      error=print_error(ret))
+        return ret
+
+    # Delete snapshot schedules
+    if args.action == "delete":
+        ret = syntax_checker(args)
+        if ret != 0:
+            return ret
+        ret = delete_schedules(args.jobname)
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED,
+                      status="Successfully deleted job "+args.jobname)
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED,
+                      status="Failed to delete job "+args.jobname,
+                      error=print_error(ret))
+        return ret
+
+    # Edit snapshot schedules
+    if args.action == "edit":
+        ret = syntax_checker(args)
+        if ret != 0:
+            return ret
+        ret = edit_schedules(args.jobname, args.schedule, args.volname)
+        if ret == 0:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED,
+                      status="Successfully edited job "+args.jobname)
+        else:
+            gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED,
+                      status="Failed to edit job "+args.jobname,
+                      error=print_error(ret))
+        return ret
+
+    ret = INVALID_ARG
+    return ret
+
+def gf_event(event_type, **kwargs):
+    if EVENTS_ENABLED:
+        from events.gf_event import gf_event as gfevent
+        gfevent(event_type, **kwargs)
+
+
+def main(argv):
+    initLogger()
+    ret = -1
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="action",
+                                       metavar=('{init, status, enable,'
+                                               ' disable, list, add,'
+                                               ' delete, edit}'))
+    subparsers.add_parser('init',
+                          help="Initialise the node for snapshot scheduling")
+
+    subparsers.add_parser("status",
+                          help="Check if snapshot scheduling is "
+                          "enabled or disabled")
+    subparsers.add_parser("enable",
+                          help="Enable snapshot scheduling")
+    subparsers.add_parser("disable",
+                          help="Disable snapshot scheduling")
+    subparsers.add_parser("disable_force")
+    subparsers.add_parser("list",
+                          help="List snapshot schedules")
+    parser_add = subparsers.add_parser("add",
+                                       help="Add snapshot schedules")
+    parser_add.add_argument("jobname", help="Job Name")
+    parser_add.add_argument("schedule", help="Schedule")
+    parser_add.add_argument("volname", help="Volume Name")
+
+    parser_delete = subparsers.add_parser("delete",
+                                          help="Delete snapshot schedules")
+    parser_delete.add_argument("jobname", help="Job Name")
+    parser_edit = subparsers.add_parser("edit",
+                                        help="Edit snapshot schedules")
+    parser_edit.add_argument("jobname", help="Job Name")
+    parser_edit.add_argument("schedule", help="Schedule")
+    parser_edit.add_argument("volname", help="Volume Name")
+
+    args = parser.parse_args(argv)
+
+    if not os.path.exists(SHARED_STORAGE_DIR):
+        output("Failed: "+SHARED_STORAGE_DIR+" does not exist.")
+        return SHARED_STORAGE_DIR_DOESNT_EXIST
+
+    if not os.path.ismount(SHARED_STORAGE_DIR):
+        output("Failed: Shared storage is not mounted at "+SHARED_STORAGE_DIR)
+        return SHARED_STORAGE_NOT_MOUNTED
+
+    if not os.path.exists(SHARED_STORAGE_DIR+"/snaps/"):
+        try:
+            os.makedirs(SHARED_STORAGE_DIR+"/snaps/")
+        except OSError as e:
+            if errno != EEXIST:
+                log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", e)
+                output("Failed to create %s. Error: %s"
+                       % (SHARED_STORAGE_DIR+"/snaps/", e))
+                return INTERNAL_ERROR
+
+    if not os.path.exists(GCRON_ENABLED):
+        f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0o644)
+        os.close(f)
+
+    if not os.path.exists(LOCK_FILE_DIR):
+        try:
+            os.makedirs(LOCK_FILE_DIR)
+        except OSError as e:
+            if errno != EEXIST:
+                log.error("Failed to create %s : %s", LOCK_FILE_DIR, e)
+                output("Failed to create %s. Error: %s"
+                       % (LOCK_FILE_DIR, e))
+                return INTERNAL_ERROR
+
+    try:
+        f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0o644)
+        try:
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            ret = perform_operation(args)
+            fcntl.flock(f, fcntl.LOCK_UN)
+        except IOError:
+            log.info("%s is being processed by another agent.", LOCK_FILE)
+            output("Another snap_scheduler command is running. "
+                   "Please try again after some time.")
+            return ANOTHER_TRANSACTION_IN_PROGRESS
+        os.close(f)
+    except OSError as e:
+        log.error("Failed to open %s : %s", LOCK_FILE, e)
+        output("Failed to open %s. Error: %s" % (LOCK_FILE, e))
+        return INTERNAL_ERROR
+
+    return ret
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
diff --git a/extras/specgen.scm b/extras/specgen.scm
index 57640705f07..0edbb6f6280 100755
--- a/extras/specgen.scm
+++ b/extras/specgen.scm
@@ -1,7 +1,7 @@
 #!/usr/bin/guile -s
 !#
 
-;;; Copyright (C) 2007-2010 Gluster Inc. <http://www.gluster.com>
+;;; Copyright (c) 2007-2011 Gluster Inc. <http://www.gluster.com>
 ;;;  
 ;;; This program is free software; you can redistribute it and/or modify
 ;;; it under the terms of the GNU General Public License as published by
diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb
new file mode 100755
index 00000000000..1aff43377db
--- /dev/null
+++ b/extras/statedumpparse.rb
@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+
+require 'time'
+require 'optparse'
+
+unless Array.instance_methods.include? :to_h
+  class Array
+    def to_h
+      h = {}
+      each { |k,v| h[k]=v }
+      h
+    end
+  end
+end
+
+# statedump.c:gf_proc_dump_mempool_info uses a five-dash record separator,
+# client.c:client_fd_lk_ctx_dump uses a six-dash record separator.
+ARRSEP = /^(-{5,6}=-{5,6})?$/
+HEAD = /^\[(.*)\]$/
+INPUT_FORMATS = %w[statedump json]
+
+format = 'json'
+input_format = 'statedump'
+tz = '+0000'
+memstat_select,memstat_reject = //,/\Z./
+OptionParser.new do |op|
+  op.banner << " [<] <STATEDUMP>"
+  op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s }
+  op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s }
+  op.on("--timezone=T",
+        "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s }
+  op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s|
+    memstat_select = Regexp.new s
+  }
+  op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s|
+    memstat_reject = Regexp.new s
+  }
+end.parse!
+
+
+if format =~ /\Amemstat(?:-(.*))?/
+  memstat_type = $1 || 'plain'
+  unless %w[plain human json].include? memstat_type
+    raise "unknown memstat type #{memstat_type.dump}"
+  end
+  format = 'memstat'
+end
+
+repr, logsep = case format
+when 'yaml'
+  require 'yaml'
+
+  [proc { |e| e.to_yaml }, "\n"]
+when 'json', 'memstat'
+  require 'json'
+
+  [proc { |e| e.to_json }, " "]
+else
+  raise "unkonwn format '#{format}'"
+end
+formatter = proc { |e| puts repr.call(e) }
+
+INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'"
+
+dumpinfo = {}
+
+# parse a statedump entry
+elem_cbk = proc { |s,&cbk|
+  arraylike = false
+  s.grep(/\S/).empty? and next
+  head = nil
+  while s.last =~ /^\s*$/
+    s.pop
+  end
+  body = catch { |misc2|
+    s[0] =~ HEAD ? (head = $1) : (throw misc2)
+    body = [[]]
+    s[1..-1].each { |l|
+      if l =~ ARRSEP
+        arraylike = true
+        body << []
+        next
+      end
+      body.last << l
+    }
+
+    body.reject(&:empty?).map { |e|
+      ea = e.map { |l|
+        k,v = l.split("=",2)
+        m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v
+        [k, m ? (m[2] ? Float(v) : Integer(v)) : v]
+      }
+      begin
+        ea.to_h
+      rescue
+        throw misc2
+      end
+    }
+  }
+
+  if body
+    cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])]
+  else
+    STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep)
+  end
+}
+
+# aggregator routine
+aggr = case format
+when 'memstat'
+  meminfo = {}
+  # commit memory-related entries to meminfo
+  proc { |k,r|
+    case k
+    when /memusage/
+      (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject
+    when "mempool"
+      r.each {|e|
+        kk = "mempool:#{e['pool-name']}"
+        (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject
+      }
+    end
+  }
+else
+  # just format data, don't actually aggregate anything
+  proc { |pair| formatter.call pair }
+end
+
+# processing the data
+case input_format
+when 'statedump'
+  acc = []
+  $<.each { |l|
+    l = l.strip
+    if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/
+      dumpinfo["_meta"]||={}
+      (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ")
+      next
+    end
+
+    if l =~ HEAD
+      elem_cbk.call(acc, &aggr)
+      acc = [l]
+      next
+    end
+
+    acc << l
+  }
+  elem_cbk.call(acc, &aggr)
+when 'json'
+  $<.each { |l|
+    r = JSON.load l
+    case r
+    when Array
+      aggr[r]
+    when Hash
+      dumpinfo.merge! r
+    end
+  }
+end
+
+# final actions: output aggregated data
+case format
+when 'memstat'
+  ma = meminfo.values.map(&:to_a).inject(:+)
+  totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h
+  tt = ma.transpose[1].inject(:+)
+
+  summary_sep,showm = case memstat_type
+  when 'json'
+    ["", proc { |k,v| puts({type: k, value: v}.to_json) }]
+  when 'plain', 'human'
+    # human-friendly number representation
+    hr = proc { |n|
+      qa = %w[B kB MB GB]
+      q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1
+      "%.2f%s" % [n.to_f / (1 << q*10), qa[q]]
+    }
+
+    templ = "%{val} %{key}"
+    tft = proc { |t| t }
+    nft = if memstat_type == 'human'
+      nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max
+      proc { |n|
+        hn = hr[n]
+        " " * (nw - hn.size) + hn
+      }
+    else
+      nw = tt.to_s.size
+      proc { |n| "%#{nw}d" % n }
+    end
+    ## Alternative template, key first:
+    # templ = "%{key} %{val}"
+    # tw = ma.transpose[0].map(&:size).max
+    # tft = proc { |t| t + " " * [tw - t.size, 0].max }
+    # nft = (memstat_type == 'human') ? hr : proc { |n| n }
+    ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }]
+  else
+    raise 'this should be impossible'
+  end
+
+  ma.sort_by { |k,v| v }.each(&showm)
+  print summary_sep
+  totals.each { |coll,t| showm.call "Total #{coll}", t }
+  showm.call "TOTAL", tt
+else
+  formatter.call dumpinfo
+end
diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh
new file mode 100755
index 00000000000..710aaf5fd3c
--- /dev/null
+++ b/extras/stop-all-gluster-processes.sh
@@ -0,0 +1,193 @@
+#!/bin/bash
+#
+# Kill all the processes/services except glusterd
+#
+# Usage: ./extras/stop-all-gluster-processes.sh [-g] [-h]
+#    options:
+#    -g  Terminate in graceful mode
+#    -h  Show this message, then exit
+#
+# eg:
+#  1. ./extras/stop-all-gluster-processes.sh
+#  2. ./extras/stop-all-gluster-processes.sh -g
+#
+# By default, this script executes in force mode, i.e. all of brick, gsyncd
+# and other glustershd services/processes are killed without checking for
+# ongoing tasks such as geo-rep, self-heal, rebalance and etc. which may lead
+# to inconsistency after the node is brought back.
+#
+# On specifying '-g' option this script works in graceful mode, to maintain
+# data consistency the script fails with a valid exit code incase if any of
+# the gluster processes are busy in doing their jobs.
+#
+# The author of page [1] proposes user-defined exit codes to the range 64 - 113
+# Find the better explanation behind the choice in the link
+#
+# The exit code returned by stop-all-gluster-processes.sh:
+#   0       No errors/Success
+#   64      Rebalance is in progress
+#   65      Self-Heal is in progress
+#   66      Tier daemon running on this node
+#   127     option not found
+#
+# [1] http://www.tldp.org/LDP/abs/html/exitcodes.html
+
+
+# global
+errors=0
+
+# find the mounts and return their pids
+get_mount_pids()
+{
+    local opts
+    local pid
+
+    for opts in $(grep -w fuse.glusterfs /proc/mounts| awk '{print $1":/"$2}');
+    do
+        IFS=' ' read -r -a volinfo <<< $(echo "${opts}" | sed 's/:\// /g')
+        pid+="$(ps -Ao pid,args | grep -w "volfile-server=${volinfo[0]}" |
+                grep -w "volfile-id=/${volinfo[1]}" | grep -w "${volinfo[2]}" |
+                awk '{print $1}') "
+    done
+    echo "${pid}"
+}
+
+# handle mount processes i.e. 'glusterfs'
+kill_mounts()
+{
+    local signal=${1}
+    local pid
+
+    for pid in $(get_mount_pids);
+    do
+        echo "sending SIG${signal} to mount process with pid: ${pid}";
+        kill -${signal} ${pid};
+    done
+}
+
+# handle brick processes and node services
+kill_bricks_and_services()
+{
+    local signal=${1}
+    local pidfile
+    local pid
+
+    for pidfile in $(find /var/run/gluster/ -name '*.pid');
+    do
+        local pid=$(cat ${pidfile});
+        echo "sending SIG${signal} to pid: ${pid}";
+        kill -${signal} ${pid};
+    done
+}
+
+# for geo-replication, only 'monitor' has pid file written, other
+# processes are not having a pid file, so get it through 'ps' and
+# handle these processes
+kill_georep_gsync()
+{
+    local signal=${1}
+
+    # FIXME: add strick/better check
+    local gsyncpid=$(ps -Ao pid,args | grep gluster | grep gsync |
+                     awk '{print $1}');
+    if [ -n "${gsyncpid}" ]
+    then
+        echo "sending SIG${signal} to geo-rep gsync process ${gsyncpid}";
+        kill -${signal} ${gsyncpid} || errors=$((${errors} + 1));
+    fi
+}
+
+# check if all processes are ready to die
+check_background_tasks()
+{
+    volumes=$(gluster vol list)
+    quit=0
+    for volname in ${volumes};
+    do
+        # tiering
+        if [[ $(gluster volume tier ${volname} status 2> /dev/null |
+                grep "localhost" | grep -c "in progress") -gt 0 ]]
+        then
+            quit=66
+            break;
+        fi
+
+        # rebalance
+        if [[ $(gluster volume rebalance ${volname} status 2> /dev/null |
+                grep -c "in progress") -gt 0 ]]
+        then
+            quit=64
+            break;
+        fi
+
+        # self heal
+        if [[ $(gluster volume heal ${volname} info | grep "Number of entries" |
+                awk '{ sum+=$4} END {print sum}') -gt 0 ]];
+        then
+            quit=65
+            break;
+        fi
+
+        # geo-rep, snapshot and quota doesn't need grace checks,
+        # as they ensures the consistancy on force kills
+    done
+
+    echo ${quit}
+}
+
+usage()
+{
+    cat <<EOM
+Usage: $0 [-g] [-h]
+    options:
+    -g  Terminate in graceful mode
+    -h  Show this message, then exit
+
+eg:
+ 1. $0
+ 2. $0 -g
+EOM
+}
+
+main()
+{
+    while getopts "gh" opt; do
+        case $opt in
+            g)
+                # graceful mode
+                quit=$(check_background_tasks)
+                if [[ ${quit} -ne 0 ]]
+                then
+                    exit ${quit};
+                fi
+                # else safe to kill
+                ;;
+            h)
+                usage
+                exit 0;
+                ;;
+            *)
+                usage
+                exit 127;
+                ;;
+        esac
+    done
+    # remove all the options that have been parsed by getopts
+    shift $((OPTIND-1))
+
+    kill_mounts TERM
+    kill_georep_gsync TERM
+    kill_bricks_and_services TERM
+
+    sleep 5;
+    echo ""
+
+    # still not Terminated? let's pass SIGKILL
+    kill_mounts KILL
+    kill_georep_gsync KILL
+    kill_bricks_and_services KILL
+
+    exit ${errors};
+}
+
+main "$@"
diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c
index 3f8e4b1244d..e013a6e6e8a 100644
--- a/extras/stripe-merge.c
+++ b/extras/stripe-merge.c
@@ -1,48 +1,503 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * stripe-merge.c
+ *
+ * This program recovers an original file based on the striped files stored on
+ * the individual bricks of a striped volume. The file format and stripe
+ * geometry is validated through the extended attributes stored in the file.
+ *
+ * TODO: Support optional xattr recovery (i.e., user xattrs). Perhaps provide a
+ * 	 command-line flag to toggle this behavior.
+ */
+
 #include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/xattr.h>
+#include <fnmatch.h>
 
-int
-main (int argc, char *argv[])
-{
-	int fds[argc-1];
-	char buf[argc-1][4096];
-	int i;
-	int max_ret, ret;
-
-	if (argc < 2) {
-		printf ("Usage: %s file1 file2 ... >file\n", argv[0]);
-		return 1;
-	}
-
-	for (i=0; i<argc-1; i++) {
-		fds[i] = open (argv[i+1], O_RDONLY);
-		if (fds[i] == -1) {
-			perror (argv[i+1]);
-			return 1;
-		}
-	}
-
-	max_ret = 0;
-	do {
-		char newbuf[4096] = {0, };
-		int j;
-
-		max_ret = 0;
-		for (i=0; i<argc-1; i++) {
-			memset (buf[i], 0, 4096);
-			ret = read (fds[i], buf[i], 4096); 
-			if (ret > max_ret)
-				max_ret = ret;
-		}
-		for (i=0; i<max_ret;i++)
-			for (j=0; j<argc-1; j++)
-				newbuf[i] |= buf[j][i];
-		write (1, newbuf, max_ret);
-	} while (max_ret);
-
-	return 0;
+#define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index"
+#define ATTRNAME_STRIPE_COUNT "trusted.*.stripe-count"
+#define ATTRNAME_STRIPE_SIZE "trusted.*.stripe-size"
+#define ATTRNAME_STRIPE_COALESCE "trusted.*.stripe-coalesce"
+
+#define INVALID_FD -1
+#define INVALID_MODE UINT32_MAX
+
+struct file_stripe_info {
+    int stripe_count;
+    int stripe_size;
+    int coalesce;
+    mode_t mode;
+    int fd[0];
+};
+
+static int
+close_files(struct file_stripe_info *);
+
+static struct file_stripe_info *
+alloc_file_stripe_info(int count)
+{
+    int i;
+    struct file_stripe_info *finfo;
+
+    finfo = calloc(1, sizeof(struct file_stripe_info) + (sizeof(int) * count));
+    if (!finfo)
+        return NULL;
+
+    for (i = 0; i < count; i++)
+        finfo->fd[i] = INVALID_FD;
+
+    finfo->mode = INVALID_MODE;
+    finfo->coalesce = INVALID_FD;
+
+    return finfo;
+}
+
+/*
+ * Search for an attribute matching the provided pattern. Return a count for
+ * the total number of matching entries (including 0). Allocate a buffer for
+ * the first matching entry found.
+ */
+static int
+get_stripe_attr_name(const char *path, const char *pattern, char **attrname)
+{
+    char attrbuf[4096];
+    char *ptr, *match = NULL;
+    int len, r, match_count = 0;
+
+    if (!path || !pattern || !attrname)
+        return -1;
+
+    len = listxattr(path, attrbuf, sizeof(attrbuf));
+    if (len < 0)
+        return len;
+
+    ptr = attrbuf;
+    while (ptr) {
+        r = fnmatch(pattern, ptr, 0);
+        if (!r) {
+            if (!match)
+                match = ptr;
+            match_count++;
+        } else if (r != FNM_NOMATCH) {
+            return -1;
+        }
+
+        len -= strlen(ptr) + 1;
+        if (len > 0)
+            ptr += strlen(ptr) + 1;
+        else
+            ptr = NULL;
+    }
+
+    if (match)
+        *attrname = strdup(match);
+
+    return match_count;
 }
 
+/*
+ * Get the integer representation of a named attribute.
+ */
+static int
+get_stripe_attr_val(const char *path, const char *attr, int *val)
+{
+    char attrbuf[4096];
+    int len;
+
+    if (!path || !attr || !val)
+        return -1;
+
+    len = getxattr(path, attr, attrbuf, sizeof(attrbuf));
+    if (len < 0)
+        return len;
+
+    *val = atoi(attrbuf);
+
+    return 0;
+}
+
+/*
+ * Get an attribute name/value (assumed to be an integer) pair based on a
+ * specified search pattern. A buffer is allocated for the exact attr name
+ * returned. Optionally, skip the pattern search if a buffer is provided
+ * (which should contain an attribute name).
+ *
+ * Returns the attribute count or -1 on error. The value parameter is set only
+ * when a single attribute is found.
+ */
+static int
+get_attr(const char *path, const char *pattern, char **buf, int *val)
+{
+    int count = 1;
+
+    if (!buf)
+        return -1;
+
+    if (!*buf) {
+        count = get_stripe_attr_name(path, pattern, buf);
+        if (count > 1) {
+            /* pattern isn't good enough */
+            fprintf(stderr,
+                    "ERROR: duplicate attributes found "
+                    "matching pattern: %s\n",
+                    pattern);
+            free(*buf);
+            *buf = NULL;
+            return count;
+        } else if (count < 1) {
+            return count;
+        }
+    }
+
+    if (get_stripe_attr_val(path, *buf, val) < 0)
+        return -1;
+
+    return count;
+}
+
+/*
+ * validate_and_open_files()
+ *
+ * Open the provided source files and validate the extended attributes. Verify
+ * that the geometric attributes are consistent across all of the files and
+ * print a warning if any files are missing. We proceed without error in the
+ * latter case to support partial recovery.
+ */
+static struct file_stripe_info *
+validate_and_open_files(char *paths[], int count)
+{
+    int i, val, tmp;
+    struct stat sbuf;
+    char *stripe_count_attr = NULL;
+    char *stripe_size_attr = NULL;
+    char *stripe_index_attr = NULL;
+    char *stripe_coalesce_attr = NULL;
+    struct file_stripe_info *finfo = NULL;
+
+    for (i = 0; i < count; i++) {
+        if (!paths[i])
+            goto err;
+
+        /*
+         * Check the stripe count first so we can allocate the info
+         * struct with the appropriate number of fds.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, &stripe_count_attr,
+                     &val) != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_COUNT);
+            goto err;
+        }
+        if (!finfo) {
+            finfo = alloc_file_stripe_info(val);
+            if (!finfo)
+                goto err;
+
+            if (val != count)
+                fprintf(stderr,
+                        "WARNING: %s: stripe-count "
+                        "(%d) != file count (%d). Result may "
+                        "be incomplete.\n",
+                        paths[i], val, count);
+
+            finfo->stripe_count = val;
+        } else if (val != finfo->stripe_count) {
+            fprintf(stderr,
+                    "ERROR %s: invalid stripe count: %d "
+                    "(expected %d)\n",
+                    paths[i], val, finfo->stripe_count);
+            goto err;
+        }
+
+        /*
+         * Get and validate the chunk size.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, &val) !=
+            1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_SIZE);
+            goto err;
+        }
+
+        if (!finfo->stripe_size) {
+            finfo->stripe_size = val;
+        } else if (val != finfo->stripe_size) {
+            fprintf(stderr,
+                    "ERROR: %s: invalid stripe size: %d "
+                    "(expected %d)\n",
+                    paths[i], val, finfo->stripe_size);
+            goto err;
+        }
+
+        /*
+         * stripe-coalesce is a backward compatible attribute. If the
+         * attribute does not exist, assume a value of zero for the
+         * traditional stripe format.
+         */
+        tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE,
+                       &stripe_coalesce_attr, &val);
+        if (!tmp) {
+            val = 0;
+        } else if (tmp != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_COALESCE);
+            goto err;
+        }
+
+        if (finfo->coalesce == INVALID_FD) {
+            finfo->coalesce = val;
+        } else if (val != finfo->coalesce) {
+            fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", paths[i]);
+            goto err;
+        }
+
+        /*
+         * Get/validate the stripe index and open the file in the
+         * appropriate fd slot.
+         */
+        if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, &stripe_index_attr,
+                     &val) != 1) {
+            fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i],
+                    ATTRNAME_STRIPE_INDEX);
+            goto err;
+        }
+        if (finfo->fd[val] != INVALID_FD) {
+            fprintf(stderr,
+                    "ERROR: %s: duplicate stripe index: "
+                    "%d\n",
+                    paths[i], val);
+            goto err;
+        }
+
+        finfo->fd[val] = open(paths[i], O_RDONLY);
+        if (finfo->fd[val] < 0)
+            goto err;
+
+        /*
+         * Get the creation mode for the file.
+         */
+        if (fstat(finfo->fd[val], &sbuf) < 0)
+            goto err;
+        if (finfo->mode == INVALID_MODE) {
+            finfo->mode = sbuf.st_mode;
+        } else if (sbuf.st_mode != finfo->mode) {
+            fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]);
+            goto err;
+        }
+    }
+
+    free(stripe_count_attr);
+    free(stripe_size_attr);
+    free(stripe_index_attr);
+    free(stripe_coalesce_attr);
+
+    return finfo;
+err:
+
+    free(stripe_count_attr);
+    free(stripe_size_attr);
+    free(stripe_index_attr);
+    free(stripe_coalesce_attr);
+
+    if (finfo) {
+        close_files(finfo);
+        free(finfo);
+    }
+
+    return NULL;
+}
+
+static int
+close_files(struct file_stripe_info *finfo)
+{
+    int i, ret;
+
+    if (!finfo)
+        return -1;
+
+    for (i = 0; i < finfo->stripe_count; i++) {
+        if (finfo->fd[i] == INVALID_FD)
+            continue;
+
+        ret = close(finfo->fd[i]);
+        if (ret < 0)
+            return ret;
+    }
+
+    return ret;
+}
+
+/*
+ * Generate the original file using files striped in the coalesced format.
+ * Data in the striped files is stored at a coalesced offset based on the
+ * stripe number.
+ *
+ * Walk through the finfo fds (which are already ordered) and and iteratively
+ * copy stripe_size bytes from the source files to the target file. If a source
+ * file is missing, seek past the associated stripe_size bytes in the target
+ * file.
+ */
+static int
+generate_file_coalesce(int target, struct file_stripe_info *finfo)
+{
+    char *buf;
+    int ret = 0;
+    int r, w, i;
+
+    buf = malloc(finfo->stripe_size);
+    if (!buf)
+        return -1;
+
+    i = 0;
+    while (1) {
+        if (finfo->fd[i] == INVALID_FD) {
+            if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0)
+                break;
+
+            i = (i + 1) % finfo->stripe_count;
+            continue;
+        }
+
+        r = read(finfo->fd[i], buf, finfo->stripe_size);
+        if (r < 0) {
+            ret = r;
+            break;
+        }
+        if (!r)
+            break;
+
+        w = write(target, buf, r);
+        if (w < 0) {
+            ret = w;
+            break;
+        }
+
+        i = (i + 1) % finfo->stripe_count;
+    }
+
+    free(buf);
+    return ret;
+}
+
+/*
+ * Generate the original file using files striped with the traditional stripe
+ * format. Data in the striped files is stored at the equivalent offset from
+ * the source file.
+ */
+static int
+generate_file_traditional(int target, struct file_stripe_info *finfo)
+{
+    int i, j, max_ret, ret;
+    char buf[finfo->stripe_count][4096];
+
+    do {
+        char newbuf[4096] = {
+            0,
+        };
+
+        max_ret = 0;
+        for (i = 0; i < finfo->stripe_count; i++) {
+            memset(buf[i], 0, 4096);
+            ret = read(finfo->fd[i], buf[i], 4096);
+            if (ret > max_ret)
+                max_ret = ret;
+        }
+        for (i = 0; i < max_ret; i++)
+            for (j = 0; j < finfo->stripe_count; j++)
+                newbuf[i] |= buf[j][i];
+        write(target, newbuf, max_ret);
+    } while (max_ret);
+
+    return 0;
+}
+
+static int
+generate_file(int target, struct file_stripe_info *finfo)
+{
+    if (finfo->coalesce)
+        return generate_file_coalesce(target, finfo);
+
+    return generate_file_traditional(target, finfo);
+}
+
+static void
+usage(char *name)
+{
+    fprintf(stderr,
+            "Usage: %s [-o <outputfile>] <inputfile1> "
+            "<inputfile2> ...\n",
+            name);
+}
+
+int
+main(int argc, char *argv[])
+{
+    int file_count, opt;
+    char *opath = NULL;
+    int targetfd;
+    struct file_stripe_info *finfo;
+
+    while ((opt = getopt(argc, argv, "o:")) != -1) {
+        switch (opt) {
+            case 'o':
+                opath = optarg;
+                break;
+            default:
+                usage(argv[0]);
+                return -1;
+        }
+    }
+
+    file_count = argc - optind;
+
+    if (!opath || !file_count) {
+        usage(argv[0]);
+        return -1;
+    }
+
+    finfo = validate_and_open_files(&argv[optind], file_count);
+    if (!finfo)
+        goto err;
+
+    targetfd = open(opath, O_RDWR | O_CREAT, finfo->mode);
+    if (targetfd < 0)
+        goto err;
+
+    if (generate_file(targetfd, finfo) < 0)
+        goto err;
+
+    if (fsync(targetfd) < 0)
+        fprintf(stderr, "ERROR: %s\n", strerror(errno));
+    if (close(targetfd) < 0)
+        fprintf(stderr, "ERROR: %s\n", strerror(errno));
+
+    close_files(finfo);
+    free(finfo);
+
+    return 0;
+
+err:
+    if (finfo) {
+        close_files(finfo);
+        free(finfo);
+    }
+
+    return -1;
+}
diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am
new file mode 100644
index 00000000000..61446a9b84a
--- /dev/null
+++ b/extras/systemd/Makefile.am
@@ -0,0 +1,17 @@
+CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service gluster-ta-volume.service
+EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in gluster-ta-volume.service.in
+
+if USE_SYSTEMD
+systemd_DATA = gluster-ta-volume.service
+endif
+
+if WITH_SERVER
+if USE_SYSTEMD
+# systemddir is already defined through configure.ac
+systemd_DATA += glusterd.service glusterfssharedstorage.service
+
+if BUILD_EVENTS
+systemd_DATA += glustereventsd.service
+endif
+endif
+endif
diff --git a/extras/systemd/gluster-ta-volume.service.in b/extras/systemd/gluster-ta-volume.service.in
new file mode 100644
index 00000000000..2802bca05bf
--- /dev/null
+++ b/extras/systemd/gluster-ta-volume.service.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=GlusterFS, Thin-arbiter process to maintain quorum for replica volume
+After=network.target
+
+[Service]
+Environment="LOG_LEVEL=WARNING"
+ExecStart=@prefix@/sbin/glusterfsd -N --volfile-id ta -f @GLUSTERD_WORKDIR@/thin-arbiter/thin-arbiter.vol --brick-port 24007 --xlator-option ta-server.transport.socket.listen-port=24007 -LWARNING
+Restart=always
+KillMode=process
+SuccessExitStatus=15
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in
new file mode 100644
index 00000000000..abb0d82911f
--- /dev/null
+++ b/extras/systemd/glusterd.service.in
@@ -0,0 +1,26 @@
+[Unit]
+Description=GlusterFS, a clustered file-system server
+Documentation=man:glusterd(8)
+StartLimitBurst=6
+StartLimitIntervalSec=3600
+Requires=@RPCBIND_SERVICE@
+After=network.target @RPCBIND_SERVICE@
+Before=network-online.target
+
+[Service]
+Type=forking
+PIDFile=@localstatedir@/run/glusterd.pid
+LimitNOFILE=65536
+Environment="LOG_LEVEL=INFO"
+EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd
+ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid  --log-level $LOG_LEVEL $GLUSTERD_OPTIONS
+KillMode=process
+TimeoutSec=300
+SuccessExitStatus=15
+Restart=on-abnormal
+RestartSec=60
+StartLimitBurst=6
+StartLimitInterval=3600
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/systemd/glustereventsd.service.in b/extras/systemd/glustereventsd.service.in
new file mode 100644
index 00000000000..f80b78199f6
--- /dev/null
+++ b/extras/systemd/glustereventsd.service.in
@@ -0,0 +1,16 @@
+[Unit]
+Description=Gluster Events Notifier
+After=network.target
+Documentation=man:glustereventsd(8)
+
+
+[Service]
+Environment=PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES_EXPANDED@:$PYTHONPATH
+Type=simple
+ExecStart=@SBIN_DIR@/glustereventsd --pid-file @localstatedir@/run/glustereventsd.pid
+ExecReload=/bin/kill -SIGUSR2 $MAINPID
+KillMode=control-group
+PIDFile=@localstatedir@/run/glustereventsd.pid
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/systemd/glusterfssharedstorage.service.in b/extras/systemd/glusterfssharedstorage.service.in
new file mode 100644
index 00000000000..723ff49afb7
--- /dev/null
+++ b/extras/systemd/glusterfssharedstorage.service.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=Mount glusterfs sharedstorage
+Requires=glusterd.service remote-fs-pre.target local-fs.target
+
+[Service]
+Type=forking
+ExecStart=@GLUSTERFS_LIBEXECDIR@/mount-shared-storage.sh
+Restart=on-failure
+RestartSec=3
+RestartForceExitStatus=1
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/test/bug-920583.t b/extras/test/bug-920583.t
new file mode 100755
index 00000000000..eedbb800ac0
--- /dev/null
+++ b/extras/test/bug-920583.t
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+##Copy this file to tests/bugs before running run.sh (cp extras/test/bug-920583.t tests/bugs/)
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+logdir=`gluster --print-logdir`
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+function log-file-name()
+{
+    logfilename=$M0".log"
+    echo ${logfilename:1} | tr / -
+}
+
+log_file=$logdir"/"`log-file-name`
+
+lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l`
+no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l`
+mount -t glusterfs $H0:/$V0 $M0 -o "xlator-option=*dht.assert-no-child-down=yes,xlator-option=*dht.lookup-unhashed=yes"
+touch $M0/file1;
+
+new_lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l`
+new_no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l`
+EXPECT "1" expr $new_lookup_unhashed_count - $lookup_unhashed_count
+EXPECT "1" expr $new_no_child_down_count - $no_child_down_count
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/extras/test/gluster_commands.sh b/extras/test/gluster_commands.sh
index 896a05b6ff6..cb2a55fd5b5 100755
--- a/extras/test/gluster_commands.sh
+++ b/extras/test/gluster_commands.sh
@@ -1,21 +1,13 @@
 #!/bin/bash
 
-#   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
 
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 
 # This script tests the basics gluster cli commands.
@@ -54,13 +46,13 @@ gluster volume start vol
 gluster volume info
 sleep 1
 mount -t glusterfs `hostname`:vol /mnt/client
-sleep 1 
+sleep 1
 df -h
 
 echo "adding-brick......."
 gluster volume add-brick vol `hostname`:/exports/exp2
 gluster volume info
-sleep 1 
+sleep 1
 umount /mnt/client
 mount -t glusterfs `hostname`:vol /mnt/client
 df -h
@@ -102,7 +94,7 @@ sleep 1
 echo "removing brick......."
 gluster --mode=script volume remove-brick vol `hostname`:/exports/exp2
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
@@ -127,7 +119,7 @@ sleep 1
 echo "starting replicate volume......"
 gluster volume start mirror
 gluster volume info
-sleep 1 
+sleep 1
 mount -t glusterfs `hostname`:mirror /mnt/client
 sleep 1
 df -h
@@ -136,7 +128,7 @@ sleep 1
 echo "adding-brick......."
 gluster volume add-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
@@ -168,7 +160,7 @@ gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports
 #sleep for 5 seconds
 sleep 5
 gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 status
-gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp5 abort
+gluster volume replace-brick mirror `hostname`:/exports/exp5 `hostname`:/exports/exp1 abort
 
 gluster volume info
 sleep 1
@@ -178,14 +170,14 @@ sleep 1
 echo "removeing-brick....."
 gluster --mode=script volume remove-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
 echo "stopping replicate volume....."
 gluster --mode=script volume stop mirror
 gluster volume info
-sleep 1 
+sleep 1
 umount /mnt/client
 df -h
 
@@ -206,14 +198,14 @@ gluster volume start str
 gluster volume info
 sleep 1
 mount -t glusterfs `hostname`:str /mnt/client
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
 echo "adding brick...."
 gluster volume add-brick str `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
diff --git a/extras/test/ld-preload-test/README b/extras/test/ld-preload-test/README
index 725b94023c8..df80003844b 100644
--- a/extras/test/ld-preload-test/README
+++ b/extras/test/ld-preload-test/README
@@ -32,9 +32,9 @@ Instructions
         $ make
 
 (We've tested the tool on Debian and CentOS. If there are build errors or
-warnings, please do report them to glusterfs-devel@nongnu.org.)
+warnings, please do report them to gluster-devel@gluster.org.)
 
 2. Run the test.
         $ ./test-preload.sh > preload.log
 
-3. Mail the log to glusterfs-devel@nongnu.org.
+3. Mail the log to gluster-devel@gluster.org.
diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c
index 900d967a05b..d120c053a69 100644
--- a/extras/test/ld-preload-test/ld-preload-lib.c
+++ b/extras/test/ld-preload-test/ld-preload-lib.c
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 /* LD PRELOAD'able library
  * A very simple library that intercepts booster supported system calls
  * and prints a log message to stdout.
@@ -44,594 +34,582 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <dirent.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <sys/sendfile.h>
 
 /* Err number that is assigned to errno so that test application can
  * verify that the function was intercepted correctly.
  */
-#define PRELOAD_ERRNO_VERF      6449
-#define set_errno()             (errno = PRELOAD_ERRNO_VERF)
+#define PRELOAD_ERRNO_VERF 6449
+#define set_errno() (errno = PRELOAD_ERRNO_VERF)
 
-inline void
-intercept (char *call, int tabs)
+void
+intercept(char *call, int tabs)
 {
-        while (tabs > 0) {
-                fprintf (stdout, "\t");
-                --tabs;
-        }
+    while (tabs > 0) {
+        fprintf(stdout, "\t");
+        --tabs;
+    }
 
-        fprintf (stdout, "Intercepted by %s", call);
+    fprintf(stdout, "Intercepted by %s", call);
 }
 
 int
-creat64 (const char *pathname, mode_t mode)
+creat64(const char *pathname, mode_t mode)
 {
-        intercept ("creat64", 2);
-        set_errno ();
-        return -1;
+    intercept("creat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-creat (const char *pathname, mode_t mode)
+creat(const char *pathname, mode_t mode)
 {
-        intercept ("creat", 2);
-        set_errno ();
-        return -1;
+    intercept("creat", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-close (int fd)
+close(int fd)
 {
-        intercept ("close", 2);
-        set_errno ();
-        return -1;
+    intercept("close", 2);
+    set_errno();
+    return -1;
 }
 
 int
-open64 (const char *pathname, int flags, ...)
+open64(const char *pathname, int flags, ...)
 {
-        intercept ("open64", 2);
-        set_errno ();
-        return -1;
+    intercept("open64", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-open (const char *pathname, int flags, ...)
+open(const char *pathname, int flags, ...)
 {
-        intercept ("open", 2);
-        set_errno ();
-        return -1;
+    intercept("open", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-read (int fd, void *buf, size_t count)
+read(int fd, void *buf, size_t count)
 {
-        intercept ("read", 2);
-        set_errno ();
-        return -1;
+    intercept("read", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-readv (int fd, const struct iovec *vector, int count)
+readv(int fd, const struct iovec *vector, int count)
 {
-        intercept ("readv", 2);
-        set_errno ();
-        return -1;
+    intercept("readv", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pread (int fd, void *buf, size_t count, unsigned long offset)
+pread(int fd, void *buf, size_t count, unsigned long offset)
 {
-        intercept ("pread", 2);
-        set_errno ();
-        return -1;
+    intercept("pread", 2);
+    set_errno();
+    return -1;
 }
 
-
 ssize_t
-pread64 (int fd, void *buf, size_t count, uint64_t offset)
+pread64(int fd, void *buf, size_t count, uint64_t offset)
 {
-        intercept ("pread64", 2);
-        set_errno ();
-        return -1;
+    intercept("pread64", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-write (int fd, const void *buf, size_t count)
+write(int fd, const void *buf, size_t count)
 {
-        intercept ("write", 2);
-        set_errno ();
-        return -1;
+    intercept("write", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-writev (int fd, const struct iovec *vector, int count)
+writev(int fd, const struct iovec *vector, int count)
 {
-        intercept ("writev", 2);
-        set_errno ();
-        return -1;
+    intercept("writev", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pwrite (int fd, const void *buf, size_t count, unsigned long offset)
+pwrite(int fd, const void *buf, size_t count, unsigned long offset)
 {
-        intercept ("pwrite", 2);
-        set_errno ();
-        return -1;
+    intercept("pwrite", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-pwrite64 (int fd, const void *buf, size_t count, uint64_t offset)
+pwrite64(int fd, const void *buf, size_t count, uint64_t offset)
 {
-        intercept ("pwrite64", 2);
-        set_errno ();
-        return -1;
+    intercept("pwrite64", 2);
+    set_errno();
+    return -1;
 }
 
-
 off_t
-lseek (int fildes, unsigned long offset, int whence)
+lseek(int fildes, unsigned long offset, int whence)
 {
-        intercept ("lseek", 2);
-        set_errno ();
-        return -1;
+    intercept("lseek", 2);
+    set_errno();
+    return -1;
 }
 
 off_t
-lseek64 (int fildes, uint64_t offset, int whence)
+lseek64(int fildes, uint64_t offset, int whence)
 {
-        intercept ("lseek64", 2);
-        set_errno ();
-        return -1;
+    intercept("lseek64", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-dup (int fd)
+dup(int fd)
 {
-        intercept ("dup", 2);
-        set_errno ();
-        return -1;
+    intercept("dup", 2);
+    set_errno();
+    return -1;
 }
 
 int
-dup2 (int oldfd, int newfd)
+dup2(int oldfd, int newfd)
 {
-        intercept ("dup2", 2);
-        set_errno ();
-        return -1;
+    intercept("dup2", 2);
+    set_errno();
+    return -1;
 }
 
 int
-mkdir (const char *pathname, mode_t mode)
+mkdir(const char *pathname, mode_t mode)
 {
-        intercept ("mkdir", 2);
-        set_errno ();
-        return -1;
+    intercept("mkdir", 2);
+    set_errno();
+    return -1;
 }
 
 int
-rmdir (const char *pathname)
+rmdir(const char *pathname)
 {
-        intercept ("rmdir", 2);
-        set_errno ();
-        return -1;
+    intercept("rmdir", 2);
+    set_errno();
+    return -1;
 }
 
 int
-chmod (const char *pathname, mode_t mode)
+chmod(const char *pathname, mode_t mode)
 {
-        intercept ("chmod", 2);
-        set_errno ();
-        return -1;
+    intercept("chmod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-chown (const char *pathname, uid_t owner, gid_t group)
+chown(const char *pathname, uid_t owner, gid_t group)
 {
-        intercept ("chown", 2);
-        set_errno ();
-        return -1;
+    intercept("chown", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fchmod (int fd, mode_t mode)
+fchmod(int fd, mode_t mode)
 {
-        intercept ("fchmod", 2);
-        set_errno ();
-        return -1;
+    intercept("fchmod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fchown (int fd, uid_t uid, gid_t gid)
+fchown(int fd, uid_t uid, gid_t gid)
 {
-        intercept ("fchown", 2);
-        set_errno ();
-        return -1;
+    intercept("fchown", 2);
+    set_errno();
+    return -1;
 }
 
-int fsync (int fd)
+int
+fsync(int fd)
 {
-        intercept ("fsync", 2);
-        set_errno ();
-        return -1;
+    intercept("fsync", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-ftruncate (int fd, off_t length)
+ftruncate(int fd, off_t length)
 {
-        intercept ("ftruncate", 1);
-        set_errno ();
-        return -1;
+    intercept("ftruncate", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-ftruncate64 (int fd, off_t length)
+ftruncate64(int fd, off_t length)
 {
-        intercept ("ftruncate64", 1);
-        set_errno ();
-        return -1;
+    intercept("ftruncate64", 1);
+    set_errno();
+    return -1;
 }
 
 int
-link (const char *oldpath, const char *newname)
+link(const char *oldpath, const char *newname)
 {
-        intercept ("link", 2);
-        set_errno ();
-        return -1;
+    intercept("link", 2);
+    set_errno();
+    return -1;
 }
 
 int
-rename (const char *oldpath, const char *newpath)
+rename(const char *oldpath, const char *newpath)
 {
-        intercept ("rename", 2);
-        set_errno ();
-        return -1;
+    intercept("rename", 2);
+    set_errno();
+    return -1;
 }
 
 int
-utimes (const char *path, const struct timeval times[2])
+utimes(const char *path, const struct timeval times[2])
 {
-        intercept ("utimes", 2);
-        set_errno ();
-        return -1;
+    intercept("utimes", 2);
+    set_errno();
+    return -1;
 }
 
 int
-utime (const char *path, const struct utimbuf *buf)
+futimes(int fd, const struct timeval times[2])
 {
-        intercept ("utime", 2);
-        set_errno ();
-        return -1;
+    intercept("futimes", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-mknod (const char *path, mode_t mode, dev_t dev)
+utime(const char *path, const struct utimbuf *buf)
 {
-        intercept ("mknod", 2);
-        set_errno ();
-        return -1;
+    intercept("utime", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__xmknod (int ver, const char *path, mode_t mode, dev_t *dev)
+mknod(const char *path, mode_t mode, dev_t dev)
 {
-        intercept ("__xmknod", 2);
-        set_errno ();
-        return -1;
+    intercept("mknod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-mkfifo (const char *path, mode_t mode)
+__xmknod(int ver, const char *path, mode_t mode, dev_t *dev)
 {
-        intercept ("mkfifo", 2);
-        set_errno ();
-        return -1;
+    intercept("__xmknod", 2);
+    set_errno();
+    return -1;
 }
 
 int
-unlink (const char *path)
+mkfifo(const char *path, mode_t mode)
 {
-        intercept ("unlink", 2);
-        set_errno ();
-        return -1;
+    intercept("mkfifo", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-symlink (const char *oldpath, const char *newpath)
+unlink(const char *path)
 {
-        intercept ("symlink", 2);
-        set_errno ();
-        return -1;
+    intercept("unlink", 2);
+    set_errno();
+    return -1;
 }
 
 int
-readlink (const char *path, char *buf, size_t bufsize)
+symlink(const char *oldpath, const char *newpath)
 {
-        intercept ("readlink", 1);
-        set_errno ();
-        return -1;
+    intercept("symlink", 2);
+    set_errno();
+    return -1;
 }
 
+int
+readlink(const char *path, char *buf, size_t bufsize)
+{
+    intercept("readlink", 1);
+    set_errno();
+    return -1;
+}
 
 char *
-realpath (const char *path, char *resolved)
+realpath(const char *path, char *resolved)
 {
-        intercept ("realpath", 1);
-        set_errno ();
-        return NULL;
+    intercept("realpath", 1);
+    set_errno();
+    return NULL;
 }
 
-
 DIR *
-opendir (const char *path)
+opendir(const char *path)
 {
-        intercept ("opendir", 2);
-        set_errno ();
-        return NULL;
+    intercept("opendir", 2);
+    set_errno();
+    return NULL;
 }
 
-
 struct dirent *
-readdir (DIR *dir)
+readdir(DIR *dir)
 {
-        intercept ("readdir\t", 2);
-        set_errno ();
-        return NULL;
+    intercept("readdir\t", 2);
+    set_errno();
+    return NULL;
 }
 
 struct dirent *
-readdir64 (DIR *dir)
+readdir64(DIR *dir)
 {
-        intercept ("readdir64", 2);
-        set_errno ();
-        return NULL;
+    intercept("readdir64", 2);
+    set_errno();
+    return NULL;
 }
 
-
 int
-readdir_r (DIR *dir, struct dirent *entry, struct dirent **result)
+readdir_r(DIR *dir, struct dirent *entry, struct dirent **result)
 {
-        intercept ("readdir_r", 1);
-        set_errno ();
-        return -1;
+    intercept("readdir_r", 1);
+    set_errno();
+    return -1;
 }
 
 int
-readdir64_r (DIR *dir, struct dirent *entry, struct dirent **result)
+readdir64_r(DIR *dir, struct dirent *entry, struct dirent **result)
 {
-        intercept ("readdir64_r", 1);
-        set_errno ();
-        return -1;
+    intercept("readdir64_r", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-closedir (DIR *dh)
+closedir(DIR *dh)
 {
-        intercept ("closedir", 1);
-        set_errno ();
-        return -1;
+    intercept("closedir", 1);
+    set_errno();
+    return -1;
 }
 
 int
-__xstat (int ver, const char *path, struct stat *buf)
+__xstat(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__xstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__xstat\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-__xstat64 (int ver, const char *path, struct stat *buf)
+__xstat64(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__xstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__xstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-stat (const char *path, struct stat *buf)
+stat(const char *path, struct stat *buf)
 {
-        intercept ("stat", 2);
-        set_errno ();
-        return -1;
+    intercept("stat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-stat64 (const char *path, struct stat *buf)
+stat64(const char *path, struct stat *buf)
 {
-        intercept ("stat64", 2);
-        set_errno ();
-        return -1;
+    intercept("stat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__fxstat (int ver, int fd, struct stat *buf)
+__fxstat(int ver, int fd, struct stat *buf)
 {
-        intercept ("__fxstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__fxstat\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-__fxstat64 (int ver, int fd, struct stat *buf)
+__fxstat64(int ver, int fd, struct stat *buf)
 {
-        intercept ("__fxstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__fxstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fstat (int fd, struct stat *buf)
+fstat(int fd, struct stat *buf)
 {
-        intercept ("fstat", 2);
-        set_errno ();
-        return -1;
+    intercept("fstat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-fstat64 (int fd , struct stat *buf)
+fstat64(int fd, struct stat *buf)
 {
-        intercept ("fstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("fstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__lxstat (int ver, const char *path, struct stat *buf)
+__lxstat(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__lxstat\t", 2);
-        set_errno ();
-        return -1;
+    intercept("__lxstat\t", 2);
+    set_errno();
+    return -1;
 }
 
 int
-__lxstat64 (int ver, const char *path, struct stat *buf)
+__lxstat64(int ver, const char *path, struct stat *buf)
 {
-        intercept ("__lxstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("__lxstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lstat (const char *path, struct stat *buf)
+lstat(const char *path, struct stat *buf)
 {
-        intercept ("lstat", 2);
-        set_errno ();
-        return -1;
+    intercept("lstat", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lstat64 (const char *path, struct stat *buf)
+lstat64(const char *path, struct stat *buf)
 {
-        intercept ("lstat64", 2);
-        set_errno ();
-        return -1;
+    intercept("lstat64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-statfs (const char *path, struct statfs *buf)
+statfs(const char *path, struct statfs *buf)
 {
-        intercept ("statfs", 2);
-        set_errno ();
-        return -1;
+    intercept("statfs", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-statfs64 (const char *path, struct statfs *buf)
+statfs64(const char *path, struct statfs *buf)
 {
-        intercept ("statfs64", 2);
-        set_errno ();
-        return -1;
+    intercept("statfs64", 2);
+    set_errno();
+    return -1;
 }
 
 int
-statvfs (const char *path, struct statvfs *buf)
+statvfs(const char *path, struct statvfs *buf)
 {
-        intercept ("statvfs\t", 2);
-        set_errno ();
-        return -1;
+    intercept("statvfs\t", 2);
+    set_errno();
+    return -1;
 }
 
-
 int
-statvfs64 (const char *path, struct statvfs *buf)
+statvfs64(const char *path, struct statvfs *buf)
 {
-        intercept ("statvfs64", 2);
-        set_errno ();
-        return -1;
+    intercept("statvfs64", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-getxattr (const char *path, const char *name, void *value, size_t size)
+getxattr(const char *path, const char *name, void *value, size_t size)
 {
-        intercept ("getxattr", 1);
-        set_errno ();
-        return -1;
+    intercept("getxattr", 1);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-lgetxattr (const char *path, const char *name, void *value, size_t size)
+lgetxattr(const char *path, const char *name, void *value, size_t size)
 {
-        intercept ("lgetxattr", 1);
-        set_errno ();
-        return -1;
+    intercept("lgetxattr", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-remove (const char* path)
+remove(const char *path)
 {
-        intercept ("remove", 2);
-        set_errno ();
-        return -1;
+    intercept("remove", 2);
+    set_errno();
+    return -1;
 }
 
 int
-lchown (const char *path, uid_t owner, gid_t group)
+lchown(const char *path, uid_t owner, gid_t group)
 {
-        intercept ("lchown", 2);
-        set_errno ();
-        return -1;
+    intercept("lchown", 2);
+    set_errno();
+    return -1;
 }
 
 void
-rewinddir (DIR *dirp)
+rewinddir(DIR *dirp)
 {
-        intercept ("rewinddir", 1);
-        set_errno ();
-        return;
+    intercept("rewinddir", 1);
+    set_errno();
+    return;
 }
 
 void
-seekdir (DIR *dirp, off_t offset)
+seekdir(DIR *dirp, off_t offset)
 {
-        intercept ("seekdir", 2);
-        set_errno ();
-        return;
+    intercept("seekdir", 2);
+    set_errno();
+    return;
 }
 
 off_t
-telldir (DIR *dirp)
+telldir(DIR *dirp)
 {
-        intercept ("telldir", 2);
-        set_errno ();
-        return -1;
+    intercept("telldir", 2);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-sendfile (int out_fd, int in_fd, off_t *offset, size_t count)
+sendfile(int out_fd, int in_fd, off_t *offset, size_t count)
 {
-        intercept ("sendfile\t", 1);
-        set_errno ();
-        return -1;
+    intercept("sendfile\t", 1);
+    set_errno();
+    return -1;
 }
 
 ssize_t
-sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count)
+sendfile64(int out_fd, int in_fd, off_t *offset, size_t count)
 {
-        intercept ("sendfile64", 1);
-        set_errno ();
-        return -1;
+    intercept("sendfile64", 1);
+    set_errno();
+    return -1;
 }
 
-
 int
-fcntl (int fd, int cmd, ...)
+fcntl(int fd, int cmd, ...)
 {
-        intercept ("fcntl", 2);
-        set_errno ();
-        return -1;
+    intercept("fcntl", 2);
+    set_errno();
+    return -1;
 }
-
diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c
index b67dad23c15..54dde8c7d54 100644
--- a/extras/test/ld-preload-test/ld-preload-test.c
+++ b/extras/test/ld-preload-test/ld-preload-test.c
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 /*
  * LD PRELOAD Test Tool
  *
@@ -56,322 +46,313 @@
 #include <sys/uio.h>
 #include <utime.h>
 #include <sys/time.h>
-#include <attr/xattr.h>
+#include <sys/xattr.h>
 #include <sys/sendfile.h>
 
-
-#define PRELOAD_ERRNO_VERF      6449
-inline void
+#define PRELOAD_ERRNO_VERF 6449
+void
 check_err(int ret, char *call, int tabs)
 {
-        while (tabs > 0) {
-                fprintf (stdout, "\t");
-                --tabs;
-        }
-        if (ret != -1) {
-                fprintf (stdout, "Not intercepted: %s\n", call);
-                return;
-        }
-
-        if (errno != PRELOAD_ERRNO_VERF) {
-                fprintf (stdout, "Not intercepted: %s: err: %s\n", call,
-                         strerror (errno));
-                return;
-        }
+    while (tabs > 0) {
+        fprintf(stdout, "\t");
+        --tabs;
+    }
+    if (ret != -1) {
+        fprintf(stdout, "Not intercepted: %s\n", call);
+        return;
+    }
 
-        fprintf (stdout, "Intercept verified: %s\n", call);
+    if (errno != PRELOAD_ERRNO_VERF) {
+        fprintf(stdout, "Not intercepted: %s: err: %s\n", call,
+                strerror(errno));
         return;
+    }
+
+    fprintf(stdout, "Intercept verified: %s\n", call);
+    return;
 }
 
 void
-usage (FILE *fp)
+usage(FILE *fp)
 {
-        fprintf (fp, "Usage: ld-preload-test <Options>\n");
-        fprintf (fp, "Options\n");
-        fprintf (fp, "\t--path\t\tPathname is used as the file/directory"
-                     " created for the test.\n");
-
+    fprintf(fp, "Usage: ld-preload-test <Options>\n");
+    fprintf(fp, "Options\n");
+    fprintf(fp,
+            "\t--path\t\tPathname is used as the file/directory"
+            " created for the test.\n");
 }
 
-
 int
-run_file_tests (char *testfile)
+run_file_tests(char *testfile)
 {
-        int             ret = -1;
-        struct stat     buf;
+    int ret = -1;
+    struct stat buf;
 
-        assert (testfile);
-        fprintf (stdout, "Testing creat");
-        ret = creat (testfile, S_IRWXU);
-        check_err (ret, "creat", 2);
+    assert(testfile);
+    fprintf(stdout, "Testing creat");
+    ret = creat(testfile, S_IRWXU);
+    check_err(ret, "creat", 2);
 
-        fprintf (stdout, "Testing close");
-        ret = close (ret);
-        check_err (ret, "close", 2);
+    fprintf(stdout, "Testing close");
+    ret = close(ret);
+    check_err(ret, "close", 2);
 
-        fprintf (stdout, "Testing open");
-        ret = open (testfile, O_RDONLY);
-        check_err (ret, "open", 2);
+    fprintf(stdout, "Testing open");
+    ret = open(testfile, O_RDONLY);
+    check_err(ret, "open", 2);
 
-        fprintf (stdout, "Testing read");
-        ret = read (0, NULL, 0);
-        check_err (ret, "read", 2);
+    fprintf(stdout, "Testing read");
+    ret = read(0, NULL, 0);
+    check_err(ret, "read", 2);
 
-        fprintf (stdout, "Testing readv");
-        ret = readv (0, NULL, 0);
-        check_err (ret, "readv", 2);
+    fprintf(stdout, "Testing readv");
+    ret = readv(0, NULL, 0);
+    check_err(ret, "readv", 2);
 
-        fprintf (stdout, "Testing pread");
-        ret = pread (0, NULL, 0, 0);
-        check_err (ret, "pread", 2);
+    fprintf(stdout, "Testing pread");
+    ret = pread(0, NULL, 0, 0);
+    check_err(ret, "pread", 2);
 
-        fprintf (stdout, "Testing write");
-        ret = write (0, NULL, 0);
-        check_err (ret, "write", 2);
+    fprintf(stdout, "Testing write");
+    ret = write(0, NULL, 0);
+    check_err(ret, "write", 2);
 
-        fprintf (stdout, "Testing writev");
-        ret = writev (0, NULL, 0);
-        check_err (ret, "writev", 2);
+    fprintf(stdout, "Testing writev");
+    ret = writev(0, NULL, 0);
+    check_err(ret, "writev", 2);
 
-        fprintf (stdout, "Testing pwrite");
-        ret = pwrite (0, NULL, 0, 0);
-        check_err (ret, "pwrite", 2);
+    fprintf(stdout, "Testing pwrite");
+    ret = pwrite(0, NULL, 0, 0);
+    check_err(ret, "pwrite", 2);
 
-        fprintf (stdout, "Testing lseek");
-        ret = lseek (0, 0, 0);
-        check_err (ret, "lseek", 2);
+    fprintf(stdout, "Testing lseek");
+    ret = lseek(0, 0, 0);
+    check_err(ret, "lseek", 2);
 
-        fprintf (stdout, "Testing dup");
-        ret = dup (0);
-        check_err (ret, "dup", 2);
+    fprintf(stdout, "Testing dup");
+    ret = dup(0);
+    check_err(ret, "dup", 2);
 
-        fprintf (stdout, "Testing dup2");
-        ret = dup2 (0, 0);
-        check_err (ret, "dup2", 2);
+    fprintf(stdout, "Testing dup2");
+    ret = dup2(0, 0);
+    check_err(ret, "dup2", 2);
 
-        fprintf (stdout, "Testing fchmod");
-        ret = fchmod (0, 0);
-        check_err (ret, "fchmod", 2);
+    fprintf(stdout, "Testing fchmod");
+    ret = fchmod(0, 0);
+    check_err(ret, "fchmod", 2);
 
-        fprintf (stdout, "Testing fchown");
-        ret = fchown (0, 0, 0);
-        check_err (ret, "fchown", 2);
+    fprintf(stdout, "Testing fchown");
+    ret = fchown(0, 0, 0);
+    check_err(ret, "fchown", 2);
 
-        fprintf (stdout, "Testing fsync");
-        ret = fsync (0);
-        check_err (ret, "fsync", 2);
+    fprintf(stdout, "Testing fsync");
+    ret = fsync(0);
+    check_err(ret, "fsync", 2);
 
-        fprintf (stdout, "Testing ftruncate");
-        ret = ftruncate (0, 0);
-        check_err (ret, "ftruncate", 1);
+    fprintf(stdout, "Testing ftruncate");
+    ret = ftruncate(0, 0);
+    check_err(ret, "ftruncate", 1);
 
-        fprintf (stdout, "Testing fstat");
-        ret = fstat (0, &buf);
-        check_err (ret, "fstat", 1);
+    fprintf(stdout, "Testing fstat");
+    ret = fstat(0, &buf);
+    check_err(ret, "fstat", 1);
 
-        fprintf (stdout, "Testing sendfile");
-        ret = sendfile (0, 0, NULL, 0);
-        check_err (ret, "sendfile", 1);
+    fprintf(stdout, "Testing sendfile");
+    ret = sendfile(0, 0, NULL, 0);
+    check_err(ret, "sendfile", 1);
 
-        fprintf (stdout, "Testing fcntl");
-        ret = fcntl (0, 0, NULL);
-        check_err (ret, "fcntl", 2);
+    fprintf(stdout, "Testing fcntl");
+    ret = fcntl(0, 0, NULL);
+    check_err(ret, "fcntl", 2);
 
-        fprintf (stdout, "Testing close");
-        ret = close (ret);
-        check_err (ret, "close", 2);
+    fprintf(stdout, "Testing close");
+    ret = close(ret);
+    check_err(ret, "close", 2);
 
-        fprintf (stdout, "Testing remove");
-        ret = remove (testfile);
-        check_err (ret, "remove", 2);
+    fprintf(stdout, "Testing remove");
+    ret = remove(testfile);
+    check_err(ret, "remove", 2);
 
-        return ret;
+    return ret;
 }
 
-
 int
-run_attr_tests (char *testfile)
+run_attr_tests(char *testfile)
 {
-        int             ret = -1;
-        char            *res = NULL;
-        struct stat     buf;
-        struct statfs   sbuf;
-        struct statvfs  svbuf;
-
-        assert (testfile);
-
-        fprintf (stdout, "Testing chmod");
-        ret = chmod (testfile, 0);
-        check_err (ret, "chmod", 2);
-
-        fprintf (stdout, "Testing chown");
-        ret = chown (testfile, 0, 0);
-        check_err (ret, "chown", 2);
-
-        fprintf (stdout, "Testing link");
-        ret = link (testfile, testfile);
-        check_err (ret, "link", 2);
-
-        fprintf (stdout, "Testing rename");
-        ret = rename (testfile, testfile);
-        check_err (ret, "rename", 2);
-
-        fprintf (stdout, "Testing utimes");
-        ret = utimes (testfile, NULL);
-        check_err (ret, "utimes", 2);
-
-        fprintf (stdout, "Testing utime");
-        ret = utime (testfile, NULL);
-        check_err (ret, "utime", 2);
-
-        fprintf (stdout, "Testing unlink");
-        ret = unlink (testfile);
-        check_err (ret, "unlink", 2);
-
-        fprintf (stdout, "Testing symlink");
-        ret = symlink (testfile, testfile);
-        check_err (ret, "symlink", 2);
-
-        fprintf (stdout, "Testing readlink");
-        ret = readlink (testfile, testfile, 0);
-        check_err (ret, "readlink", 2);
-
-        fprintf (stdout, "Testing realpath");
-        ret = 0;
-        res = realpath ((const char *)testfile, testfile);
-        if (!res)
-                ret = -1;
-        check_err (ret, "realpath", 2);
-
-        fprintf (stdout, "Testing stat");
-        ret = stat (testfile, &buf);
-        check_err (ret, "stat", 1);
-
-        fprintf (stdout, "Testing lstat");
-        ret = lstat (testfile, &buf);
-        check_err (ret, "lstat", 1);
-
-        fprintf (stdout, "Testing statfs");
-        ret = statfs (testfile, &sbuf);
-        check_err (ret, "statfs", 2);
-
-        fprintf (stdout, "Testing statvfs");
-        ret = statvfs (testfile, &svbuf);
-        check_err (ret, "statvfs", 1);
-
-        fprintf (stdout, "Testing getxattr");
-        ret = getxattr (testfile, NULL, NULL, 0);
-        check_err (ret, "getxattr", 2);
-
-        fprintf (stdout, "Testing lgetxattr");
-        ret = lgetxattr (testfile, NULL, NULL, 0);
-        check_err (ret, "lgetxattr", 1);
-
-        fprintf (stdout, "Testing lchown");
-        ret = lchown (testfile, 0, 0);
-        check_err (ret, "lchown", 2);
-        return 0;
+    int ret = -1;
+    char *res = NULL;
+    struct stat buf;
+    struct statfs sbuf;
+    struct statvfs svbuf;
+
+    assert(testfile);
+
+    fprintf(stdout, "Testing chmod");
+    ret = chmod(testfile, 0);
+    check_err(ret, "chmod", 2);
+
+    fprintf(stdout, "Testing chown");
+    ret = chown(testfile, 0, 0);
+    check_err(ret, "chown", 2);
+
+    fprintf(stdout, "Testing link");
+    ret = link(testfile, testfile);
+    check_err(ret, "link", 2);
+
+    fprintf(stdout, "Testing rename");
+    ret = rename(testfile, testfile);
+    check_err(ret, "rename", 2);
+
+    fprintf(stdout, "Testing utimes");
+    ret = utimes(testfile, NULL);
+    check_err(ret, "utimes", 2);
+
+    fprintf(stdout, "Testing utime");
+    ret = utime(testfile, NULL);
+    check_err(ret, "utime", 2);
+
+    fprintf(stdout, "Testing unlink");
+    ret = unlink(testfile);
+    check_err(ret, "unlink", 2);
+
+    fprintf(stdout, "Testing symlink");
+    ret = symlink(testfile, testfile);
+    check_err(ret, "symlink", 2);
+
+    fprintf(stdout, "Testing readlink");
+    ret = readlink(testfile, testfile, 0);
+    check_err(ret, "readlink", 2);
+
+    fprintf(stdout, "Testing realpath");
+    ret = 0;
+    res = realpath((const char *)testfile, testfile);
+    if (!res)
+        ret = -1;
+    check_err(ret, "realpath", 2);
+
+    fprintf(stdout, "Testing stat");
+    ret = stat(testfile, &buf);
+    check_err(ret, "stat", 1);
+
+    fprintf(stdout, "Testing lstat");
+    ret = lstat(testfile, &buf);
+    check_err(ret, "lstat", 1);
+
+    fprintf(stdout, "Testing statfs");
+    ret = statfs(testfile, &sbuf);
+    check_err(ret, "statfs", 2);
+
+    fprintf(stdout, "Testing statvfs");
+    ret = statvfs(testfile, &svbuf);
+    check_err(ret, "statvfs", 1);
+
+    fprintf(stdout, "Testing getxattr");
+    ret = getxattr(testfile, NULL, NULL, 0);
+    check_err(ret, "getxattr", 2);
+
+    fprintf(stdout, "Testing lgetxattr");
+    ret = lgetxattr(testfile, NULL, NULL, 0);
+    check_err(ret, "lgetxattr", 1);
+
+    fprintf(stdout, "Testing lchown");
+    ret = lchown(testfile, 0, 0);
+    check_err(ret, "lchown", 2);
+    return 0;
 }
 
-
 int
-run_dev_tests (char *testfile)
+run_dev_tests(char *testfile)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        assert (testfile);
+    assert(testfile);
 
-        fprintf (stdout, "Testing mknod");
-        ret = mknod (testfile, 0, 0);
-        check_err (ret, "mknod", 2);
+    fprintf(stdout, "Testing mknod");
+    ret = mknod(testfile, 0, 0);
+    check_err(ret, "mknod", 2);
 
-        fprintf (stdout, "Testing mkfifo");
-        ret = mkfifo (testfile, 0);
-        check_err (ret, "mkfifo", 2);
-        return 0;
+    fprintf(stdout, "Testing mkfifo");
+    ret = mkfifo(testfile, 0);
+    check_err(ret, "mkfifo", 2);
+    return 0;
 }
 
 int
-run_dir_tests (char *testpath)
+run_dir_tests(char *testpath)
 {
-        int             ret = -1;
-        DIR             *dh = NULL;
-        struct dirent   *dire = NULL;
-
-        assert (testpath);
-
-        fprintf (stdout, "Testing mkdir");
-        ret = mkdir (testpath, 0);
-        check_err (ret, "mkdir", 2);
-
-        fprintf (stdout, "Testing rmdir");
-        ret = rmdir (testpath);
-        check_err (ret, "rmdir", 2);
-
-        fprintf (stdout, "Testing opendir");
-        ret = 0;
-        dh = opendir (testpath);
-        if (!dh)
-                ret = -1;
-        check_err (ret, "opendir", 2);
-
-        fprintf (stdout, "Testing readdir");
-        ret = 0;
-        dire = readdir (dh);
-        if (!dire)
-                ret = -1;
-        check_err (ret, "readdir", 1);
-
-        fprintf (stdout, "Testing readdir_r");
-        ret = readdir_r (dh, dire, &dire);
-        check_err (ret, "readdir_r", 1);
-
-        fprintf (stdout, "Testing rewinddir");
-        rewinddir (dh);
-        check_err (-1, "rewinddir", 1);
-
-        fprintf (stdout, "Testing seekdir");
-        seekdir (dh, 0);
-        check_err (-1, "seekdir", 2);
-
-        fprintf (stdout, "Testing telldir");
-        ret = telldir (dh);
-        check_err (ret, "telldir", 2);
-
-        fprintf (stdout, "Testing closedir");
-        ret = closedir (dh);
-        check_err (ret, "closedir", 2);
-        return 0;
+    int ret = -1;
+    DIR *dh = NULL;
+    struct dirent *dire = NULL;
+
+    assert(testpath);
+
+    fprintf(stdout, "Testing mkdir");
+    ret = mkdir(testpath, 0);
+    check_err(ret, "mkdir", 2);
+
+    fprintf(stdout, "Testing rmdir");
+    ret = rmdir(testpath);
+    check_err(ret, "rmdir", 2);
+
+    fprintf(stdout, "Testing opendir");
+    ret = 0;
+    dh = opendir(testpath);
+    if (!dh)
+        ret = -1;
+    check_err(ret, "opendir", 2);
+
+    fprintf(stdout, "Testing readdir");
+    ret = 0;
+    dire = readdir(dh);
+    if (!dire)
+        ret = -1;
+    check_err(ret, "readdir", 1);
+
+    fprintf(stdout, "Testing readdir_r");
+    ret = readdir_r(dh, dire, &dire);
+    check_err(ret, "readdir_r", 1);
+
+    fprintf(stdout, "Testing rewinddir");
+    rewinddir(dh);
+    check_err(-1, "rewinddir", 1);
+
+    fprintf(stdout, "Testing seekdir");
+    seekdir(dh, 0);
+    check_err(-1, "seekdir", 2);
+
+    fprintf(stdout, "Testing telldir");
+    ret = telldir(dh);
+    check_err(ret, "telldir", 2);
+
+    fprintf(stdout, "Testing closedir");
+    ret = closedir(dh);
+    check_err(ret, "closedir", 2);
+    return 0;
 }
 
-
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        char            *testpath = NULL;
-        int             x = 0;
-
-        for (;x < argc; ++x) {
-                if (strcmp (argv[x], "--path") == 0) {
-                        testpath = argv[x+1];
-                        continue;
-                }
+    char *testpath = NULL;
+    int x = 0;
 
+    for (; x < argc; ++x) {
+        if (strcmp(argv[x], "--path") == 0) {
+            testpath = argv[x + 1];
+            continue;
         }
+    }
 
-        if (!testpath) {
-                fprintf (stderr, "--path not specified\n");
-                usage (stderr);
-                return -1;
-        }
+    if (!testpath) {
+        fprintf(stderr, "--path not specified\n");
+        usage(stderr);
+        return -1;
+    }
 
-        run_file_tests (testpath);
-        run_dir_tests (testpath);
-        run_attr_tests (testpath);
-        run_dev_tests (testpath);
+    run_file_tests(testpath);
+    run_dir_tests(testpath);
+    run_attr_tests(testpath);
+    run_dev_tests(testpath);
 
-        return 0;
+    return 0;
 }
-
-
diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c
new file mode 100644
index 00000000000..509952b4180
--- /dev/null
+++ b/extras/test/open-fd-tests.c
@@ -0,0 +1,67 @@
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/xattr.h>
+#include <errno.h>
+#include <string.h>
+
+int
+main(int argc, char *argv[])
+{
+    int ret = -1;
+    int fd = 0;
+    char *filename = NULL;
+    int loop = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char string[1024] = {
+        0,
+    };
+
+    if (argc > 1)
+        filename = argv[1];
+
+    if (!filename)
+        filename = "temp-fd-test-file";
+
+    fd = open(filename, O_RDWR | O_CREAT | O_TRUNC);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    while (loop < 1000) {
+        /* Use it as a mechanism to test time delays */
+        memset(string, 0, 1024);
+        scanf("%s", string);
+
+        ret = write(fd, string, strlen(string));
+        if (ret != strlen(string)) {
+            fprintf(stderr, "write failed : %s (%s %d)\n", strerror(errno),
+                    string, loop);
+            goto out;
+        }
+
+        ret = write(fd, "\n", 1);
+        if (ret != 1) {
+            fprintf(stderr, "write failed : %s (%d)\n", strerror(errno), loop);
+            goto out;
+        }
+
+        loop++;
+    }
+
+    fprintf(stdout, "finishing the test after %d loops\n", loop);
+
+    ret = 0;
+out:
+    if (fd)
+        close(fd);
+
+    return ret;
+}
diff --git a/extras/test/run.sh b/extras/test/run.sh
index e014a1033f1..4b3839cf679 100755
--- a/extras/test/run.sh
+++ b/extras/test/run.sh
@@ -1,21 +1,12 @@
 #!/bin/sh
 
-#   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
-
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 # Running gluster sanity test which starts glusterd and runs gluster commands, and exit at the first failure.
 $PWD/gluster_commands.sh
diff --git a/extras/test/stop_glusterd.sh b/extras/test/stop_glusterd.sh
index 7dfc8686ce8..a2db13f4241 100755
--- a/extras/test/stop_glusterd.sh
+++ b/extras/test/stop_glusterd.sh
@@ -1,21 +1,12 @@
 #!/bin/bash
 
-#   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
-
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 #This script stops the glusterd running on the machine. Helpful for gluster sanity script
 
diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c
new file mode 100644
index 00000000000..1d9c125db67
--- /dev/null
+++ b/extras/test/test-ffop.c
@@ -0,0 +1,920 @@
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/xattr.h>
+#include <errno.h>
+#include <string.h>
+#include <dirent.h>
+
+int
+fd_based_fops_1(char *filename);  // for fd based fops after unlink
+int
+fd_based_fops_2(char *filename);  // for fd based fops before unlink
+int
+dup_fd_based_fops(char *filename);  // fops based on fd after dup
+int
+path_based_fops(char *filename);  // for fops based on path
+int
+dir_based_fops(char *filename);  // for fops which operate on directory
+int
+link_based_fops(
+    char *filename);  // for fops which operate in link files (symlinks)
+int
+test_open_modes(
+    char *filename);  // to test open syscall with open modes available.
+int
+generic_open_read_write(
+    char *filename,
+    int flag);  // generic function which does open write and read.
+
+int
+main(int argc, char *argv[])
+{
+    int ret = -1;
+    char filename[255] = {
+        0,
+    };
+
+    if (argc > 1)
+        strcpy(filename, argv[1]);
+    else
+        strcpy(filename, "temp-xattr-test-file");
+
+    ret = fd_based_fops_1(strcat(filename, "_1"));
+    if (ret < 0)
+        fprintf(stderr, "fd based file operation 1 failed\n");
+    else
+        fprintf(stdout, "fd based file operation 1 passed\n");
+
+    ret = fd_based_fops_2(strcat(filename, "_2"));
+    if (ret < 0)
+        fprintf(stderr, "fd based file operation 2 failed\n");
+    else
+        fprintf(stdout, "fd based file operation 2 passed\n");
+
+    ret = dup_fd_based_fops(strcat(filename, "_3"));
+    if (ret < 0)
+        fprintf(stderr, "dup fd based file operation failed\n");
+    else
+        fprintf(stdout, "dup fd based file operation passed\n");
+
+    ret = path_based_fops(strcat(filename, "_4"));
+    if (ret < 0)
+        fprintf(stderr, "path based file operation failed\n");
+    else
+        fprintf(stdout, "path based file operation passed\n");
+
+    ret = dir_based_fops(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "directory based file operation failed\n");
+    else
+        fprintf(stdout, "directory based file operation passed\n");
+
+    ret = link_based_fops(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "link based file operation failed\n");
+    else
+        fprintf(stdout, "link based file operation passed\n");
+
+    ret = test_open_modes(strcat(filename, "_5"));
+    if (ret < 0)
+        fprintf(stderr, "testing modes of 'open' call failed\n");
+    else
+        fprintf(stdout, "testing modes of 'open' call passed\n");
+
+out:
+    return ret;
+}
+
+int
+fd_based_fops_1(char *filename)
+{
+    int fd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = unlink(filename);
+    if (ret < 0) {
+        fprintf(stderr, "unlink failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(wstr, "This is my string\n");
+    ret = write(fd, wstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(fd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = ftruncate(fd, 0);
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fstat(fd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(fd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(fd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(fd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(fd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (fd)
+        close(fd);
+
+    return ret;
+}
+
+int
+fd_based_fops_2(char *filename)
+{
+    int fd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = ftruncate(fd, 0);
+
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(wstr, "This is my second string\n");
+    ret = write(fd, wstr, strlen(wstr));
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(fd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = fstat(fd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(fd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(fd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(fd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(fd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(fd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+out:
+    if (fd)
+        close(fd);
+    unlink(filename);
+
+    return ret;
+}
+
+int
+path_based_fops(char *filename)
+{
+    int ret = -1;
+    int fd = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char newfilename[255] = {
+        0,
+    };
+
+    fd = creat(filename, 0644);
+    if (fd < 0) {
+        fprintf(stderr, "creat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = truncate(filename, 0);
+    if (ret < 0) {
+        fprintf(stderr, "truncate failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = stat(filename, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "stat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chmod(filename, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chown(filename, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "chown failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = setxattr(filename, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "setxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = listxattr(filename, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "listxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = getxattr(filename, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "getxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = removexattr(filename, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "removexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = access(filename, R_OK | W_OK);
+    if (ret < 0) {
+        fprintf(stderr, "access failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newfilename, filename);
+    strcat(newfilename, "_new");
+    ret = rename(filename, newfilename);
+    if (ret < 0) {
+        fprintf(stderr, "rename failed: %s\n", strerror(errno));
+        goto out;
+    }
+    unlink(newfilename);
+
+out:
+    if (fd)
+        close(fd);
+
+    unlink(filename);
+    return ret;
+}
+
+int
+dup_fd_based_fops(char *filename)
+{
+    int fd = 0;
+    int newfd = 0;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    char wstr[50] = {
+        0,
+    };
+    char rstr[50] = {
+        0,
+    };
+
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "open failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    newfd = dup(fd);
+    if (newfd < 0) {
+        ret = -1;
+        fprintf(stderr, "dup failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    close(fd);
+
+    strcpy(wstr, "This is my string\n");
+    ret = write(newfd, wstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "write failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lseek(newfd, 0, SEEK_SET);
+    if (ret < 0) {
+        fprintf(stderr, "lseek failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = read(newfd, rstr, strlen(wstr));
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "read failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = memcmp(rstr, wstr, strlen(wstr));
+    if (ret != 0) {
+        ret = -1;
+        fprintf(stderr, "read returning junk\n");
+        goto out;
+    }
+
+    ret = ftruncate(newfd, 0);
+    if (ret < 0) {
+        fprintf(stderr, "ftruncate failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fstat(newfd, &stbuf);
+    if (ret < 0) {
+        fprintf(stderr, "fstat failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchmod(newfd, 0640);
+    if (ret < 0) {
+        fprintf(stderr, "fchmod failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fchown(newfd, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "fchown failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsync(newfd);
+    if (ret < 0) {
+        fprintf(stderr, "fsync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fsetxattr(newfd, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fdatasync(newfd);
+    if (ret < 0) {
+        fprintf(stderr, "fdatasync failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = flistxattr(newfd, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "flistxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fgetxattr(newfd, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = fremovexattr(newfd, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (newfd)
+        close(newfd);
+    ret = unlink(filename);
+    if (ret < 0)
+        fprintf(stderr, "unlink failed : %s\n", strerror(errno));
+
+    return ret;
+}
+
+int
+dir_based_fops(char *dirname)
+{
+    int ret = -1;
+    DIR *dp = NULL;
+    char buff[255] = {
+        0,
+    };
+    struct dirent *dbuff = {
+        0,
+    };
+    struct stat stbuff = {
+        0,
+    };
+    char newdname[255] = {
+        0,
+    };
+    char *cwd = NULL;
+
+    ret = mkdir(dirname, 0755);
+    if (ret < 0) {
+        fprintf(stderr, "mkdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    dp = opendir(dirname);
+    if (dp == NULL) {
+        fprintf(stderr, "opendir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    dbuff = readdir(dp);
+    if (NULL == dbuff) {
+        fprintf(stderr, "readdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = closedir(dp);
+    if (ret < 0) {
+        fprintf(stderr, "closedir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = stat(dirname, &stbuff);
+    if (ret < 0) {
+        fprintf(stderr, "stat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chmod(dirname, 0744);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = chown(dirname, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "chmod failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = setxattr(dirname, "trusted.xattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "setxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = listxattr(dirname, NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "listxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = getxattr(dirname, "trusted.xattr-test", NULL, 0);
+    if (ret <= 0) {
+        ret = -1;
+        fprintf(stderr, "getxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = removexattr(dirname, "trusted.xattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "removexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newdname, dirname);
+    strcat(newdname, "/../");
+    ret = chdir(newdname);
+    if (ret < 0) {
+        fprintf(stderr, "chdir failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    cwd = getcwd(buff, 255);
+    if (NULL == cwd) {
+        fprintf(stderr, "getcwd failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newdname, dirname);
+    strcat(newdname, "new");
+    ret = rename(dirname, newdname);
+    if (ret < 0) {
+        fprintf(stderr, "rename failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = rmdir(newdname);
+    if (ret < 0) {
+        fprintf(stderr, "rmdir failed: %s\n", strerror(errno));
+        return ret;
+    }
+
+out:
+    rmdir(dirname);
+    return ret;
+}
+
+int
+link_based_fops(char *filename)
+{
+    int ret = -1;
+    int fd = 0;
+    char newname[255] = {
+        0,
+    };
+    char linkname[255] = {
+        0,
+    };
+    struct stat lstbuf = {
+        0,
+    };
+
+    fd = creat(filename, 0644);
+    if (fd < 0) {
+        fd = 0;
+        fprintf(stderr, "creat failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(newname, filename);
+    strcat(newname, "_hlink");
+    ret = link(filename, newname);
+    if (ret < 0) {
+        fprintf(stderr, "link failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = unlink(filename);
+    if (ret < 0) {
+        fprintf(stderr, "unlink failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    strcpy(linkname, filename);
+    strcat(linkname, "_slink");
+    ret = symlink(newname, linkname);
+    if (ret < 0) {
+        fprintf(stderr, "symlink failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lstat(linkname, &lstbuf);
+    if (ret < 0) {
+        fprintf(stderr, "lstbuf failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lchown(linkname, 10001, 10001);
+    if (ret < 0) {
+        fprintf(stderr, "lchown failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lsetxattr(linkname, "trusted.lxattr-test", "working", 8, 0);
+    if (ret < 0) {
+        fprintf(stderr, "lsetxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = llistxattr(linkname, NULL, 0);
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "llistxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lgetxattr(linkname, "trusted.lxattr-test", NULL, 0);
+    if (ret < 0) {
+        ret = -1;
+        fprintf(stderr, "lgetxattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = lremovexattr(linkname, "trusted.lxattr-test");
+    if (ret < 0) {
+        fprintf(stderr, "lremovexattr failed: %s\n", strerror(errno));
+        goto out;
+    }
+
+out:
+    if (fd)
+        close(fd);
+    unlink(linkname);
+    unlink(newname);
+}
+
+int
+test_open_modes(char *filename)
+{
+    int ret = -1;
+
+    ret = generic_open_read_write(filename, O_CREAT | O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_WRONLY failed: \n");
+        goto out;
+    }
+
+    ret = generic_open_read_write(filename, O_CREAT | O_RDWR);
+    if (ret != 0) {
+        fprintf(stderr, "flag O_CREAT|O_RDWR failed\n");
+        goto out;
+    }
+
+    ret = generic_open_read_write(filename, O_CREAT | O_RDONLY);
+    if (ret != 0) {
+        fprintf(stderr, "flag O_CREAT|O_RDONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_WRONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_RDWR);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_RDWR failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_RDONLY);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_RDONLY failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_TRUNC | O_WRONLY);
+    if (3 != ret) {
+        fprintf(stderr, "flag O_TRUNC|O_WRONLY failed\n");
+        goto out;
+    }
+
+#if 0 /* undefined behaviour, unable to reliably test */
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_TRUNC|O_RDONLY);
+        if (0 != ret) {
+               fprintf (stderr, "flag O_TRUNC|O_RDONLY failed\n");
+               goto out;
+        }
+#endif
+
+    ret = generic_open_read_write(filename, O_CREAT | O_RDWR | O_SYNC);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n");
+        goto out;
+    }
+
+    ret = creat(filename, 0644);
+    close(ret);
+    ret = generic_open_read_write(filename, O_CREAT | O_EXCL);
+    if (0 != ret) {
+        fprintf(stderr, "flag O_CREAT|O_EXCL failed\n");
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+int
+generic_open_read_write(char *filename, int flag)
+{
+    int fd = 0;
+    int ret = -1;
+    char wstring[50] = {
+        0,
+    };
+    char rstring[50] = {
+        0,
+    };
+
+    fd = open(filename, flag);
+    if (fd < 0) {
+        if (flag == O_CREAT | O_EXCL && errno == EEXIST) {
+            unlink(filename);
+            return 0;
+        } else {
+            fd = 0;
+            fprintf(stderr, "open failed: %s\n", strerror(errno));
+            return 1;
+        }
+    }
+
+    strcpy(wstring, "My string to write\n");
+    ret = write(fd, wstring, strlen(wstring));
+    if (ret <= 0) {
+        if (errno != EBADF) {
+            fprintf(stderr, "write failed: %s\n", strerror(errno));
+            close(fd);
+            unlink(filename);
+            return 2;
+        }
+    }
+
+    ret = lseek(fd, 0, SEEK_SET);
+    if (ret < 0) {
+        close(fd);
+        unlink(filename);
+        return 4;
+    }
+
+    ret = read(fd, rstring, strlen(wstring));
+    if (ret < 0) {
+        close(fd);
+        unlink(filename);
+        return 3;
+    }
+
+    /* Compare the rstring with wstring. But we do not want to return
+     * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or
+     * O_TRUNC|O_RDONLY. Because in that case we are not writing
+     * anything to the file.*/
+
+    ret = memcmp(wstring, rstring, strlen(wstring));
+    if (0 != ret && !(flag == O_CREAT | O_RDONLY || flag == O_RDONLY ||
+                      flag == O_TRUNC | O_RDONLY)) {
+        fprintf(stderr, "read is returning junk\n");
+        close(fd);
+        unlink(filename);
+        return 4;
+    }
+
+    close(fd);
+    unlink(filename);
+    return 0;
+}
diff --git a/extras/thin-arbiter/setup-thin-arbiter.sh b/extras/thin-arbiter/setup-thin-arbiter.sh
new file mode 100755
index 00000000000..0681b30ef3f
--- /dev/null
+++ b/extras/thin-arbiter/setup-thin-arbiter.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+#  Copyright (c) 2018-2019 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
+
+#  This tool has been developed to setup thin-arbiter process on a node.
+#  Seting up a thin arbiter process involves following files -
+#  1 - thin-arbiter.vol
+#  Thin-arbiter (TA) process will use the graph in this file to load the
+#  required translators.
+#  2 - gluster-ta-volume.service (generated by gluster-ta-volume.service.in)
+#  TA process would be running as systemd service.
+#
+#  TA process uses a location to save TA id files for every subvolume.
+#  This location can be taken as input from user. Once provided and the
+#  TA process is started on a node, it can not be changed using this
+#  script or by any other mean. The same location should be used in
+#  the gluster CLI when creating thin-arbiter volumes.
+
+MYPATH=`dirname $0`
+
+volloc="/var/lib/glusterd/thin-arbiter"
+mkdir -p $volloc
+
+if [ -f /etc/glusterfs/thin-arbiter.vol ]; then
+    volfile=/etc/glusterfs/thin-arbiter.vol
+else
+    volfile=$MYPATH/thin-arbiter.vol
+fi
+
+tafile="$volloc/thin-arbiter.vol"
+
+
+help () {
+    echo " "
+    echo '  This tool helps to setup thin-arbiter (TA) process on a node.
+  TA process uses a location to save TA id files for every subvolume.
+  This location can be taken as input from user. Once provided and the
+  TA process is started on a node, it can not be changed using this script
+  or by any other mean. The same location should be used in gluster CLI
+  when creating thin-arbiter volumes.
+
+  usage: setup-thin-arbiter.sh [-s] [-h]
+    options:
+    -s - Setup thin-arbiter file path and start process
+    -h - Show this help message and exit
+'
+}
+
+volfile_set_brick_path () {
+    while read -r line
+    do
+        dir=`echo "$line" | cut -d' ' -f 2`
+        if [ "$dir" = "directory" ]; then
+            bpath=`echo "$line" | cut -d' ' -f 3`
+            sed -i -- 's?'$bpath'?'$1'?g' $tafile
+            return
+        fi
+    done < $tafile
+}
+
+check_ta_proc () {
+    pro=`ps aux | grep thin-arbiter.vol | grep "volfile-id"`
+    if [ "${pro}" = '' ]; then
+            echo ""
+    else
+        curr_loc=`cat $volloc/thin-arbiter.vol | grep option | grep directory`
+        loc=`echo "${curr_loc##* }"`
+        echo "******************************************************"
+        echo "Error:"
+        echo "Thin-arbiter process is running with thin-arbiter path = $loc"
+        echo "Can not change TA path on this host now."
+        echo "$pro"
+        echo "******************************************************"
+        exit 1
+    fi
+}
+
+getpath () {
+    check_ta_proc
+    echo "******************************************************"
+    echo "User will be required to enter a path/folder for arbiter volume."
+    echo "Please note that this path will be used for ALL VOLUMES using this"
+    echo "node to host thin-arbiter. After setting, if a volume"
+    echo "has been created using this host and path then path for"
+    echo "thin-arbiter can not be changed "
+    echo "******************************************************"
+    echo " "
+    while true;
+    do
+        echo -n "Enter brick path for thin arbiter volumes: "
+        echo " "
+        read tapath
+        if [ "${tapath}" = '' ]; then
+            echo "Please enter valid path"
+            continue
+        else
+            echo "Entered brick path : $tapath "
+            echo "Please note that this brick path will be used for ALL"
+            echo "VOLUMES using this node to host thin-arbiter brick"
+            echo -n "Want to continue? (y/N): "
+            echo " "
+            read cont
+
+            if [ "${cont}" = 'N' ] || [ "${cont}" = 'n' ]; then
+                exit 0
+            else
+                break
+            fi
+        fi
+    done
+}
+
+setup () {
+    getpath
+    mkdir -p $tapath/.glusterfs/indices
+    if [ -d $tapath/.glusterfs/indices ]; then
+        echo " "
+    else
+        echo "Could not create $tapath/.glusterfs/indices directory, check provided ta path."
+        exit 1
+    fi
+
+    cp -f --backup --suffix=_old $volfile $volloc/thin-arbiter.vol
+    volfile_set_brick_path "$tapath"
+
+    echo "Directory path to be used for thin-arbiter volume is: $tapath"
+    echo " "
+    echo "========================================================"
+
+    if [ -f /usr/lib/systemd/system/gluster-ta-volume.service ]; then
+        echo "Starting thin-arbiter process"
+    else
+        cp $MYPATH/../systemd/gluster-ta-volume.service /etc/systemd/system/
+        echo "Starting thin-arbiter process"
+        chmod 0644 /etc/systemd/system/gluster-ta-volume.service
+    fi
+
+    systemctl daemon-reload
+    systemctl enable gluster-ta-volume
+    systemctl stop gluster-ta-volume
+    systemctl start gluster-ta-volume
+
+    if [ $? == 0 ]; then
+        echo "thin-arbiter process has been setup and running"
+    else
+        echo "Failed to setup thin arbiter"
+        exit 1
+    fi
+
+}
+
+main()
+{
+
+    if [ "$#" -ne 1 ]; then
+        help
+        exit 0
+    fi
+
+    while getopts "sh" opt; do
+        case $opt in
+            h)
+                help
+                exit 0
+                ;;
+            s)
+                setup
+                exit 0
+                ;;
+            *)
+                help
+                exit 0
+                ;;
+        esac
+    done
+}
+
+main "$@"
diff --git a/extras/thin-arbiter/thin-arbiter.vol b/extras/thin-arbiter/thin-arbiter.vol
new file mode 100644
index 00000000000..c76babc7b3c
--- /dev/null
+++ b/extras/thin-arbiter/thin-arbiter.vol
@@ -0,0 +1,57 @@
+volume ta-posix
+    type storage/posix
+    option directory /mnt/thin-arbiter
+end-volume
+
+volume ta-thin-arbiter
+    type features/thin-arbiter
+    subvolumes ta-posix
+end-volume
+
+volume ta-locks
+    type features/locks
+    option notify-contention yes
+    subvolumes ta-thin-arbiter
+end-volume
+
+volume ta-upcall
+    type features/upcall
+    option cache-invalidation off
+    subvolumes ta-locks
+end-volume
+
+volume ta-io-threads
+    type performance/io-threads
+    subvolumes ta-upcall
+end-volume
+
+volume ta-index
+    type features/index
+    option xattrop-pending-watchlist trusted.afr.ta-
+    option xattrop-dirty-watchlist trusted.afr.dirty
+    option index-base /mnt/thin-arbiter/.glusterfs/indices
+    subvolumes ta-io-threads
+end-volume
+
+volume /mnt/thin-arbiter
+    type debug/io-stats
+    option count-fop-hits off
+    option latency-measurement off
+    option unique-id /mnt/thin-arbiter
+    subvolumes ta-index
+end-volume
+
+volume ta-server
+    type protocol/server
+    option transport.listen-backlog 10
+    option transport.socket.keepalive-count 9
+    option transport.socket.keepalive-interval 2
+    option transport.socket.keepalive-time 20
+    option transport.tcp-user-timeout 0
+    option transport.socket.keepalive 1
+    option auth.addr./mnt/thin-arbiter.allow *
+    option auth-path /mnt/thin-arbiter
+    option transport.address-family inet
+    option transport-type tcp
+    subvolumes /mnt/thin-arbiter
+end-volume
diff --git a/extras/volfilter.py b/extras/volfilter.py
new file mode 100644
index 00000000000..5558a1beff4
--- /dev/null
+++ b/extras/volfilter.py
@@ -0,0 +1,168 @@
+# Copyright (c) 2010-2011 Red Hat, Inc.
+#
+# This file is part of HekaFS.
+#
+# HekaFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License, version 3, as published by the Free
+# Software Foundation.
+#
+# HekaFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License * along
+# with HekaFS.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+import copy
+import string
+import sys
+import types
+
+good_xlators = [
+	"cluster/afr",
+	"cluster/dht",
+	"cluster/distribute",
+	"cluster/replicate",
+	"cluster/stripe",
+	"debug/io-stats",
+	"features/access-control",
+	"features/locks",
+	"features/marker",
+	"features/uidmap",
+	"performance/io-threads",
+	"protocol/client",
+	"protocol/server",
+	"storage/posix",
+]
+
+def copy_stack (old_xl, suffix, recursive=False):
+	if recursive:
+		new_name = old_xl.name + "-" + suffix
+	else:
+		new_name = suffix
+	new_xl = Translator(new_name)
+	new_xl.type = old_xl.type
+	# The results with normal assignment here are . . . amusing.
+	new_xl.opts = copy.deepcopy(old_xl.opts)
+	for sv in old_xl.subvols:
+		new_xl.subvols.append(copy_stack(sv, suffix, True))
+	# Patch up the path at the bottom.
+	if new_xl.type == "storage/posix":
+		new_xl.opts["directory"] += ("/" + suffix)
+	return new_xl
+
+def cleanup (parent, graph):
+	if parent.type in good_xlators:
+		# Temporary fix so that HekaFS volumes can use the
+		# SSL-enabled multi-threaded socket transport.
+		if parent.type == "protocol/server":
+			parent.type = "protocol/server2"
+			parent.opts["transport-type"] = "ssl"
+		elif parent.type == "protocol/client":
+			parent.type = "protocol/client2"
+			parent.opts["transport-type"] = "ssl"
+		sv = []
+		for child in parent.subvols:
+			sv.append(cleanup(child, graph))
+		parent.subvols = sv
+	else:
+		parent = cleanup(parent.subvols[0], graph)
+	return parent
+
+class Translator:
+	def __init__ (self, name):
+		self.name = name
+		self.type = ""
+		self.opts = {}
+		self.subvols = []
+		self.dumped = False
+	def __repr__ (self):
+		return "<Translator %s>" % self.name
+
+def load (path):
+	# If it's a string, open it; otherwise, assume it's already a
+	# file-like object (most notably from urllib*).
+	if type(path) in (str,):
+		fp = file(path, "r")
+	else:
+		fp = path
+	all_xlators = {}
+	xlator = None
+	last_xlator = None
+	while True:
+		text = fp.readline()
+		if text == "":
+			break
+		text = text.split()
+		if not len(text):
+			continue
+		if text[0] == "volume":
+			if xlator:
+				raise RuntimeError("nested volume definition")
+			xlator = Translator(text[1])
+			continue
+		if not xlator:
+			raise RuntimeError("text outside volume definition")
+		if text[0] == "type":
+			xlator.type = text[1]
+			continue
+		if text[0] == "option":
+			xlator.opts[text[1]] = ''.join(text[2:])
+			continue
+		if text[0] == "subvolumes":
+			for sv in text[1:]:
+				xlator.subvols.append(all_xlators[sv])
+			continue
+		if text[0] == "end-volume":
+			all_xlators[xlator.name] = xlator
+			last_xlator = xlator
+			xlator = None
+			continue
+		raise RuntimeError("unrecognized keyword %s" % text[0])
+	if xlator:
+		raise RuntimeError("unclosed volume definition")
+	return all_xlators, last_xlator
+
+def generate (graph, last, stream=sys.stdout):
+	for sv in last.subvols:
+		if not sv.dumped:
+			generate(graph, sv, stream)
+			print("", file=stream)
+			sv.dumped = True
+	print("volume %s" % last.name, file=stream)
+	print("    type %s" % last.type, file=stream)
+	for k, v in last.opts.items():
+		print("    option %s %s" % (k, v), file=stream)
+	if last.subvols:
+		print("    subvolumes %s" % ''.join(
+			[ sv.name for sv in last.subvols ]), file=stream)
+	print("end-volume", file=stream)
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+	suffix = "-" + old_xl.type.split("/")[1]
+	if len(old_xl.name) > len(suffix):
+		if old_xl.name[-len(suffix):] == suffix:
+			old_xl.name = old_xl.name[:-len(suffix)]
+	new_xl = Translator(old_xl.name+suffix)
+	new_xl.type = old_xl.type
+	new_xl.opts = old_xl.opts
+	new_xl.subvols = old_xl.subvols
+	graph[new_xl.name] = new_xl
+	old_xl.name += ("-" + filt_type.split("/")[1])
+	old_xl.type = filt_type
+	old_xl.opts = opts
+	old_xl.subvols = [new_xl]
+	graph[old_xl.name] = old_xl
+
+def delete (graph, victim):
+	if len(victim.subvols) != 1:
+		raise RuntimeError("attempt to delete non-unary translator")
+	for xl in graph.itervalues():
+		while xl.subvols.count(victim):
+			i = xl.subvols.index(victim)
+			xl.subvols[i] = victim.subvols[0]
+
+if __name__ == "__main__":
+	graph, last = load(sys.argv[1])
+	generate(graph, last)
diff --git a/extras/volgen/Common.py b/extras/volgen/Common.py
deleted file mode 100644
index 298dee98943..00000000000
--- a/extras/volgen/Common.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import os, sys, re, string
-
-def check_duplicate_entry(args):
-    """Check duplicate entries in incoming arguments"""
-    _tmp = []
-    for server in args:
-        if server not in _tmp:
-            _tmp.append (server)
-        else:
-            print "Duplicate arguments detected (%s)" % server
-            raise ValueError
-
-    return
-
-def args2dict(args):
-
-    keyvalue = {}
-    for arg in args:
-        if int(arg.find(':')) == -1:
-            continue
-        first = arg.split(':')[0]
-        keyvalue[first] = []
-
-    for arg in args:
-        if int(arg.find(':')) == -1:
-            continue
-        first = arg.split(':')[0]
-        if arg.split(':')[1] not in keyvalue[first]:
-            if arg.split(':')[1][0] != '/':
-                print "Absolute export path required for %s" % arg
-                raise ValueError
-            keyvalue[first].append (arg.split(':')[1])
-
-    return keyvalue
-
-def args2array(args):
-
-    array = []
-
-    for arg in args:
-        if int(arg.find(':')) == -1:
-            continue
-        array.append(arg)
-
-    return array
-
-def list_export_vols(configdir, volumename):
-
-    list_export = []
-    if os.path.isdir(configdir):
-        for line in os.listdir(configdir):
-            if re.match(r'[a-zA-Z0-9_]\S+%s-export.vol' % volumename, line):
-                list_export.append(line)
-
-    return list_export
-
-def get_old_server_args(exports, configdir):
-
-    list_args = []
-    for export in exports:
-        array = gfParser("%s/%s" % (configdir, export))
-        for dt in array:
-            if dt.has_key('option'):
-                if re.match("\w+tory", dt['option']):
-                    list_args.append(export.split('-')[0] + ":" + dt['option'].split()[1])
-
-    return list_args
-
-def gfParser (volfile):
-
-    volfile_rl = open (volfile).readlines()
-    volume_array = []
-    for line in volfile_rl:
-        line = line.strip()
-        volfile_dict = {}
-        if re.match(r"[a-zA-Z0-9_]+", line):
-            if line.split() > 1:
-                volfile_dict[line.split()[0]] = string.join (line.split()[1:], ' ')  
-            else: 
-                volfile_dict[line.split()[0]] = " "
-            volume_array.append(volfile_dict)
-
-    return volume_array
diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py
deleted file mode 100644
index 2f548d7c73c..00000000000
--- a/extras/volgen/CreateVolfile.py
+++ /dev/null
@@ -1,684 +0,0 @@
-#    Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com>
-#    This file is part of GlusterFS.
-
-#    GlusterFS is free software; you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published
-#    by the Free Software Foundation; either version 3 of the License,
-#    or (at your option) any later version.
-
-#    GlusterFS is distributed in the hope that it will be useful, but
-#    WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#    General Public License for more details.
-
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see
-#    <http://www.gnu.org/licenses/>.
-
-import os, sys, string
-
-#Cachesize calculator
-cache_size = "`echo $(( $(grep 'MemTotal' /proc/meminfo | sed 's/[^0-9]//g') / 5120 ))`"
-
-class CreateVolfile:
-
-    def __init__ (self, server_dict, server, transport,
-                  transports, options, server_array):
-
-        self.host_dict = server_dict
-        self.host = server
-        self.volume_name = options.volume_name
-        self.transport = transport
-        self.transports = transports
-        self.auth_parameters = options.auth_param
-        self.raid_type = options.raid_type
-        self.ib_devport = options.ib_dev
-        self.conf_dir = options.conf_dir
-        self.host_array = server_array
-        self.enable_safe_mode = options.enable_safe_mode
-        self.volume_size_server = options.size_server
-        self.volume_size_client = options.size_client
-        self.nfs = options.need_nfs
-        self.num_replica = options.num_replica
-        self.num_stripe = options.num_stripe
-        if options.port:
-            self.gfs_port    = options.port
-            self.gfs_ib_port = options.port + 1
-        else:
-            self.gfs_port    = 24007
-            self.gfs_ib_port = 24008
-
-    def create_pmap_mount_volfile (self):
-
-        raid_type = self.raid_type
-
-        mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport))
-        mount_fd = file ("%s" % (mount_volfile), "w")
-
-        print "Generating client volfiles.. '%s'" % mount_volfile
-
-        cmdline = string.join (sys.argv, ' ')
-
-        mount_fd.write ("## file auto generated by %s\n" % sys.argv[0])
-        mount_fd.write ("# Cmd line:\n")
-        mount_fd.write ("# $ %s\n\n" % cmdline)
-
-        if raid_type is not None:
-            # Used for later usage
-            mount_fd.write ("# RAID %d\n" % raid_type)
-
-        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
-        subvolumes = []
-        for host in self.host_dict.keys():
-            i = 1
-            for export in self.host_dict[host]:
-                mount_fd.write ("volume %s-%s\n" % (host,export))
-                mount_fd.write ("    type protocol/client\n")
-                mount_fd.write ("    option transport-type %s\n" %
-                                self.transport)
-                mount_fd.write ("    option remote-host %s\n" % host)
-                if self.transport == 'ib-verbs':
-                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
-                                    self.ib_devport)
-                if self.transport == 'tcp':
-                    mount_fd.write ("    option transport.socket.nodelay on\n")
-
-                mount_fd.write ("    option remote-subvolume %s\n" % export)
-                mount_fd.write ("end-volume\n\n")
-                i += 1
-                subvolumes.append(str(host) + '-' + str(export))
-
-        # Stripe section.. if given
-        if raid_type is 0:
-            max_stripe_idx = len (subvolumes) / self.num_stripe
-            stripe_idx = 0
-            index = 0
-            while index < max_stripe_idx:
-                mount_fd.write ("volume stripe-%d\n" % index)
-                mount_fd.write ("    type cluster/stripe\n")
-                mount_fd.write ("#    option block-size 128k\n")
-                mount_fd.write ("#    option use-xattr no\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[stripe_idx])
-                sub_idx = 1
-                while sub_idx < self.num_stripe:
-                    mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                stripe_idx += self.num_stripe
-                index +=1
-
-        # Replicate section
-        if raid_type is 1:
-            max_mirror_idx = len (subvolumes) / self.num_replica
-            mirror_idx = 0
-            index = 0
-            while index < max_mirror_idx:
-                mount_fd.write ("volume mirror-%d\n" % index)
-                mount_fd.write ("    type cluster/replicate\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[mirror_idx])
-                sub_idx = 1
-                while sub_idx < self.num_replica:
-                    mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                mirror_idx += self.num_replica
-                index += 1
-
-        # Distribute section
-        if raid_type is 0:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("stripe-%d" % flag)
-                flag += 1
-
-        if raid_type is 1:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("mirror-%d" % flag)
-                flag += 1
-
-        if len (subvolumes) > 1:
-            mount_fd.write ("volume distribute\n")
-            mount_fd.write ("    type cluster/distribute\n")
-            mount_fd.write ("#   option unhashed-sticky-bit yes"
-                           "  # Used for migrating data while adding new nodes\n")
-            mount_fd.write ("#   option min-free-disk 5%"
-                           "  # Minimum free disk available on the volume\n")
-            mount_fd.write ("    subvolumes %s\n" %
-                                 string.join (subvolumes,' '))
-            mount_fd.write ("end-volume\n\n")
-            subvolumes[0] = "distribute"
-
-        if self.volume_size_client:
-            mount_fd.write ("volume quota\n")
-            mount_fd.write ("    type features/quota\n")
-            mount_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_client)
-            mount_fd.write ("#   option minimum-free-disk-limit 10GB\n"
-                            "#   minimum free disk value (default) 0\n")
-            mount_fd.write ("#   option refresh-interval 10\n")
-            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
-            mount_fd.write ("end-volume\n\n")
-            subvolumes[0] = "quota"
-
-        if self.enable_safe_mode:
-            return
-
-        self.performance_mode (subvolumes[0], mount_fd)
-
-        return
-
-
-    def performance_mode (self, cluster_subvolume, volfile_fd):
-
-        volfile_fd.write ("volume writebehind\n")
-        volfile_fd.write ("    type performance/write-behind\n")
-        volfile_fd.write ("    option cache-size 4MB\n")
-        volfile_fd.write ("#   option enable-trickling-writes yes # Flush final write calls when network is free\n")
-        volfile_fd.write ("#   option enable-O_SYNC yes # Enable O_SYNC for write-behind\n")
-        volfile_fd.write ("#   option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n")
-        if self.volume_size_client:
-            volfile_fd.write ("    subvolumes quota\n")
-        else:
-            volfile_fd.write ("    subvolumes %s\n" % cluster_subvolume)
-            volfile_fd.write ("end-volume\n\n")
-
-        volfile_fd.write ("volume readahead\n")
-        volfile_fd.write ("    type performance/read-ahead\n")
-        volfile_fd.write ("    option page-count 4\n")
-        volfile_fd.write ("#   option force-atime-update yes # force updating atimes, default off\n")
-        volfile_fd.write ("    subvolumes writebehind\n")
-        volfile_fd.write ("end-volume\n\n")
-
-        volfile_fd.write ("volume iocache\n")
-        volfile_fd.write ("    type performance/io-cache\n")
-        volfile_fd.write ("    option cache-size %sMB\n" % cache_size)
-        volfile_fd.write ("    option cache-timeout 1\n")
-        volfile_fd.write ("#   option priority *.html:1,abc*:2 # Priority list for iocaching files\n")
-        volfile_fd.write ("    subvolumes readahead\n")
-        volfile_fd.write ("end-volume\n\n")
-
-        volfile_fd.write ("volume quickread\n")
-        volfile_fd.write ("    type performance/quick-read\n")
-        volfile_fd.write ("    option cache-timeout 1\n")
-        volfile_fd.write ("    option max-file-size 64kB\n")
-        volfile_fd.write ("    subvolumes iocache\n")
-        volfile_fd.write ("end-volume\n\n")
-
-        volfile_fd.write ("volume statprefetch\n")
-        volfile_fd.write ("    type performance/stat-prefetch\n")
-        volfile_fd.write ("    subvolumes quickread\n")
-        volfile_fd.write ("end-volume\n\n")
-
-        return
-
-    def create_pmap_nfs_volfile (self):
-
-        raid_type = self.raid_type
-
-        mount_volfile = "%s/%s-%s-nfs.vol" % (self.conf_dir, str(self.volume_name), str(self.transport))
-        mount_fd = file ("%s" % (mount_volfile), "w")
-
-        print "Generating client volfiles.. '%s'" % mount_volfile
-
-        cmdline = string.join (sys.argv, ' ')
-
-        mount_fd.write ("## file auto generated by %s\n" % sys.argv[0])
-        mount_fd.write ("# Cmd line:\n")
-        mount_fd.write ("# $ %s\n\n" % cmdline)
-
-        if raid_type is not None:
-            # Used for later usage
-            mount_fd.write ("# RAID %d\n" % raid_type)
-
-        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
-        subvolumes = []
-        for host in self.host_dict.keys():
-            i = 1
-            for export in self.host_dict[host]:
-                mount_fd.write ("volume %s-%d\n" % (host,i))
-                mount_fd.write ("    type protocol/client\n")
-                mount_fd.write ("    option transport-type %s\n" %
-                                self.transport)
-                mount_fd.write ("    option remote-host %s\n" % host)
-                if self.transport == 'ib-verbs':
-                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
-                                    self.ib_devport)
-                if self.transport == 'tcp':
-                    mount_fd.write ("    option transport.socket.nodelay on\n")
-
-                mount_fd.write ("    option remote-subvolume %s\n" % export)
-                mount_fd.write ("end-volume\n\n")
-                subvolumes.append(str(host) + '-' + str(i))
-                i += 1
-
-        # Stripe section.. if given
-        if raid_type is 0:
-            max_stripe_idx = len (subvolumes) / self.num_stripe
-            stripe_idx = 0
-            index = 0
-            while index < max_stripe_idx:
-                mount_fd.write ("volume stripe-%d\n" % index)
-                mount_fd.write ("    type cluster/stripe\n")
-                mount_fd.write ("#    option block-size 128k\n")
-                mount_fd.write ("#    option use-xattr no\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[stripe_idx])
-                sub_idx = 1
-                while sub_idx < self.num_stripe:
-                    mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                stripe_idx += self.num_stripe
-                index +=1
-
-        # Replicate section
-        if raid_type is 1:
-            max_mirror_idx = len (subvolumes) / self.num_replica
-            mirror_idx = 0
-            index = 0
-            while index < max_mirror_idx:
-                mount_fd.write ("volume mirror-%d\n" % index)
-                mount_fd.write ("    type cluster/replicate\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[mirror_idx])
-                sub_idx = 1
-                while sub_idx < self.num_replica:
-                    mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                mirror_idx += self.num_replica
-                index += 1
-
-        # Distribute section
-        if raid_type is 0:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("stripe-%d" % flag)
-                flag += 1
-
-        if raid_type is 1:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("mirror-%d" % flag)
-                flag += 1
-
-        if len (subvolumes) > 1:
-            mount_fd.write ("volume distribute\n")
-            mount_fd.write ("    type cluster/distribute\n")
-            mount_fd.write ("#   option unhashed-sticky-bit yes"
-                           "  # Used for migrating data while adding new nodes\n")
-            mount_fd.write ("#   option min-free-disk 5%"
-                           "  # Minimum free disk available on the volume\n")
-            mount_fd.write ("    subvolumes %s\n" %
-                                 string.join (subvolumes,' '))
-            mount_fd.write ("end-volume\n\n")
-            subvolumes[0] = "distribute"
-
-
-        mount_fd.write ("volume nfsxlator\n")
-        mount_fd.write ("    type nfs/server\n")
-        mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
-        mount_fd.write ("#   option rpc-auth.auth-unix         off  #Enabled by default\n")
-        mount_fd.write ("#   option rpc-auth.auth-null         off  #Enabled by default\n")
-        mount_fd.write ("#   By default all addresses are rejected until allowed.\n")
-        mount_fd.write ("#   option rpc-auth.addr.reject       127.*\n")
-        mount_fd.write ("#   option rpc-auth.addr.allow        localhost\n")
-        mount_fd.write ("    option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters))
-        mount_fd.write ("#   By default insecure ports are not allowed.\n")
-        mount_fd.write ("#   option rpc-auth.ports.insecure    on\n")
-        mount_fd.write ("#   option rpc-auth.ports.<volume>.insecure  on\n")
-        mount_fd.write ("#   By default all access is read-write.\n")
-        mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
-        mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
-        mount_fd.write ("#   option nfs3.read-size             128Kb\n")
-        mount_fd.write ("#   option nfs3.write-size            32Kb\n")
-        mount_fd.write ("#   option nfs3.readdir-size          64Kb\n")
-        mount_fd.write ("#   option nfs3.<volume>.read-size    64Kb\n")
-        mount_fd.write ("#   option nfs3.<volume>.write-size   64Kb\n")
-        mount_fd.write ("#   option nfs3.posix1.readdir-size   128Kb\n")
-        mount_fd.write ("end-volume\n\n")
-
-        return
-
-    def create_pmap_export_volfile (self):
-
-        cmdline = string.join (sys.argv, ' ')
-        i = 0
-        for export in self.host_dict[self.host]:
-            export_volfile = "%s/%s-%d.vol" % (self.conf_dir,
-                                               str(self.volume_name + '.' + self.host),
-                                               i)
-            i += 1
-            exp_fd = file ("%s" % (export_volfile),"w")
-
-            print "Generating server volfiles.. for server %s as '%s'" % (self.host,
-                                                                          export_volfile)
-
-            exp_fd.write ("## file auto generated by %s\n" % sys.argv[0])
-            exp_fd.write ("# Cmd line:\n")
-            exp_fd.write ("# $ %s\n\n" % cmdline)
-            exp_fd.write ("volume posix\n")
-            exp_fd.write ("    type storage/posix\n")
-            exp_fd.write ("#   option o-direct enable # (default: disable) boolean type only\n")
-            exp_fd.write ("#   option export-statfs-size no # (default: yes) boolean type only\n")
-            exp_fd.write ("#   option mandate-attribute off # (default: on) boolean type only\n")
-            exp_fd.write ("#   option span-devices 8 # (default: 0) integer value\n")
-            exp_fd.write ("#   option background-unlink yes # (default: no) boolean type\n")
-
-            exp_fd.write ("    option directory %s\n" % export)
-            exp_fd.write ("end-volume\n\n")
-
-            if self.nfs:
-                exp_fd.write ("volume posix-ac\n")
-                exp_fd.write ("    type features/access-control\n")
-                exp_fd.write ("    subvolumes posix\n")
-                exp_fd.write ("end-volume\n\n")
-
-            if self.volume_size_server:
-                exp_fd.write ("volume quota\n")
-                exp_fd.write ("    type features/quota\n")
-                exp_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_server)
-                exp_fd.write ("#   option minimum-free-disk-limit 10GB"
-                              "  # minimum free disk value (default) 0\n")
-                exp_fd.write ("#   option refresh-interval 10\n")
-                if self.nfs:
-                    exp_fd.write ("    subvolumes posix-ac\n")
-                else:
-                    exp_fd.write ("    subvolumes posix\n")
-                exp_fd.write ("end-volume\n\n")
-
-            exp_fd.write ("volume locks\n")
-            exp_fd.write ("    type features/locks\n")
-            exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
-            if self.volume_size_server:
-                exp_fd.write ("    subvolumes quota\n")
-            elif self.nfs:
-                exp_fd.write ("    subvolumes posix-ac\n")
-            else:
-                exp_fd.write ("    subvolumes posix\n")
-            exp_fd.write ("end-volume\n\n")
-
-            exp_fd.write ("volume replace-brick\n")
-            exp_fd.write ("    type protocol/client\n")
-            if self.transport:
-                exp_fd.write ("    option transport-type %s\n" % self.transport)
-            if self.gfs_port:
-                exp_fd.write ("    option transport.remote-port 34034\n")
-            exp_fd.write ("    option ping-timeout 42\n")
-            exp_fd.write ("end-volume\n\n")
-
-            exp_fd.write ("volume pump\n")
-            exp_fd.write ("    type cluster/pump\n")
-            exp_fd.write ("    subvolumes locks replace-brick\n")
-            exp_fd.write ("end-volume\n\n")
-
-            exp_fd.write ("volume %s\n" % export)
-            exp_fd.write ("    type performance/io-threads\n")
-            exp_fd.write ("    option thread-count 8\n")
-            exp_fd.write ("#   option autoscaling yes # Heuristic for autoscaling threads on demand\n")
-            exp_fd.write ("#   option min-threads 2 # min count for thread pool\n")
-            exp_fd.write ("#   option max-threads 64 # max count for thread pool\n")
-            exp_fd.write ("    subvolumes pump\n")
-            exp_fd.write ("end-volume\n\n")
-
-            for transport in self.transports:
-                exp_fd.write ("volume server-%s\n" % export)
-                exp_fd.write ("    type protocol/server\n")
-                exp_fd.write ("    option transport-type %s\n" % transport)
-                exp_fd.write ("    option auth.addr.%s.allow %s\n" %
-                                  (export, self.auth_parameters))
-                if transport == 'ib-verbs':
-                    exp_fd.write ("    option transport.ib-verbs.port %d\n" %
-                                  self.ib_devport)
-                if transport == 'tcp':
-                    exp_fd.write ("    option transport.socket.nodelay on\n")
-                    exp_fd.write ("    option transport.socket.listen-port %d\n" % self.gfs_port)
-                exp_fd.write ("    subvolumes %s\n" % export)
-                exp_fd.write ("end-volume\n\n")
-
-        return
-
-
-    def create_mount_volfile (self):
-
-        raid_type = self.raid_type
-
-        mount_volfile = "%s/%s-%s.vol" % (self.conf_dir, str(self.volume_name), str(self.transport))
-        mount_fd = file ("%s" % (mount_volfile), "w")
-
-        print "Generating client volfiles.. '%s'" % mount_volfile
-
-        cmdline = string.join (sys.argv, ' ')
-
-        mount_fd.write ("## file auto generated by %s\n" % sys.argv[0])
-        mount_fd.write ("# Cmd line:\n")
-        mount_fd.write ("# $ %s\n\n" % cmdline)
-
-        if raid_type is not None:
-            # Used for later usage
-            mount_fd.write ("# RAID %d\n" % raid_type)
-
-        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
-        subvolumes = []
-        for host in self.host_dict.keys():
-            i = 1
-            for exports in self.host_dict[host]:
-                mount_fd.write ("volume %s-%s\n" % (host,exports))
-                mount_fd.write ("    type protocol/client\n")
-                mount_fd.write ("    option transport-type %s\n" %
-                                self.transport)
-                mount_fd.write ("    option remote-host %s\n" % host)
-                if self.transport == 'ib-verbs':
-                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
-                                    self.ib_devport)
-                    mount_fd.write ("    option remote-port %d\n" %
-                                    self.gfs_ib_port)
-                if self.transport == 'tcp':
-                    mount_fd.write ("    option transport.socket.nodelay on\n")
-                    mount_fd.write ("    option remote-port %d\n" %
-                                    self.gfs_port)
-
-                mount_fd.write ("    option remote-subvolume %s\n" % exports)
-                mount_fd.write ("end-volume\n\n")
-                subvolumes.append(str(host) + '-' + str(exports))
-
-        # Stripe section.. if given
-        if raid_type is 0:
-            max_stripe_idx = len (subvolumes) / self.num_stripe
-            stripe_idx = 0
-            index = 0
-            while index < max_stripe_idx:
-                mount_fd.write ("volume stripe-%d\n" % index)
-                mount_fd.write ("    type cluster/stripe\n")
-                mount_fd.write ("#    option block-size 128k\n")
-                mount_fd.write ("#    option use-xattr no\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[stripe_idx])
-                sub_idx = 1
-                while sub_idx < self.num_stripe:
-                    mount_fd.write (" %s" % subvolumes[stripe_idx+sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                stripe_idx += self.num_stripe
-                index +=1
-
-        # Replicate section
-        if raid_type is 1:
-            max_mirror_idx = len (subvolumes) / self.num_replica
-            mirror_idx = 0
-            index = 0
-            while index < max_mirror_idx:
-                mount_fd.write ("volume mirror-%d\n" % index)
-                mount_fd.write ("    type cluster/replicate\n")
-                mount_fd.write ("    subvolumes %s" % subvolumes[mirror_idx])
-                sub_idx = 1
-                while sub_idx < self.num_replica:
-                    mount_fd.write (" %s" % subvolumes[mirror_idx + sub_idx])
-                    sub_idx += 1
-                mount_fd.write ("\nend-volume\n\n")
-                mirror_idx += self.num_replica
-                index += 1
-
-        # Distribute section
-        if raid_type is 0:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("stripe-%d" % flag)
-                flag += 1
-
-        if raid_type is 1:
-            subvolumes = []
-            flag = 0
-            while flag < index:
-                subvolumes.append ("mirror-%d" % flag)
-                flag += 1
-
-        if len (subvolumes) > 1:
-            mount_fd.write ("volume distribute\n")
-            mount_fd.write ("    type cluster/distribute\n")
-            mount_fd.write ("#   option unhashed-sticky-bit yes"
-                           "  # Used for migrating data while adding new nodes\n")
-            mount_fd.write ("#   option min-free-disk 5%"
-                           "  # Minimum free disk available on the volume\n")
-            mount_fd.write ("    subvolumes %s\n" %
-                                 string.join (subvolumes,' '))
-            mount_fd.write ("end-volume\n\n")
-            subvolumes[0] = "distribute"
-
-
-        if self.nfs:
-            mount_fd.write ("volume nfsxlator\n")
-            mount_fd.write ("    type nfs/server\n")
-            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
-            mount_fd.write ("#   option rpc-auth.auth-unix         off  #Enabled by default\n")
-            mount_fd.write ("#   option rpc-auth.auth-null         off  #Enabled by default\n")
-            mount_fd.write ("#   By default all addresses are rejected until allowed.\n")
-            mount_fd.write ("#   option rpc-auth.addr.reject       127.*\n")
-            mount_fd.write ("#   option rpc-auth.addr.allow        localhost\n")
-            mount_fd.write ("    option rpc-auth.addr.%s.allow %s\n" % (subvolumes[0], self.auth_parameters))
-            mount_fd.write ("#   By default insecure ports are not allowed.\n")
-            mount_fd.write ("#   option rpc-auth.ports.insecure    on\n")
-            mount_fd.write ("#   option rpc-auth.ports.<volume>.insecure  on\n")
-            mount_fd.write ("#   By default all access is read-write.\n")
-            mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
-            mount_fd.write ("#   option nfs3.<volume>.volume-access       read-only\n")
-            mount_fd.write ("#   option nfs3.read-size             128Kb\n")
-            mount_fd.write ("#   option nfs3.write-size            32Kb\n")
-            mount_fd.write ("#   option nfs3.readdir-size          64Kb\n")
-            mount_fd.write ("#   option nfs3.<volume>.read-size    64Kb\n")
-            mount_fd.write ("#   option nfs3.<volume>.write-size   64Kb\n")
-            mount_fd.write ("#   option nfs3.posix1.readdir-size   128Kb\n")
-            mount_fd.write ("end-volume\n\n")
-            return
-
-        if self.volume_size_client:
-            mount_fd.write ("volume quota\n")
-            mount_fd.write ("    type features/quota\n")
-            mount_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_client)
-            mount_fd.write ("#   option minimum-free-disk-limit 10GB\n"
-                            "#   minimum free disk value (default) 0\n")
-            mount_fd.write ("#   option refresh-interval 10\n")
-            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
-            mount_fd.write ("end-volume\n\n")
-            subvolumes[0] = "quota"
-
-        if self.enable_safe_mode:
-            return
-
-        self.performance_mode (subvolumes[0], mount_fd)
-
-        return
-
-    def create_export_volfile (self):
-
-        cmdline = string.join (sys.argv, ' ')
-        export_volfile = "%s/%s-export.vol" % (self.conf_dir, str(self.host + '-' + self.volume_name))
-        exp_fd = file ("%s" % (export_volfile),"w")
-
-        print "Generating server volfiles.. for server %s as '%s'" % (self.host,
-                                                                      export_volfile)
-
-        exp_fd.write ("## file auto generated by %s\n" %
-                      sys.argv[0])
-        exp_fd.write ("# Cmd line:\n")
-        exp_fd.write ("# $ %s\n\n" % cmdline)
-        total_bricks = []
-        i=1
-        for export in self.host_dict[self.host]:
-            exp_fd.write ("volume posix%d\n" % i)
-            exp_fd.write ("    type storage/posix\n")
-            exp_fd.write ("#   option o-direct enable # (default: disable) boolean type only\n")
-            exp_fd.write ("#   option export-statfs-size no # (default: yes) boolean type only\n")
-            exp_fd.write ("#   option mandate-attribute off # (default: on) boolean type only\n")
-            exp_fd.write ("#   option span-devices 8 # (default: 0) integer value\n")
-            exp_fd.write ("#   option background-unlink yes # (default: no) boolean type\n")
-
-            exp_fd.write ("    option directory %s\n" % export)
-            exp_fd.write ("end-volume\n\n")
-
-            if self.nfs:
-                exp_fd.write ("volume posix-ac%d\n" % i)
-                exp_fd.write ("    type features/access-control\n")
-                exp_fd.write ("    subvolumes posix%d\n" % i)
-                exp_fd.write ("end-volume\n\n")
-
-            if self.volume_size_server:
-                exp_fd.write ("volume quota%d\n" % i)
-                exp_fd.write ("    type features/quota\n")
-                exp_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_server)
-                exp_fd.write ("#   option minimum-free-disk-limit 10GB"
-                              "  # minimum free disk value (default) 0\n")
-                exp_fd.write ("#   option refresh-interval 10\n")
-                if self.nfs:
-                    exp_fd.write ("    subvolumes posix-ac%d\n" % i)
-                else:
-                    exp_fd.write ("    subvolumes posix%d\n" % i)
-                exp_fd.write ("end-volume\n\n")
-
-
-            exp_fd.write ("volume locks%d\n" % i)
-            exp_fd.write ("    type features/locks\n")
-            exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
-            if self.volume_size_server:
-                exp_fd.write ("    subvolumes quota%d\n" % i)
-            else:
-                exp_fd.write ("    subvolumes posix%d\n" % i)
-            exp_fd.write ("end-volume\n\n")
-
-            exp_fd.write ("volume %s\n" % export)
-            exp_fd.write ("    type performance/io-threads\n")
-            exp_fd.write ("    option thread-count 8\n")
-            exp_fd.write ("#   option autoscaling yes # Heuristic for autoscaling threads on demand\n")
-            exp_fd.write ("#   option min-threads 2 # min count for thread pool\n")
-            exp_fd.write ("#   option max-threads 64 # max count for thread pool\n")
-
-            exp_fd.write ("    subvolumes locks%d\n" % i)
-            exp_fd.write ("end-volume\n\n")
-
-            total_bricks.append("%s" % export)
-            i += 1
-
-        for transport in self.transports:
-            exp_fd.write ("volume server-%s\n" % transport)
-            exp_fd.write ("    type protocol/server\n")
-            exp_fd.write ("    option transport-type %s\n" % transport)
-            for brick in total_bricks:
-                exp_fd.write ("    option auth.addr.%s.allow %s\n" %
-                              (brick, self.auth_parameters))
-
-            if transport == 'ib-verbs':
-                exp_fd.write ("    option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port)
-                exp_fd.write ("    option transport.ib-verbs.port %d\n" %
-                              self.ib_devport)
-            if transport == 'tcp':
-                exp_fd.write ("    option transport.socket.listen-port %d\n" % self.gfs_port)
-                exp_fd.write ("    option transport.socket.nodelay on\n")
-
-            exp_fd.write ("    subvolumes %s\n" %
-                          string.join(total_bricks, ' '))
-            exp_fd.write ("end-volume\n\n")
-
-        return
-
diff --git a/extras/volgen/Makefile.am b/extras/volgen/Makefile.am
deleted file mode 100644
index e36bd159b38..00000000000
--- a/extras/volgen/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-
-volgendir = $(datadir)/glusterfs
-dist_volgen_DATA = CreateVolfile.py Common.py
-
-dist_bin_SCRIPTS = glusterfs-volgen
-
-CLEANFILES = 
-
diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in
deleted file mode 100755
index 4b3742366c5..00000000000
--- a/extras/volgen/glusterfs-volgen.in
+++ /dev/null
@@ -1,227 +0,0 @@
-#!/usr/bin/python
-
-#    Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com>
-#    This file is part of GlusterFS.
-
-#    GlusterFS is free software; you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published
-#    by the Free Software Foundation; either version 3 of the License,
-#    or (at your option) any later version.
-
-#    GlusterFS is distributed in the hope that it will be useful, but
-#    WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#    General Public License for more details.
-
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see
-#    <http://www.gnu.org/licenses/>.
-
-
-import getopt, sys, os, string
-from optparse import OptionParser,OptionGroup,make_option
-
-prefix = "@prefix@"
-
-if not (prefix + "/share/glusterfs") in sys.path:
-    sys.path.append(prefix + "/share/glusterfs")
-
-from CreateVolfile import *
-from Common import *
-
-def generate_volume_files ():
-
-    usage_str = "%s%s" % ("%prog: -n <VOLUMENAME> -t <TRANSPORT> -p <NUMBER> -a <AUTH> ",
-                          "-r <TYPE> [--num-replica N] [--num-stripe N]")
-    version_str = "%prog 3.0"
-    desc_str = "A tool to generate volume files for GlusterFS."
-
-    parse = OptionParser(usage=usage_str, version=version_str, description=desc_str)
-
-    # Basic option list
-    group = OptionGroup(parse, "Basic Options")
-    group.add_option("-n", "--name", dest="volume_name",
-                help="<volume-name>")
-    group.add_option("-t", "--transport", dest="transport_type",
-                default="tcp", help="tcp,ib-verbs default: tcp")
-    group.add_option("-p", "--port", type="int",
-                dest="port", help="<port> number")
-    group.add_option("--auth", dest="auth_param", default="*",
-                help="comma seperated ip range")
-    group.add_option("-r", "--raid", type="int", dest="raid_type",
-                help="0|1")
-    group.add_option("-c", "--conf-dir", dest="conf_dir",
-                     default=os.getcwd(),
-                     help="output directory for volume files")
-    group.add_option("--nfs", action="store_true", dest="need_nfs",
-                default=False, help="enable nfs translator")
-    parse.add_option_group(group)
-
-    # Advanced option list
-    group = OptionGroup(parse, "Advanced Options")
-    group.add_option("--ibdev", type="int", dest="ib_dev",
-                     default=1, help="Infiniband device number <N>")
-    group.add_option("--volume-size-server", dest="size_server",
-                     help="volume size for each server")
-    group.add_option("--volume-size-client", dest="size_client",
-                     help="volume size for each client")
-    group.add_option("--safe-mode", action="store_true",
-                     dest="enable_safe_mode", default=False,
-                     help="generate volume files in safe mode")
-    group.add_option("--num-replica", type="int", dest="num_replica",
-                     help="number of file replica needed")
-    group.add_option("--num-stripe", type="int", dest="num_stripe",
-                     help="number of stripe count needed")
-    group.add_option("--portmapper-mode", action="store_true", dest="portmapper",
-                     default=False, help="generate volume files for glusterfs portmapper friendly way")
-
-    parse.add_option_group(group)
-
-    # Changes for Dynamic Volume Manager
-    #group = OptionGroup(parse, "Dynamic Volume Options")
-    #group.add_option("--add-server", dest="add_server_args",
-    #                 help="Add server to existing volume")
-    #group.add_option("--migrate", dest="migrate_server_args",
-    #                 help="Migrate servers from existing volumes")
-
-    #parse.add_option_group(group)
-
-    (options, args) = parse.parse_args()
-
-    if options.volume_name is None:
-        print "Error: volume name is mandatory, please provide volume name"
-        raise ValueError
-
-    if options.need_nfs and options.size_client:
-        print "Error: Currently quota not supported with native NFS, please use server side instead"
-        raise ValueError
-
-    if options.transport_type:
-        transports = options.transport_type.split(',')
-        for transport in transports:
-            if (transport != "tcp" and transport != "ib-verbs"):
-                print "Error: --transport: option " + transport + \
-                    " is not valid transport type"
-                raise ValueError
-
-    if options.raid_type:
-        if (options.raid_type != 1 and options.raid_type != 0):
-            print "Error: --raid: option " + str(options.raid_type) + " is not valid raid type"
-            raise ValueError
-
-    if options.conf_dir:
-        if not os.path.isdir(options.conf_dir):
-            print "Specified directory %s doesn't exist" % options.conf_dir
-            raise ValueError
-
-    server_dict = {}
-
-    # Dynamic Volume Manager
-    #if options.add_server_args:
-    #    add_server_list = (options.add_server_args).strip().split()
-    #    check_duplicate_entry(add_server_list)
-    #    old_server_list = get_old_server_args(list_export_vols(options.conf_dir,
-    #                                                           options.volume_name),
-    #                                          options.conf_dir)
-    #    for new_server in add_server_list:
-    #        old_server_list.append(new_server)
-
-    #    server_dict = args2dict(old_server_list)
-    #    server_array = args2array(old_server_list)
-
-    #    if len (server_dict.keys()) == 0:
-    #        print "Error: no servers provided, please provide atleast one server"
-    #        raise ValueError
-
-    #else:
-    check_duplicate_entry(args)
-    server_dict = args2dict(args)
-    server_array = args2array(args)
-
-    if len (server_dict.keys()) == 0:
-        print "Error: no servers provided, please provide atleast one server"
-        raise ValueError
-
-    if options.num_replica is None:
-        options.num_replica = 2
-
-    if options.num_stripe is None:
-        options.num_stripe = 4
-
-    if options.num_replica < 2:
-        print "--num-replica option (%d) is not valid" % options.num_replica
-
-    if options.num_stripe < 2:
-        print "--num-stripe option (%d) is not valid" % options.num_stripe
-
-    if options.raid_type == 1:
-        if (len(server_array) % options.num_replica) != 0:
-            print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type,
-                                                                         len(server_array))
-            raise ValueError
-
-    if options.raid_type == 0:
-        if (len(server_array) % options.num_stripe) != 0:
-            print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type,
-                                                                         len(server_array))
-            raise ValueError
-
-    if options.portmapper is True:
-        for server in server_dict.keys():
-            create_exp = CreateVolfile (server_dict, server,
-                                        None, transports,
-                                        options, None)
-            try:
-                create_exp.create_pmap_export_volfile ()
-            except IOError, (errno, strerror):
-                print "Got %s creating server volfiles for %s" % (strerror, server)
-
-        for transport in transports:
-            create_mnt = CreateVolfile (server_dict, None,
-                                        transport, transports,
-                                        options, server_array)
-            try:
-                create_mnt.create_pmap_mount_volfile ()
-            except IOError, (errno, strerror):
-                print "Got %s creating client volfiles for transport '%s'" % (strerror, transport)
-
-        if options.need_nfs:
-            for transport in transports:
-                create_mnt = CreateVolfile (server_dict, None,
-                                            transport, transports,
-                                            options, server_array)
-                try:
-                    create_mnt.create_pmap_nfs_volfile ()
-                except IOError, (errno, strerror):
-                    print "Got %s creating nfs volfiles for transport '%s'" % (strerror, transport)
-
-        return
-
-    for server in server_dict.keys():
-        create_exp = CreateVolfile (server_dict, server,
-                                    None, transports,
-                                    options, None)
-        try:
-            create_exp.create_export_volfile ()
-        except IOError, (errno, strerror):
-            print "Got %s creating server volfiles for %s" % (strerror, server)
-
-    for transport in transports:
-        create_mnt = CreateVolfile (server_dict, None,
-                                    transport, transports,
-                                    options, server_array)
-        try:
-            create_mnt.create_mount_volfile ()
-        except IOError, (errno, strerror):
-            print "Got %s creating client volfiles for transport '%s'" % (strerror, transport)
-
-
-def main ():
-
-    try:
-        generate_volume_files()
-    except ValueError:
-        sys.exit(1)
-
-
-main()
diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases
new file mode 100644
index 00000000000..901c12418e3
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.aliases
@@ -0,0 +1,58 @@
+#
+# This is the email aliases file, mapping secondary addresses onto a single,
+# canonical address. This file should probably match the contents of .mailmap
+# in the root of the git repository.
+#
+# Format: <alias> <real>
+
+amar@gluster.com	amarts@redhat.com
+amar@del.gluster.com	amarts@redhat.com
+avati@amp.gluster.com	avati@redhat.com
+avati@blackhole.gluster.com	avati@redhat.com
+avati@dev.gluster.com	avati@redhat.com
+avati@gluster.com	avati@redhat.com
+wheelear@gmail.com      awheeler@redhat.com
+anush@gluster.com	ashetty@redhat.com
+csaba@gluster.com	csaba@redhat.com
+csaba@lowlife.hu	csaba@redhat.com
+csaba@zresearch.com	csaba@redhat.com
+gd@samba.org            gd@redhat.com
+harsha@gluster.com	fharshav@redhat.com
+harsha@zresearch.com	fharshav@redhat.com
+harsha@dev.gluster.com	fharshav@redhat.com
+harsha@harshavardhana.net	fharshav@redhat.com
+jclift@redhat.com       jclift@gluster.org
+kkeithle@linux.keithley.org	kkeithle@redhat.com
+kkeithle@f16node1.kkeithle.usersys.redhat.com	kkeithle@redhat.com
+kaushal@gluster.com	kaushal@redhat.com
+kaushikbv@gluster.com	kbudiger@redhat.com
+krishna@gluster.com	ksriniva@redhat.com
+krishna@zresearch.com	ksriniva@redhat.com
+krishna@guest-laptop	ksriniva@redhat.com
+kp@gluster.com	kparthas@redhat.com
+me@louiszuckerman.com	louiszuckerman@gmail.com
+msvbhat@gmail.com	vbhat@redhat.com
+nullpai@gmail.com       ppai@redhat.com
+vishwanath@gluster.com	vbhat@redhat.com
+obnox@samba.org         madam@redhat.com
+oleksandr@natalenko.name        o.natalenko@lanet.ua
+patrick@puiterwijk.org  puiterwijk@fedoraproject.org
+pavan@dev.gluster.com	pavan@gluster.com
+zaitcev@yahoo.com	zaitcev@kotori.zaitcev.us
+pranithk@gluster.com	pkarampu@redhat.com
+raghavendrabhat@gluster.com	raghavendra@redhat.com
+raghavendra@gluster.com	rgowdapp@redhat.com
+raghavendra@zresearch.com	rgowdapp@redhat.com
+rahulcssjce@gmail.com	rahulcs@redhat.com
+rajesh@gluster.com	rajesh@redhat.com
+rajesh.amaravathi@gmail.com	rajesh@redhat.com
+root@ravi2.(none)       ravishankar@redhat.com
+sabansal@localhost.localdomain  sabansal@redhat.com
+shehjart@zresearch.com	shehjart@gluster.com
+venky@gluster.com	vshankar@redhat.com
+vijay@gluster.com	vbellur@redhat.com
+vijay@dev.gluster.com	vbellur@redhat.com
+vijaykumar.koppad@gmail.com	vkoppad@redhat.com
+vikas@zresearch.com	vikas@gluster.com
+shishirng@gluster.com	sgowda@redhat.com
+potatogim@potatogim.net	potatogim@gluesys.com
diff --git a/extras/who-wrote-glusterfs/gitdm.config b/extras/who-wrote-glusterfs/gitdm.config
new file mode 100644
index 00000000000..e1ff2bd5bda
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.config
@@ -0,0 +1,8 @@
+#
+# This is the gitdm configuration file for GlusterFS.
+# See the gitdm.config in the gitdm repositofy for additional options and
+# comments.
+#
+
+EmailAliases gitdm.aliases
+EmailMap gitdm.domain-map
diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map
new file mode 100644
index 00000000000..7cd2bbd605b
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.domain-map
@@ -0,0 +1,29 @@
+#
+# Here is a set of mappings of domain names onto employer names.
+#
+active.by       ActiveCloud
+appeartv.com    Appear TV
+cern.ch         CERN
+cmss.chinamobile.com    China Mobile(Suzhou) Software Technology
+datalab.es      DataLab S.L.
+fb.com          Facebook
+fedoraproject.org       Fedora Project
+gluster.com     Red Hat
+gmail.com       (personal contributions)
+gooddata.com	GoodData
+hastexo.com     hastexo
+horde.com       (personal contributions)
+ibm.com		IBM
+io.com          IO
+lanet.ua        Lanet Network
+linbit.com	LINBIT
+nectec.or.th    NECTEC
+netbsd.org	NetBSD
+netdirect.ca	Net Direct
+nokia.com       Nokia
+redhat.com	Red Hat
+stepping-stone.ch       stepping stone GmbH
+xtaotech.com    XTAO Co.
+yahoo.in        (personal contributions)
+zresearch.com   Red Hat
+gluesys.com     Gluesys
diff --git a/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh
new file mode 100755
index 00000000000..70d5964c576
--- /dev/null
+++ b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Gather statistics on "Who wrote GlusterFS". The idea comes from the excellent
+# articles on http://lwn.net/ named "Who wrote <linux-version>?".
+#
+# gitdm comes from git://git.lwn.net/gitdm.git by Jonathan Corbet.
+#
+# Confguration files used:
+#  - gitdm.config: main configuration file, pointing to the others
+#  - gitdm.aliases: merge users with different emailaddresses into one
+#  - gitdm.domain-map: map domain names from emailaddresses to companies
+#
+
+DIRNAME=$(dirname $0)
+
+GITDM_REPO=git://git.lwn.net/gitdm.git
+GITDM_DIR=${DIRNAME}/gitdm
+GITDM_CMD="python ${GITDM_DIR}/gitdm"
+
+error()
+{
+        local ret=${?}
+        printf "${@}\n" > /dev/stderr
+        return ${ret}
+}
+
+check_gitdm()
+{
+        if [ ! -e "${GITDM_DIR}/gitdm" ]
+        then
+                git clone --quiet ${GITDM_REPO} ${DIRNAME}/gitdm
+        fi
+}
+
+# The first argument is the revision-range (see 'git rev-list --help').
+# REV can be empty, and the statistics will be calculated over the whole
+# current branch.
+REV=${1}
+shift
+# all remaining options are passed to gitdm, see the gitdm script for an
+# explanation of the accepted options.
+GITDM_OPTS=${@}
+
+if ! check_gitdm
+then
+        error "Could not find 'gitdm', exiting..."
+        exit 1
+fi
+
+git log --numstat -M ${REV} | ${GITDM_CMD} -b ${DIRNAME} -n ${GITDM_OPTS}
diff --git a/format-patch.sh b/format-patch.sh
deleted file mode 100755
index 12d05a03d57..00000000000
--- a/format-patch.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-
-function is_num()
-{
-    local num;
-
-    num="$1";
-
-    [ -z "$(echo $num | sed -e 's/[0-9]//g')" ]
-}
-
-
-function guess_branch()
-{
-    local branch;
-    local src_branch;
-
-    branch=$(git branch | grep '*' | cut -f2 -d' ');
-
-    if [ $branch = "master" ] ; then
-        src_branch="master";
-    else
-        src_branch=$(cat .git/logs/refs/heads/$branch | head -n 1 \
-            | sed -r -e 's/.*( [^ ]*)$/\1/g' | cut -f2 -d/);
-    fi
-
-    echo $src_branch
-}
-
-
-function main()
-{
-    local branch;
-    local bug;
-
-    branch=$(guess_branch);
-    echo
-    echo "Patches are always to be associated with a bug ID. If there is no   "
-    echo "bug filed in bugzilla for this patch, it is highly suggested to file"
-    echo "a new bug with a description and reasoning of this patchset. If this"
-    echo "is a new feature, then file a new enhancement bug with a brief      "
-    echo "summary of the feature as the description."
-    echo
-    echo -n "Enter bug ID (from http://bugs.gluster.com/): "
-    read bug;
-
-    [ -z "$bug" ] || is_num $bug || {
-        log "bug ID should be a valid bug number";
-        exit;
-    }
-
-    if [ -z "$bug" ]; then
-        git format-patch -s "$@";
-    else
-        git format-patch -s --subject-prefix="PATCH BUG:$bug" "$@";
-    fi
-}
-
-main "$@"
diff --git a/geo-replication/Makefile.am b/geo-replication/Makefile.am
new file mode 100644
index 00000000000..591b23d0eaf
--- /dev/null
+++ b/geo-replication/Makefile.am
@@ -0,0 +1,8 @@
+SUBDIRS = syncdaemon src
+
+CLEANFILES =
+
+EXTRA_DIST = gsyncd.conf.in
+
+gsyncdconfdir = $(sysconfdir)/glusterfs/
+gsyncdconf_DATA = gsyncd.conf
diff --git a/geo-replication/gsyncd.conf.in b/geo-replication/gsyncd.conf.in
new file mode 100644
index 00000000000..9688c79fab7
--- /dev/null
+++ b/geo-replication/gsyncd.conf.in
@@ -0,0 +1,349 @@
+[__meta__]
+version = 4.0
+
+[master-bricks]
+configurable=false
+
+[slave-bricks]
+configurable=false
+
+[master-volume-id]
+configurable=false
+
+[slave-volume-id]
+configurable=false
+
+[master-replica-count]
+configurable=false
+type=int
+value=1
+
+[master-disperse-count]
+configurable=false
+type=int
+value=1
+
+[master-distribution-count]
+configurable=false
+type=int
+value=1
+
+[glusterd-workdir]
+value = @GLUSTERD_WORKDIR@
+
+[gluster-logdir]
+value = /var/log/glusterfs
+
+[gluster-rundir]
+value = /var/run/gluster
+
+[gsyncd-miscdir]
+value = /var/lib/misc/gluster/gsyncd
+
+[stime-xattr-prefix]
+value=
+
+[checkpoint]
+value=0
+help=Set Checkpoint
+validation=unixtime
+type=int
+
+[gluster-cli-options]
+value=
+help=Gluster CLI Options
+
+[pid-file]
+value=${gluster_rundir}/gsyncd-${master}-${primary_slave_host}-${slavevol}.pid
+configurable=false
+template = true
+help=PID file path
+
+[state-file]
+value=${glusterd_workdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/monitor.status
+configurable=false
+template=true
+help=Status File path
+
+[georep-session-working-dir]
+value=${glusterd_workdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/
+template=true
+help=Session Working directory
+configurable=false
+
+[access-mount]
+value=false
+type=bool
+validation=bool
+help=Do not lazy unmount the master volume. This allows admin to access the mount for debugging.
+
+[slave-access-mount]
+value=false
+type=bool
+validation=bool
+help=Do not lazy unmount the slave volume. This allows admin to access the mount for debugging.
+
+[isolated-slaves]
+value=
+help=List of Slave nodes which are isolated
+
+[changelog-batch-size]
+# Max size of Changelogs to process per batch, Changelogs Processing is
+# not limited by the number of changelogs but instead based on
+# size of the changelog file, One sample changelog file size was 145408
+# with ~1000 CREATE and ~1000 DATA. 5 such files in one batch is 727040
+# If geo-rep worker crashes while processing a batch, it has to retry only
+# that batch since stime will get updated after each batch.
+value=727040
+help=Max size of Changelogs to process per batch.
+type=int
+
+[slave-timeout]
+value=120
+type=int
+help=Timeout in seconds for Slave Gsyncd. If no activity from master for this timeout, Slave gsyncd will be disconnected. Set Timeout to zero to skip this check.
+
+[connection-timeout]
+value=60
+type=int
+help=Timeout for mounts
+
+[replica-failover-interval]
+value=1
+type=int
+help=Minimum time interval in seconds for passive worker to become Active
+
+[changelog-archive-format]
+value=%Y%m
+help=Processed changelogs will be archived in working directory. Pattern for archive file
+
+[use-meta-volume]
+value=false
+type=bool
+help=Use this to set Active Passive mode to meta-volume.
+
+[meta-volume-mnt]
+value=/run/gluster/shared_storage
+help=Meta Volume or Shared Volume mount path
+
+[allow-network]
+value=
+
+[change-interval]
+value=5
+type=int
+
+[sync-method]
+value=rsync
+help=Sync method for data sync. Available methods are tar over ssh and rsync. Default is rsync.
+validation=choice
+allowed_values=tarssh,rsync
+
+[remote-gsyncd]
+value =
+help=If SSH keys are not secured with gsyncd prefix then use this configuration to set the actual path of gsyncd(Usually /usr/libexec/glusterfs/gsyncd)
+
+[gluster-command-dir]
+value=@SBIN_DIR@
+help=Directory where Gluster binaries exist on master
+
+[slave-gluster-command-dir]
+value=@SBIN_DIR@
+help=Directory where Gluster binaries exist on slave
+
+[gluster-params]
+value = aux-gfid-mount acl
+help=Parameters for Gluster Geo-rep mount in Master
+
+[slave-gluster-params]
+value = aux-gfid-mount acl
+help=Parameters for Gluster Geo-rep mount in Slave
+
+[ignore-deletes]
+value = false
+type=bool
+help=Do not sync deletes in Slave
+
+[special-sync-mode]
+# tunables for failover/failback mechanism:
+# None   - gsyncd behaves as normal
+# blind  - gsyncd works with xtime pairs to identify
+#          candidates for synchronization
+# wrapup - same as normal mode but does not assign
+#          xtimes to orphaned files
+# see crawl() for usage of the above tunables
+value =
+help=
+
+[gfid-conflict-resolution]
+value = true
+validation=bool
+type=bool
+help=Disables automatic gfid conflict resolution while syncing
+
+[working-dir]
+value = ${gsyncd_miscdir}/${master}_${primary_slave_host}_${slavevol}/
+template=true
+configurable=false
+help=Working directory for storing Changelogs
+
+[change-detector]
+value=changelog
+help=Change detector
+validation=choice
+allowed_values=changelog,xsync
+
+[cli-log-file]
+value=${gluster_logdir}/geo-replication/cli.log
+template=true
+configurable=false
+
+[cli-log-level]
+value=INFO
+help=Set CLI Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/gsyncd.log
+configurable=false
+template=true
+
+[changelog-log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/changes-${local_id}.log
+configurable=false
+template=true
+
+[gluster-log-file]
+value=${gluster_logdir}/geo-replication/${master}_${primary_slave_host}_${slavevol}/mnt-${local_id}.log
+template=true
+configurable=false
+
+[slave-log-file]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/gsyncd.log
+template=true
+configurable=false
+
+[slave-gluster-log-file]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/mnt-${master_node}-${master_brick_id}.log
+template=true
+configurable=false
+
+[slave-gluster-log-file-mbr]
+value=${gluster_logdir}/geo-replication-slaves/${master}_${primary_slave_host}_${slavevol}/mnt-mbr-${master_node}-${master_brick_id}.log
+template=true
+configurable=false
+
+[log-level]
+value=INFO
+help=Set Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[gluster-log-level]
+value=INFO
+help=Set Gluster mount Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[changelog-log-level]
+value=INFO
+help=Set Changelog Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[slave-log-level]
+value=INFO
+help=Set Slave Gsyncd Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[slave-gluster-log-level]
+value=INFO
+help=Set Slave Gluster mount Log Level
+validation=choice
+allowed_values=ERROR,INFO,WARNING,DEBUG
+
+[ssh-port]
+value=22
+validation=minmax
+min=1
+max=65535
+help=Set SSH port
+type=int
+
+[ssh-command]
+value=ssh
+help=Set ssh binary path
+validation=execpath
+
+[tar-command]
+value=tar
+help=Set tar command path
+validation=execpath
+
+[ssh-options]
+value = -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${glusterd_workdir}/geo-replication/secret.pem
+template=true
+
+[ssh-options-tar]
+value = -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${glusterd_workdir}/geo-replication/tar_ssh.pem
+template=true
+
+[gluster-command]
+value=gluster
+help=Set gluster binary path
+validation=execpath
+
+[sync-jobs]
+value=3
+help=Number of Syncer jobs
+validation=minmax
+min=1
+max=100
+type=int
+
+[rsync-command]
+value=rsync
+help=Set rsync command path
+validation=execpath
+
+[rsync-options]
+value=
+
+[rsync-ssh-options]
+value=
+
+[rsync-opt-ignore-missing-args]
+value=true
+type=bool
+
+[rsync-opt-existing]
+value=true
+type=bool
+
+[log-rsync-performance]
+value=false
+help=Log Rsync performance
+validation=bool
+type=bool
+
+[use-rsync-xattrs]
+value=false
+type=bool
+
+[sync-xattrs]
+value=true
+type=bool
+
+[sync-acls]
+value=true
+type=bool
+
+[max-rsync-retries]
+value=10
+type=int
+
+[state_socket_unencoded]
+# Unused, For backward compatibility
+value=
diff --git a/geo-replication/setup.py b/geo-replication/setup.py
new file mode 100644
index 00000000000..0eae469d2d6
--- /dev/null
+++ b/geo-replication/setup.py
@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+#
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+"""
+This setup.py only used to run tests, since geo-replication will
+be installed in /usr/local/libexec/glusterfs or /usr/libexec/glusterfs
+"""
+from setuptools import setup
+
+name = 'syncdaemon'
+
+setup(
+    name=name,
+    version="",
+    description='GlusterFS Geo Replication',
+    license='GPLV2 and LGPLV3+',
+    author='Red Hat, Inc.',
+    author_email='gluster-devel@gluster.org',
+    url='http://www.gluster.org',
+    packages=[name, ],
+    test_suite='nose.collector',
+    install_requires=[],
+    scripts=[],
+    entry_points={},
+)
diff --git a/geo-replication/src/Makefile.am b/geo-replication/src/Makefile.am
new file mode 100644
index 00000000000..9937a0bd026
--- /dev/null
+++ b/geo-replication/src/Makefile.am
@@ -0,0 +1,48 @@
+gsyncddir = $(GLUSTERFS_LIBEXECDIR)
+
+gsyncd_SCRIPTS = gverify.sh peer_gsec_create \
+	set_geo_rep_pem_keys.sh peer_mountbroker peer_mountbroker.py \
+	peer_georep-sshkey.py
+
+# peer_gsec_create and peer_add_secret_pub are not added to
+# EXTRA_DIST as it's derived from a .in file
+EXTRA_DIST = gverify.sh set_geo_rep_pem_keys.sh peer_mountbroker.py.in \
+	peer_georep-sshkey.py.in
+
+gsyncd_PROGRAMS = gsyncd
+
+gsyncd_SOURCES = gsyncd.c procdiggy.c
+
+gsyncd_LDADD = $(GF_LDADD) $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+gsyncd_LDFLAGS = $(GF_LDFLAGS)
+
+noinst_HEADERS = procdiggy.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\" -DUSE_LIBGLUSTERFS \
+	-DSBIN_DIR=\"$(sbindir)\" -DPYTHON=\"$(PYTHON)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
+
+
+install-exec-hook:
+	$(mkdir_p) $(DESTDIR)$(sbindir)
+	rm -f $(DESTDIR)$(sbindir)/gluster-mountbroker
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_mountbroker.py \
+		$(DESTDIR)$(sbindir)/gluster-mountbroker
+
+	rm -f $(DESTDIR)$(sbindir)/gluster-georep-sshkey
+	ln -s $(GLUSTERFS_LIBEXECDIR)/peer_georep-sshkey.py \
+		$(DESTDIR)$(sbindir)/gluster-georep-sshkey
+
+
+uninstall-hook:
+	rm -f $(DESTDIR)$(sbindir)/gluster-mountbroker
+	rm -f $(DESTDIR)$(sbindir)/gluster-georep-sshkey
diff --git a/geo-replication/src/gsyncd.c b/geo-replication/src/gsyncd.c
new file mode 100644
index 00000000000..b5aeec5bf33
--- /dev/null
+++ b/geo-replication/src/gsyncd.c
@@ -0,0 +1,402 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <glusterfs/compat.h>
+#include <glusterfs/syscall.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/param.h> /* for PATH_MAX */
+
+/* NOTE (USE_LIBGLUSTERFS):
+ * ------------------------
+ * When USE_LIBGLUSTERFS debugging sumbol is passed; perform
+ * glusterfs translator like initialization so that glusterfs
+ * globals, contexts are valid when glustefs api's are invoked.
+ * We unconditionally pass then while building gsyncd binary.
+ */
+#ifdef USE_LIBGLUSTERFS
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/defaults.h>
+#endif
+
+#include <glusterfs/common-utils.h>
+#include <glusterfs/run.h>
+#include "procdiggy.h"
+
+#define _GLUSTERD_CALLED_ "_GLUSTERD_CALLED_"
+#define _GSYNCD_DISPATCHED_ "_GSYNCD_DISPATCHED_"
+#define GSYNCD_CONF_TEMPLATE "geo-replication/gsyncd_template.conf"
+#define GSYNCD_PY "gsyncd.py"
+#define RSYNC "rsync"
+
+int restricted = 0;
+
+static int
+duplexpand(void **buf, size_t tsiz, size_t *len)
+{
+    size_t osiz = tsiz * *len;
+    char *p = realloc(*buf, osiz << 1);
+    if (!p) {
+        return -1;
+    }
+
+    memset(p + osiz, 0, osiz);
+    *buf = p;
+    *len <<= 1;
+
+    return 0;
+}
+
+static int
+str2argv(char *str, char ***argv)
+{
+    char *p = NULL;
+    char *savetok = NULL;
+    char *temp = NULL;
+    char *temp1 = NULL;
+    int argc = 0;
+    size_t argv_len = 32;
+    int ret = 0;
+    int i = 0;
+
+    assert(str);
+    temp = str = strdup(str);
+    if (!str)
+        goto error;
+
+    *argv = calloc(argv_len, sizeof(**argv));
+    if (!*argv)
+        goto error;
+
+    while ((p = strtok_r(str, " ", &savetok))) {
+        str = NULL;
+
+        argc++;
+        if (argc == argv_len) {
+            ret = duplexpand((void *)argv, sizeof(**argv), &argv_len);
+            if (ret == -1)
+                goto error;
+        }
+        temp1 = strdup(p);
+        if (!temp1)
+            goto error;
+        (*argv)[argc - 1] = temp1;
+    }
+
+    free(temp);
+    return argc;
+
+error:
+    fprintf(stderr, "out of memory\n");
+    free(temp);
+    for (i = 0; i < argc - 1; i++)
+        free((*argv)[i]);
+    free(*argv);
+    return -1;
+}
+
+static int
+invoke_gsyncd(int argc, char **argv)
+{
+    int i = 0;
+    int j = 0;
+    char *nargv[argc + 4];
+    char *python = NULL;
+
+    if (chdir("/") == -1)
+        goto error;
+
+    j = 0;
+    python = getenv("PYTHON");
+    if (!python)
+        python = PYTHON;
+    nargv[j++] = python;
+    nargv[j++] = GSYNCD_PREFIX "/python/syncdaemon/" GSYNCD_PY;
+    for (i = 1; i < argc; i++)
+        nargv[j++] = argv[i];
+
+    nargv[j++] = NULL;
+
+    execvp(python, nargv);
+
+    fprintf(stderr, "exec of '%s' failed\n", python);
+    return 127;
+
+error:
+    fprintf(stderr, "gsyncd initializaion failed\n");
+    return 1;
+}
+
+static int
+find_gsyncd(pid_t pid, pid_t ppid, char *name, void *data)
+{
+    char buf[NAME_MAX * 2] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *p = NULL;
+    int zeros = 0;
+    int ret = 0;
+    int fd = -1;
+    pid_t *pida = (pid_t *)data;
+
+    if (ppid != pida[0])
+        return 0;
+
+    snprintf(path, sizeof path, PROC "/%d/cmdline", pid);
+    fd = open(path, O_RDONLY);
+    if (fd == -1)
+        return 0;
+    ret = sys_read(fd, buf, sizeof(buf));
+    sys_close(fd);
+    if (ret == -1)
+        return 0;
+    for (zeros = 0, p = buf; zeros < 2 && p < buf + ret; p++)
+        zeros += !*p;
+
+    ret = 0;
+    switch (zeros) {
+        case 2:
+            if ((strcmp(basename(buf), basename(PYTHON)) ||
+                 strcmp(basename(buf + strlen(buf) + 1), GSYNCD_PY)) == 0) {
+                ret = 1;
+                break;
+            }
+            /* fallthrough */
+        case 1:
+            if (strcmp(basename(buf), GSYNCD_PY) == 0)
+                ret = 1;
+    }
+
+    if (ret == 1) {
+        if (pida[1] != -1) {
+            fprintf(stderr, GSYNCD_PY " sibling is not unique");
+            return -1;
+        }
+        pida[1] = pid;
+    }
+
+    return 0;
+}
+
+static int
+invoke_rsync(int argc, char **argv)
+{
+    int i = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    pid_t pid = -1;
+    pid_t ppid = -1;
+    pid_t pida[] = {-1, -1};
+    char *name = NULL;
+    char buf[PATH_MAX + 1] = {
+        0,
+    };
+    int ret = 0;
+
+    assert(argv[argc] == NULL);
+
+    if (argc < 2 || strcmp(argv[1], "--server") != 0)
+        goto error;
+
+    for (i = 2; i < argc && argv[i][0] == '-'; i++)
+        ;
+
+    if (!(i == argc - 2 && strcmp(argv[i], ".") == 0 &&
+          argv[i + 1][0] == '/')) {
+        fprintf(stderr, "need an rsync invocation without protected args\n");
+        goto error;
+    }
+
+    /* look up sshd we are spawned from */
+    for (pid = getpid();; pid = ppid) {
+        ppid = pidinfo(pid, &name);
+        if (ppid < 0) {
+            fprintf(stderr, "sshd ancestor not found\n");
+            goto error;
+        }
+        if (strcmp(name, "sshd") == 0) {
+            GF_FREE(name);
+            break;
+        }
+        GF_FREE(name);
+    }
+    /* look up "ssh-sibling" gsyncd */
+    pida[0] = pid;
+    ret = prociter(find_gsyncd, pida);
+    if (ret == -1 || pida[1] == -1) {
+        fprintf(stderr, "gsyncd sibling not found\n");
+        goto error;
+    }
+    /* check if rsync target matches gsyncd target */
+    snprintf(path, sizeof path, PROC "/%d/cwd", pida[1]);
+    ret = sys_readlink(path, buf, sizeof(buf));
+    if (ret == -1 || ret == sizeof(buf))
+        goto error;
+    if (strcmp(argv[argc - 1], "/") == 0 /* root dir cannot be a target */ ||
+        (strcmp(argv[argc - 1], path) /* match against gluster target */ &&
+         strcmp(argv[argc - 1], buf) /* match against file target */) != 0) {
+        fprintf(stderr, "rsync target does not match " GEOREP " session\n");
+        goto error;
+    }
+
+    argv[0] = RSYNC;
+
+    execvp(RSYNC, argv);
+
+    fprintf(stderr, "exec of " RSYNC " failed\n");
+    return 127;
+
+error:
+    fprintf(stderr, "disallowed " RSYNC " invocation\n");
+    return 1;
+}
+
+static int
+invoke_gluster(int argc, char **argv)
+{
+    int i = 0;
+    int j = 0;
+    int optsover = 0;
+    char *ov = NULL;
+
+    for (i = 1; i < argc; i++) {
+        ov = strtail(argv[i], "--");
+        if (ov && !optsover) {
+            if (*ov == '\0')
+                optsover = 1;
+            continue;
+        }
+        switch (++j) {
+            case 1:
+                if (strcmp(argv[i], "volume") != 0)
+                    goto error;
+                break;
+            case 2:
+                if (strcmp(argv[i], "info") != 0)
+                    goto error;
+                break;
+            case 3:
+                break;
+            default:
+                goto error;
+        }
+    }
+
+    argv[0] = "gluster";
+    execvp(SBIN_DIR "/gluster", argv);
+    fprintf(stderr, "exec of gluster failed\n");
+    return 127;
+
+error:
+    fprintf(stderr, "disallowed gluster invocation\n");
+    return 1;
+}
+
+struct invocable {
+    char *name;
+    int (*invoker)(int argc, char **argv);
+};
+
+struct invocable invocables[] = {{"rsync", invoke_rsync},
+                                 {"gsyncd", invoke_gsyncd},
+                                 {"gluster", invoke_gluster},
+                                 {NULL, NULL}};
+
+int
+main(int argc, char **argv)
+{
+    int ret = -1;
+    char *evas = NULL;
+    struct invocable *i = NULL;
+    char *b = NULL;
+    char *sargv = NULL;
+    int j = 0;
+
+#ifdef USE_LIBGLUSTERFS
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        return ENOMEM;
+
+    if (glusterfs_globals_init(ctx))
+        return 1;
+
+    THIS->ctx = ctx;
+    ret = default_mem_acct_init(THIS);
+    if (ret) {
+        fprintf(stderr, "internal error: mem accounting failed\n");
+        return 1;
+    }
+#endif
+
+    evas = getenv(_GLUSTERD_CALLED_);
+    if (evas && strcmp(evas, "1") == 0)
+        /* OK, we know glusterd called us, no need to look for further config
+         *...although this conclusion should not inherit to our children
+         */
+        unsetenv(_GLUSTERD_CALLED_);
+    else {
+        /* we regard all gsyncd invocations unsafe
+         * that do not come from glusterd and
+         * therefore restrict it
+         */
+        restricted = 1;
+
+        if (!getenv(_GSYNCD_DISPATCHED_)) {
+            evas = getenv("SSH_ORIGINAL_COMMAND");
+            if (evas)
+                sargv = evas;
+            else {
+                evas = getenv("SHELL");
+                if (evas && strcmp(basename(evas), "gsyncd") == 0 &&
+                    argc == 3 && strcmp(argv[1], "-c") == 0)
+                    sargv = argv[2];
+            }
+        }
+    }
+
+    if (!(sargv && restricted))
+        return invoke_gsyncd(argc, argv);
+
+    argc = str2argv(sargv, &argv);
+
+    if (argc == -1) {
+        fprintf(stderr, "internal error\n");
+        return 1;
+    }
+
+    if (setenv(_GSYNCD_DISPATCHED_, "1", 1) == -1) {
+        fprintf(stderr, "internal error\n");
+        goto out;
+    }
+
+    b = basename(argv[0]);
+    for (i = invocables; i->name; i++) {
+        if (strcmp(b, i->name) == 0)
+            return i->invoker(argc, argv);
+    }
+
+    fprintf(stderr, "invoking %s in restricted SSH session is not allowed\n",
+            b);
+
+out:
+    for (j = 1; j < argc; j++)
+        free(argv[j]);
+    free(argv);
+    return 1;
+}
diff --git a/geo-replication/src/gverify.sh b/geo-replication/src/gverify.sh
new file mode 100755
index 00000000000..f5f70d245e0
--- /dev/null
+++ b/geo-replication/src/gverify.sh
@@ -0,0 +1,276 @@
+#!/bin/bash
+
+# Script to verify the Master and Slave Gluster compatibility.
+# To use ./gverify <master volume> <slave user> <slave host> <slave volume> <ssh port> <log file>
+# Returns 0 if master and slave compatible.
+
+# Considering buffer_size 100MB
+BUFFER_SIZE=104857600;
+SSH_PORT=$5;
+master_log_file=`gluster --print-logdir`/geo-replication/gverify-mastermnt.log
+slave_log_file=`gluster --print-logdir`/geo-replication/gverify-slavemnt.log
+
+function SSHM()
+{
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -q \
+	    -oPasswordAuthentication=no \
+	    -oStrictHostKeyChecking=no \
+	    -oControlMaster=yes \
+	    "$@";
+    fi
+}
+
+function get_inode_num()
+{
+    local os
+    case `uname -s` in
+        NetBSD) os="NetBSD";;
+        Linux)  os="Linux";;
+        *)      os="Default";;
+    esac
+
+    if [[ "X$os" = "XNetBSD" ]]; then
+        echo $(stat -f "%i" "$1")
+    else
+        echo $(stat -c "%i" "$1")
+    fi
+}
+
+function umount_lazy()
+{
+    local os
+    case `uname -s` in
+        NetBSD) os="NetBSD";;
+        Linux)  os="Linux";;
+        *)      os="Default";;
+    esac
+
+    if [[ "X$os" = "XNetBSD" ]]; then
+        umount -f -R "$1"
+    else
+        umount -l "$1"
+    fi;
+}
+
+function disk_usage()
+{
+    local os
+    case `uname -s` in
+        NetBSD) os="NetBSD";;
+        Linux)  os="Linux";;
+        *)      os="Default";;
+    esac
+
+    if [[ "X$os" = "XNetBSD" ]]; then
+        echo $(df -P "$1" | tail -1)
+    else
+        echo $(df -P -B1 "$1" | tail -1)
+    fi;
+
+}
+
+function cmd_slave()
+{
+    local cmd_line;
+    cmd_line=$(cat <<EOF
+function do_verify() {
+ver=\$(gluster --version | head -1 | cut -f2 -d " ");
+echo \$ver;
+};
+source /etc/profile && do_verify;
+EOF
+);
+
+echo $cmd_line;
+}
+
+function master_stats()
+{
+    MASTERVOL=$1;
+    local inet6=$2;
+    local d;
+    local i;
+    local disk_size;
+    local used_size;
+    local ver;
+    local m_status;
+
+    d=$(mktemp -d -t ${0##*/}.XXXXXX 2>/dev/null);
+    if [ "$inet6" = "inet6" ]; then
+        glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --xlator-option="transport.address-family=inet6" --volfile-id $MASTERVOL -l $master_log_file $d;
+    else
+        glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --volfile-id $MASTERVOL -l $master_log_file $d;
+    fi
+
+    i=$(get_inode_num $d);
+    if [[ "$i" -ne "1" ]]; then
+        echo 0:0;
+        exit 1;
+    fi;
+    cd $d;
+    disk_size=$(disk_usage $d | awk "{print \$2}");
+    used_size=$(disk_usage $d | awk "{print \$3}");
+    umount_lazy $d;
+    rmdir $d;
+    ver=$(gluster --version | head -1 | cut -f2 -d " ");
+    m_status=$(echo "$disk_size:$used_size:$ver");
+    echo $m_status
+}
+
+
+function slave_stats()
+{
+    SLAVEUSER=$1;
+    SLAVEHOST=$2;
+    SLAVEVOL=$3;
+    local inet6=$4;
+    local cmd_line;
+    local ver;
+    local status;
+
+    d=$(mktemp -d -t ${0##*/}.XXXXXX 2>/dev/null);
+    if [ "$inet6" = "inet6" ]; then
+        glusterfs --xlator-option="*dht.lookup-unhashed=off" --xlator-option="transport.address-family=inet6" --volfile-server $SLAVEHOST --volfile-id $SLAVEVOL -l $slave_log_file $d;
+    else
+        glusterfs --xlator-option="*dht.lookup-unhashed=off" --volfile-server $SLAVEHOST --volfile-id $SLAVEVOL -l $slave_log_file $d;
+    fi
+
+    i=$(get_inode_num $d);
+    if [[ "$i" -ne "1" ]]; then
+        echo 0:0;
+        exit 1;
+    fi;
+    cd $d;
+    disk_size=$(disk_usage $d | awk "{print \$2}");
+    used_size=$(disk_usage $d | awk "{print \$3}");
+    no_of_files=$(find $d -maxdepth 1 -path "$d/.trashcan" -prune -o -path "$d" -o -print0 -quit);
+    umount_lazy $d;
+    rmdir $d;
+
+    cmd_line=$(cmd_slave);
+    ver=`SSHM $SLAVEUSER@$SLAVEHOST bash -c "'$cmd_line'"`;
+    status=$disk_size:$used_size:$ver:$no_of_files;
+    echo $status
+}
+
+function ping_host ()
+{
+    ### Use bash internal socket support
+    {
+        exec 100<>/dev/tcp/$1/$2
+        if [ $? -ne '0' ]; then
+            return 1;
+        else
+            exec 100>&-
+            return 0;
+        fi
+    } 1>&2 2>/dev/null
+}
+
+function main()
+{
+    log_file=$6
+    > $log_file
+
+    inet6=$7
+    local cmd_line
+    local ver
+
+    # Use FORCE_BLOCKER flag in the error message to differentiate
+    # between the errors which the force command should bypass
+
+    # Test tcp connection to port 22, this is necessary since `ping`
+    # does not work on all environments where 'ssh' is allowed but
+    # ICMP is filterd
+
+    ping_host $3 ${SSH_PORT}
+
+    if [ $? -ne 0 ]; then
+        echo "FORCE_BLOCKER|$3 not reachable." > $log_file
+        exit 1;
+    fi;
+
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    else
+        ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 "echo Testing_Passwordless_SSH";
+    fi
+
+    if [ $? -ne 0 ]; then
+        echo "FORCE_BLOCKER|Passwordless ssh login has not been setup with $3 for user $2." > $log_file
+        exit 1;
+    fi;
+
+    cmd_line=$(cmd_slave);
+    if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then
+        ver=$(ssh -p ${SSH_PORT} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 bash -c "'$cmd_line'")
+    else
+        ver=$(ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=no $2@$3 bash -c "'$cmd_line'")
+    fi
+
+    if [ -z "$ver" ]; then
+        echo "FORCE_BLOCKER|gluster command not found on $3 for user $2." > $log_file
+        exit 1;
+    fi;
+
+    ERRORS=0;
+    master_data=$(master_stats $1 ${inet6});
+    slave_data=$(slave_stats $2 $3 $4 ${inet6});
+    master_disk_size=$(echo $master_data | cut -f1 -d':');
+    slave_disk_size=$(echo $slave_data | cut -f1 -d':');
+    master_used_size=$(echo $master_data | cut -f2 -d':');
+    slave_used_size=$(echo $slave_data | cut -f2 -d':');
+    master_version=$(echo $master_data | cut -f3 -d':');
+    slave_version=$(echo $slave_data | cut -f3 -d':');
+    slave_no_of_files=$(echo $slave_data | cut -f4 -d':');
+
+    if [[ "x$master_disk_size" = "x" || "x$master_version" = "x" || "$master_disk_size" -eq "0" ]]; then
+        echo "FORCE_BLOCKER|Unable to mount and fetch master volume details. Please check the log: $master_log_file" > $log_file;
+	exit 1;
+    fi;
+
+    if [[ "x$slave_disk_size" = "x" || "x$slave_version" = "x" || "$slave_disk_size" -eq "0" ]]; then
+	echo "FORCE_BLOCKER|Unable to mount and fetch slave volume details. Please check the log: $slave_log_file" > $log_file;
+	exit 1;
+    fi;
+
+    # The above checks are mandatory and force command should be blocked
+    # if they fail. The checks below can be bypassed if force option is
+    # provided hence no FORCE_BLOCKER flag.
+
+    if [ "$slave_disk_size" -lt "$master_disk_size" ]; then
+        echo "Total disk size of master is greater than disk size of slave." >> $log_file;
+        ERRORS=$(($ERRORS + 1));
+    fi
+
+    effective_master_used_size=$(( $master_used_size + $BUFFER_SIZE ))
+    slave_available_size=$(( $slave_disk_size - $slave_used_size ))
+    master_available_size=$(( $master_disk_size - $effective_master_used_size ));
+
+    if [ "$slave_available_size" -lt "$master_available_size" ]; then
+        echo "Total available size of master is greater than available size of slave" >> $log_file;
+        ERRORS=$(($ERRORS + 1));
+    fi
+
+    if [ ! -z $slave_no_of_files ]; then
+        echo "$3::$4 is not empty. Please delete existing files in $3::$4 and retry, or use force to continue without deleting the existing files." >> $log_file;
+        ERRORS=$(($ERRORS + 1));
+    fi;
+
+    if [[ $master_version != $slave_version ]]; then
+        echo "Gluster version mismatch between master and slave. Master version: $master_version Slave version: $slave_version" >> $log_file;
+        ERRORS=$(($ERRORS + 1));
+    fi;
+
+    exit $ERRORS;
+}
+
+
+main "$@";
diff --git a/geo-replication/src/peer_georep-sshkey.py.in b/geo-replication/src/peer_georep-sshkey.py.in
new file mode 100644
index 00000000000..58696e9a616
--- /dev/null
+++ b/geo-replication/src/peer_georep-sshkey.py.in
@@ -0,0 +1,116 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+"""
+Usage:
+    gluster-georep-sshkey generate
+    or
+    gluster-georep-sshkey generate --no-prefix
+
+Generates two SSH keys(one for gsyncd access and other for tar) in all
+peer nodes and collects the public keys to the local node where it is
+initiated. Adds `command=` prefix to common_secret.pem.pub if `--no-prefix`
+argument is not passed.
+"""
+import os
+import glob
+
+from gluster.cliutils import (node_output_ok, execute, execute_in_peers,
+                              Cmd, runcli)
+from prettytable import PrettyTable
+
+
+SECRET_PEM = "@GLUSTERD_WORKDIR@/geo-replication/secret.pem"
+TAR_SSH_PEM = "@GLUSTERD_WORKDIR@/geo-replication/tar_ssh.pem"
+GSYNCD_CMD = 'command="@GLUSTERFS_LIBEXECDIR@/gsyncd" '
+TAR_CMD = 'command="tar ${SSH_ORIGINAL_COMMAND#* }" '
+COMMON_SECRET_FILE = "@GLUSTERD_WORKDIR@/geo-replication/common_secret.pem.pub"
+
+
+class NodeGenCmd(Cmd):
+    name = "node-generate"
+
+    def args(self, parser):
+        parser.add_argument("no_prefix")
+
+    def run(self, args):
+        # Regenerate if secret.pem.pub not exists
+        if not os.path.exists(SECRET_PEM + ".pub"):
+            # Cleanup old files
+            for f in glob.glob(SECRET_PEM + "*"):
+                os.remove(f)
+
+            execute(["ssh-keygen", "-N", "", "-f", SECRET_PEM])
+
+        # Regenerate if ssh_tar.pem.pub not exists
+        if not os.path.exists(TAR_SSH_PEM + ".pub"):
+            # Cleanup old files
+            for f in glob.glob(TAR_SSH_PEM + "*"):
+                os.remove(f)
+
+            execute(["ssh-keygen", "-N", "", "-f", TAR_SSH_PEM])
+
+        # Add required prefixes if prefix is not "container"
+        prefix_secret_pem_pub = ""
+        prefix_tar_ssh_pem_pub = ""
+        if args.no_prefix != "no-prefix":
+            prefix_secret_pem_pub = GSYNCD_CMD
+            prefix_tar_ssh_pem_pub = TAR_CMD
+
+        data = {"default_pub": "", "tar_pub": ""}
+        with open(SECRET_PEM + ".pub") as f:
+            data["default_pub"] = prefix_secret_pem_pub + f.read().strip()
+
+        with open(TAR_SSH_PEM + ".pub") as f:
+            data["tar_pub"] = prefix_tar_ssh_pem_pub + f.read().strip()
+
+        node_output_ok(data)
+
+
+def color_status(value):
+    if value in ["UP", "OK"]:
+        return "green"
+    return "red"
+
+
+class GenCmd(Cmd):
+    name = "generate"
+
+    def args(self, parser):
+        parser.add_argument("--no-prefix", help="Do not use prefix in "
+                            "generated pub keys", action="store_true")
+
+    def run(self, args):
+        prefix = "no-prefix" if args.no_prefix else "."
+        out = execute_in_peers("node-generate", [prefix])
+
+        common_secrets = []
+        table = PrettyTable(["NODE", "NODE STATUS", "KEYGEN STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["KEYGEN STATUS"] = "r"
+        for p in out:
+            if p.ok:
+                common_secrets.append(p.output["default_pub"])
+                common_secrets.append(p.output["tar_pub"])
+
+            table.add_row([p.hostname,
+                           "UP" if p.node_up else "DOWN",
+                           "OK" if p.ok else "NOT OK: {0}".format(
+                               p.error)])
+
+        with open(COMMON_SECRET_FILE, "w") as f:
+            f.write("\n".join(common_secrets) + "\n")
+
+        print (table)
+
+
+if __name__ == "__main__":
+    runcli()
diff --git a/geo-replication/src/peer_gsec_create.in b/geo-replication/src/peer_gsec_create.in
new file mode 100755
index 00000000000..6d4a4847013
--- /dev/null
+++ b/geo-replication/src/peer_gsec_create.in
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libexecdir=@libexecdir@
+
+if [ ! -f "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub ]; then
+    \rm -rf "$GLUSTERD_WORKDIR"/geo-replication/secret.pem*
+    ssh-keygen -N '' -f "$GLUSTERD_WORKDIR"/geo-replication/secret.pem > /dev/null
+fi
+
+if [ ! -f "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub ]; then
+    \rm -rf "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem*
+    ssh-keygen -N '' -f "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem > /dev/null
+fi
+
+if [ "Xcontainer" = "X$1" ]; then
+    output1=`cat "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub`
+    output2=`cat "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub`
+else
+    output1=`echo command=\"${libexecdir}/glusterfs/gsyncd\" ""``cat "$GLUSTERD_WORKDIR"/geo-replication/secret.pem.pub`
+    output2=`echo command=\"tar \$\{SSH_ORIGINAL_COMMAND#* \}\" ""``cat "$GLUSTERD_WORKDIR"/geo-replication/tar_ssh.pem.pub`
+fi
+echo -e "$output1\n$output2"
diff --git a/geo-replication/src/peer_mountbroker.in b/geo-replication/src/peer_mountbroker.in
new file mode 100644
index 00000000000..8ecf38ded41
--- /dev/null
+++ b/geo-replication/src/peer_mountbroker.in
@@ -0,0 +1,211 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import os
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+import json
+import sys
+
+PROG_DESCRIPTION = """
+GlusterFS Mountbroker user management
+"""
+
+args = None
+
+
+def ok(message=""):
+    if (not args and "-j" in sys.argv) or (args and args.json):
+        print(json.dumps({"ok": True, "message": message}))
+    else:
+        if message:
+            print(message)
+
+    sys.exit(0)
+
+
+def notok(message=""):
+    if (not args and "-j" in sys.argv) or (args and args.json):
+        print(json.dumps({"ok": False, "message": message}))
+    else:
+        print("error: %s" % message)
+
+    # Always return zero due to limitation while executing
+    # as `gluster system:: execute`
+    sys.exit(0)
+
+
+class NoStdErrParser(ArgumentParser):
+    """
+    with gluster system:: execute, stderr gives
+    "Unable to end. Error : Bad file descriptor" error,
+    so deriving new class, prints error message and
+    exits with zero.
+    """
+    def error(self, message):
+        notok(message)
+
+
+class MountbrokerUserMgmt(object):
+    def __init__(self, volfile):
+        self.volfile = volfile
+        self._options = {}
+        self.commented_lines = []
+        self._parse()
+
+    def _parse(self):
+        with open(self.volfile, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith("option "):
+                    key, value = line.split(" ")[1:]
+                    self._options[key] = value
+                if line.startswith("#"):
+                    self.commented_lines.append(line)
+
+    def _get_write_data(self):
+        op = "volume management\n"
+        op += "    type mgmt/glusterd\n"
+        for k, v in self._options.items():
+            op += "    option %s %s\n" % (k, v)
+        for line in self.commented_lines:
+            op += "    %s\n" % line
+        op += "end-volume"
+        return op
+
+    def save(self):
+        with open(self.volfile + "_tmp", "w") as f:
+            f.write(self._get_write_data())
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(self.volfile + "_tmp", self.volfile)
+
+    def set_opt(self, key, value):
+        self._options[key] = value.strip()
+
+    def remove_opt(self, key):
+        if key in self._options:
+            del(self._options[key])
+
+    def add_user(self, user, volumes):
+        vols = set()
+        for k, v in self._options.items():
+            if k.startswith("mountbroker-geo-replication.") \
+               and user == k.split(".")[-1]:
+                vols.update(v.split(","))
+
+        vols.update(volumes)
+        self.set_opt("mountbroker-geo-replication.%s" % user,
+                     ",".join(vols))
+
+    def remove_volume(self, user, volumes):
+        vols = set()
+        for k, v in self._options.items():
+            if k.startswith("mountbroker-geo-replication.") \
+               and user == k.split(".")[-1]:
+                vols.update(v.split(","))
+
+        for v1 in volumes:
+            vols.discard(v1)
+
+        if vols:
+            self.set_opt("mountbroker-geo-replication.%s" % user,
+                         ",".join(vols))
+        else:
+            self.remove_opt("mountbroker-geo-replication.%s" % user)
+
+    def remove_user(self, user):
+        self.remove_opt("mountbroker-geo-replication.%s" % user)
+
+    def info(self):
+        data = {"users": []}
+
+        for k, v in self._options.items():
+            if k.startswith("mountbroker-geo-replication."):
+                data["users"].append(
+                    {"name": k.split(".")[-1], "volumes": v.split(",")}
+                )
+            else:
+                data[k] = v
+
+        return data
+
+
+def format_info(data):
+    op = "%s %s\n" % ("Option".ljust(50), "Value".ljust(50))
+    op += ("-" * 101) + "\n"
+    for key, value in data.items():
+        if key != "users":
+            op += "%s %s\n" % (key.ljust(50), value)
+
+    op += "\nUsers: %s\n" % ("None" if not data["users"] else "")
+    for user in data["users"]:
+        op += "%s: %s\n" % (user["name"], ", ".join(user["volumes"]))
+    op += "\n\n"
+    return op
+
+
+def _get_args():
+    parser = NoStdErrParser(formatter_class=RawDescriptionHelpFormatter,
+                            description=PROG_DESCRIPTION)
+
+    parser.add_argument('-j', dest="json", help="JSON output",
+                        action="store_true")
+    subparsers = parser.add_subparsers(title='subcommands', dest='cmd')
+    parser_useradd = subparsers.add_parser('user')
+    parser_userdel = subparsers.add_parser('userdel')
+    parser_volumedel = subparsers.add_parser('volumedel')
+    subparsers.add_parser('info')
+    parser_opt = subparsers.add_parser('opt')
+    parser_optdel = subparsers.add_parser('optdel')
+
+    parser_useradd.add_argument('username', help="Username", type=str)
+    parser_useradd.add_argument('volumes', type=str, default='',
+                                help="Volumes list. ',' separated")
+
+    parser_volumedel.add_argument('username', help="Username", type=str)
+    parser_volumedel.add_argument('volumes', type=str, default='',
+                                help="Volumes list. ',' separated")
+
+    parser_userdel.add_argument('username', help="Username", type=str)
+
+    parser_opt.add_argument('opt_name', help="Name", type=str)
+    parser_opt.add_argument('opt_value', help="Value", type=str)
+
+    parser_optdel.add_argument('opt_name', help="Name", type=str)
+
+    return parser.parse_args()
+
+
+def main():
+    global args
+    args = _get_args()
+
+    m = MountbrokerUserMgmt("@GLUSTERD_VOLFILE@")
+
+    if args.cmd == "opt":
+        m.set_opt(args.opt_name, args.opt_value)
+    elif args.cmd == "optdel":
+        m.remove_opt(args.opt_name)
+    elif args.cmd == "userdel":
+        m.remove_user(args.username)
+    elif args.cmd == "user":
+        volumes = [v.strip() for v in args.volumes.split(",")
+                   if v.strip() != ""]
+        m.add_user(args.username, volumes)
+    elif args.cmd == "volumedel":
+        volumes = [v.strip() for v in args.volumes.split(",")
+                   if v.strip() != ""]
+        m.remove_volume(args.username, volumes)
+    elif args.cmd == "info":
+        info = m.info()
+        if not args.json:
+            info = format_info(info)
+        ok(info)
+
+    if args.cmd != "info":
+        m.save()
+        ok()
+
+if __name__ == "__main__":
+    main()
diff --git a/geo-replication/src/peer_mountbroker.py.in b/geo-replication/src/peer_mountbroker.py.in
new file mode 100644
index 00000000000..40b90ffc560
--- /dev/null
+++ b/geo-replication/src/peer_mountbroker.py.in
@@ -0,0 +1,401 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import os
+from errno import EEXIST, ENOENT
+
+from gluster.cliutils import (execute, Cmd, node_output_ok,
+                              node_output_notok, execute_in_peers,
+                              runcli, oknotok)
+from prettytable import PrettyTable
+
+LOG_DIR = "@localstatedir@/log/glusterfs/geo-replication-slaves"
+CLI_LOG = "@localstatedir@/log/glusterfs/cli.log"
+GEOREP_DIR = "@GLUSTERD_WORKDIR@/geo-replication"
+GLUSTERD_VOLFILE = "@GLUSTERD_VOLFILE@"
+
+
+class MountbrokerUserMgmt(object):
+    def __init__(self, volfile):
+        self.volfile = volfile
+        self._options = {}
+        self.commented_lines = []
+        self.user_volumes = {}
+        self._parse()
+
+    def _parse(self):
+        """ Example glusterd.vol
+        volume management
+            type mgmt/glusterd
+            option working-directory /var/lib/glusterd
+            option transport-type socket,rdma
+            option transport.socket.keepalive-time 10
+            option transport.socket.keepalive-interval 2
+            option transport.socket.read-fail-log off
+            option rpc-auth-allow-insecure on
+            option ping-timeout 0
+            option event-threads 1
+            # option base-port 49152
+            option mountbroker-root /var/mountbroker-root
+            option mountbroker-geo-replication.user1 vol1,vol2,vol3
+            option geo-replication-log-group geogroup
+            option rpc-auth-allow-insecure on
+        end-volume
+        """
+        with open(self.volfile, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith("option "):
+                    key, value = line.split()[1:]
+                    self._options[key] = value
+                if line.startswith("#"):
+                    self.commented_lines.append(line)
+
+        for k, v in self._options.items():
+            if k.startswith("mountbroker-geo-replication."):
+                user = k.split(".")[-1]
+                self.user_volumes[user] = set(v.split(","))
+
+    def get_group(self):
+        return self._options.get("geo-replication-log-group", None)
+
+    def _get_write_data(self):
+        op = "volume management\n"
+        op += "    type mgmt/glusterd\n"
+        for k, v in self._options.items():
+            if k.startswith("mountbroker-geo-replication."):
+                # Users will be added seperately
+                continue
+
+            op += "    option %s %s\n" % (k, v)
+
+        for k, v in self.user_volumes.items():
+            if v:
+                op += ("    option mountbroker-geo-replication."
+                       "%s %s\n" % (k, ",".join(v)))
+
+        for line in self.commented_lines:
+            op += "    %s\n" % line
+
+        op += "end-volume"
+        return op
+
+    def save(self):
+        with open(self.volfile + "_tmp", "w") as f:
+            f.write(self._get_write_data())
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(self.volfile + "_tmp", self.volfile)
+
+    def set_mount_root_and_group(self, mnt_root, group):
+        self._options["mountbroker-root"] = mnt_root
+        self._options["geo-replication-log-group"] = group
+
+    def add(self, volume, user):
+        user_volumes = self.user_volumes.get(user, None)
+
+        if user_volumes is not None and volume in user_volumes:
+            # User and Volume already exists
+            return
+
+        if user_volumes is None:
+            # User not exists
+            self.user_volumes[user] = set()
+
+        self.user_volumes[user].add(volume)
+
+    def remove(self, volume=None, user=None):
+        if user is not None:
+            if volume is None:
+                self.user_volumes[user] = set()
+            else:
+                try:
+                    self.user_volumes.get(user, set()).remove(volume)
+                except KeyError:
+                    pass
+        else:
+            if volume is None:
+                return
+
+            for k, v in self.user_volumes.items():
+                try:
+                    self.user_volumes[k].remove(volume)
+                except KeyError:
+                    pass
+
+    def info(self):
+        # Convert Volumes set into Volumes list
+        users = {}
+        for k, v in self.user_volumes.items():
+            users[k] = list(v)
+
+        data = {
+            "mountbroker-root": self._options.get("mountbroker-root", "None"),
+            "geo-replication-log-group": self._options.get(
+                "geo-replication-log-group", ""),
+            "users": users
+        }
+
+        return data
+
+
+class NodeSetup(Cmd):
+    # Test if group exists using `getent group <grp>`
+    # and then group add using `groupadd <grp>`
+    # chgrp -R <grp> /var/log/glusterfs/geo-replication-slaves
+    # chgrp -R <grp> /var/lib/glusterd/geo-replication
+    # chmod -R 770 /var/log/glusterfs/geo-replication-slaves
+    # chmod 770 /var/lib/glusterd/geo-replication
+    # mkdir -p <mnt_root>
+    # chmod 0711 <mnt_root>
+    # If selinux,
+    # semanage fcontext -a -e /home /var/mountbroker-root
+    # restorecon -Rv /var/mountbroker-root
+    name = "node-setup"
+
+    def args(self, parser):
+        parser.add_argument("mount_root")
+        parser.add_argument("group")
+
+    def run(self, args):
+        m = MountbrokerUserMgmt(GLUSTERD_VOLFILE)
+
+        try:
+            os.makedirs(args.mount_root)
+        except OSError as e:
+            if e.errno == EEXIST:
+                pass
+            else:
+                node_output_notok("Unable to Create {0}".format(
+                    args.mount_root))
+
+        execute(["chmod", "0711", args.mount_root])
+        try:
+            execute(["semanage", "fcontext", "-a", "-e",
+                     "/home", args.mount_root])
+        except OSError as e:
+            if e.errno == ENOENT:
+                pass
+            else:
+                node_output_notok(
+                    "Unable to run semanage: {0}".format(e))
+
+        try:
+            execute(["restorecon", "-Rv", args.mount_root])
+        except OSError as e:
+            if e.errno == ENOENT:
+                pass
+            else:
+                node_output_notok(
+                    "Unable to run restorecon: {0}".format(e))
+
+        rc, out, err = execute(["getent", "group", args.group])
+        if rc != 0:
+            node_output_notok("User Group not exists")
+
+        execute(["chgrp", "-R", args.group, GEOREP_DIR])
+        execute(["chgrp", "-R", args.group, LOG_DIR])
+        execute(["chgrp", args.group, CLI_LOG])
+        execute(["chmod", "770", GEOREP_DIR])
+        execute(["find", LOG_DIR, "-type", "d", "-exec", "chmod", "770", "{}",
+                 "+"])
+        execute(["find", LOG_DIR, "-type", "f", "-exec", "chmod", "660", "{}",
+                 "+"])
+        execute(["chmod", "660", CLI_LOG])
+
+        m.set_mount_root_and_group(args.mount_root, args.group)
+        m.save()
+
+        node_output_ok()
+
+
+def color_status(value):
+    if value.lower() in ("up", "ok", "yes"):
+        return "green"
+    else:
+        return "red"
+
+
+class CliSetup(Cmd):
+    # gluster-mountbroker setup <MOUNT_ROOT> <GROUP>
+    name = "setup"
+
+    def args(self, parser):
+        parser.add_argument("mount_root",
+                            help="Path to the mountbroker-root directory.")
+        parser.add_argument("group",
+                            help="Group to be used for setup.")
+
+    def run(self, args):
+        out = execute_in_peers("node-setup", [args.mount_root,
+                                              args.group])
+        table = PrettyTable(["NODE", "NODE STATUS", "SETUP STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["SETUP STATUS"] = "r"
+        for p in out:
+            table.add_row([p.hostname,
+                           "UP" if p.node_up else "DOWN",
+                           "OK" if p.ok else "NOT OK: {0}".format(
+                               p.error)])
+
+        print(table)
+
+
+class NodeStatus(Cmd):
+    # Check if Group exists
+    # Check if user exists
+    # Check directory permission /var/log/glusterfs/geo-replication-slaves
+    # and /var/lib/glusterd/geo-replication
+    # Check mount root and its permissions
+    # Check glusterd.vol file for user, group, dir existance
+    name = "node-status"
+
+    def run(self, args):
+        m = MountbrokerUserMgmt(GLUSTERD_VOLFILE)
+        data = m.info()
+        data["group_exists"] = False
+        data["path_exists"] = False
+
+        rc, out, err = execute(["getent", "group",
+                                data["geo-replication-log-group"]])
+
+        if rc == 0:
+            data["group_exists"] = True
+
+        if os.path.exists(data["mountbroker-root"]):
+            data["path_exists"] = True
+
+        node_output_ok(data)
+
+
+class CliStatus(Cmd):
+    # gluster-mountbroker status
+    name = "status"
+
+    def run(self, args):
+        out = execute_in_peers("node-status")
+        table = PrettyTable(["NODE", "NODE STATUS", "MOUNT ROOT",
+                             "GROUP", "USERS"])
+        table.align["NODE STATUS"] = "r"
+
+        for p in out:
+            node_data = p.output
+            if node_data == "" or node_data == "N/A":
+                node_data = {}
+
+            users_row_data = ""
+            for k, v in node_data.get("users", {}).items():
+                users_row_data += "{0}({1}) ".format(k, ", ".join(v))
+
+            if not users_row_data:
+                users_row_data = "None"
+
+            mount_root = node_data.get("mountbroker-root", "None")
+            if mount_root != "None":
+                mount_root += "({0})".format(oknotok(
+                    node_data.get("path_exists", False)))
+
+            grp = node_data.get("geo-replication-log-group", "None")
+            if grp != "None":
+                grp += "({0})".format(oknotok(
+                    node_data.get("group_exists", False)))
+
+            table.add_row([p.hostname,
+                           "UP" if p.node_up else "DOWN",
+                           mount_root,
+                           grp,
+                           users_row_data])
+
+        print(table)
+
+
+class NodeAdd(Cmd):
+    # useradd -m -g <grp> <usr>
+    # useradd to glusterd.vol
+    name = "node-add"
+
+    def args(self, parser):
+        parser.add_argument("volume")
+        parser.add_argument("user")
+
+    def run(self, args):
+        m = MountbrokerUserMgmt(GLUSTERD_VOLFILE)
+        grp = m.get_group()
+        if grp is None:
+            node_output_notok("Group is not available")
+
+        m.add(args.volume, args.user)
+        m.save()
+        node_output_ok()
+
+
+class CliAdd(Cmd):
+    # gluster-mountbroker add <VOLUME> <USER>
+    name = "add"
+
+    def args(self, parser):
+        parser.add_argument("volume",
+                            help="Volume to be added.")
+        parser.add_argument("user",
+                            help="User for which volume is to be added.")
+
+    def run(self, args):
+        out = execute_in_peers("node-add", [args.volume,
+                                            args.user])
+        table = PrettyTable(["NODE", "NODE STATUS", "ADD STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["ADD STATUS"] = "r"
+
+        for p in out:
+            table.add_row([p.hostname,
+                           "UP" if p.node_up else "DOWN",
+                           "OK" if p.ok else "NOT OK: {0}".format(
+                               p.error)])
+
+        print(table)
+
+
+class NodeRemove(Cmd):
+    # userremove from glusterd.vol file
+    name = "node-remove"
+
+    def args(self, parser):
+        parser.add_argument("volume")
+        parser.add_argument("user")
+
+    def run(self, args):
+        m = MountbrokerUserMgmt(GLUSTERD_VOLFILE)
+        volume = None if args.volume == "." else args.volume
+        user = None if args.user == "." else args.user
+        m.remove(volume=volume, user=user)
+        m.save()
+        node_output_ok()
+
+
+class CliRemove(Cmd):
+    # gluster-mountbroker remove --volume <VOLUME> --user <USER>
+    name = "remove"
+
+    def args(self, parser):
+        parser.add_argument("--volume", default=".", help="Volume to be removed.")
+        parser.add_argument("--user", default=".",
+                            help="User for which volume has to be removed.")
+
+    def run(self, args):
+        out = execute_in_peers("node-remove", [args.volume,
+                                               args.user])
+        table = PrettyTable(["NODE", "NODE STATUS", "REMOVE STATUS"])
+        table.align["NODE STATUS"] = "r"
+        table.align["REMOVE STATUS"] = "r"
+
+        for p in out:
+            table.add_row([p.hostname,
+                           "UP" if p.node_up else "DOWN",
+                           "OK" if p.ok else "NOT OK: {0}".format(
+                               p.error)])
+
+        print(table)
+
+if __name__ == "__main__":
+    runcli()
diff --git a/geo-replication/src/procdiggy.c b/geo-replication/src/procdiggy.c
new file mode 100644
index 00000000000..8068ef79a42
--- /dev/null
+++ b/geo-replication/src/procdiggy.c
@@ -0,0 +1,136 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <ctype.h>
+#include <sys/param.h> /* for PATH_MAX */
+
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syscall.h>
+#include "procdiggy.h"
+
+pid_t
+pidinfo(pid_t pid, char **name)
+{
+    char buf[NAME_MAX * 2] = {
+        0,
+    };
+    FILE *f = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *p = NULL;
+    int ret = 0;
+    pid_t lpid = -1;
+
+    if (name)
+        *name = NULL;
+
+    snprintf(path, sizeof path, PROC "/%d/status", pid);
+
+    f = fopen(path, "r");
+    if (!f)
+        return -1;
+
+    for (;;) {
+        size_t len;
+        memset(buf, 0, sizeof(buf));
+        if (fgets(buf, sizeof(buf), f) == NULL || (len = strlen(buf)) == 0 ||
+            buf[len - 1] != '\n') {
+            lpid = -1;
+            goto out;
+        }
+        buf[len - 1] = '\0';
+
+        if (name && !*name) {
+            p = strtail(buf, "Name:");
+            if (p) {
+                while (isspace(*++p))
+                    ;
+                *name = gf_strdup(p);
+                if (!*name) {
+                    lpid = -2;
+                    goto out;
+                }
+                continue;
+            }
+        }
+
+        p = strtail(buf, "PPid:");
+        if (p)
+            break;
+    }
+
+    while (isspace(*++p))
+        ;
+    ret = gf_string2int(p, &lpid);
+    if (ret == -1)
+        lpid = -1;
+
+out:
+    fclose(f);
+    if (lpid == -1 && name && *name)
+        GF_FREE(*name);
+    if (lpid == -2)
+        fprintf(stderr, "out of memory\n");
+    return lpid;
+}
+
+int
+prociter(int (*proch)(pid_t pid, pid_t ppid, char *tmpname, void *data),
+         void *data)
+{
+    char *name = NULL;
+    DIR *d = NULL;
+    struct dirent *de = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    pid_t pid = -1;
+    pid_t ppid = -1;
+    int ret = 0;
+
+    d = sys_opendir(PROC);
+    if (!d)
+        return -1;
+
+    for (;;) {
+        errno = 0;
+        de = sys_readdir(d, scratch);
+        if (!de || errno != 0)
+            break;
+
+        if (gf_string2int(de->d_name, &pid) != -1 && pid >= 0) {
+            ppid = pidinfo(pid, &name);
+            switch (ppid) {
+                case -1:
+                    continue;
+                case -2:
+                    break;
+            }
+            ret = proch(pid, ppid, name, data);
+            GF_FREE(name);
+            if (ret)
+                break;
+        }
+    }
+    sys_closedir(d);
+    if (!de && errno) {
+        fprintf(stderr, "failed to traverse " PROC " (%s)\n", strerror(errno));
+        ret = -1;
+    }
+
+    return ret;
+}
diff --git a/geo-replication/src/procdiggy.h b/geo-replication/src/procdiggy.h
new file mode 100644
index 00000000000..e17ccd31c89
--- /dev/null
+++ b/geo-replication/src/procdiggy.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifdef __NetBSD__
+#include <sys/syslimits.h>
+#endif /* __NetBSD__ */
+
+#define PROC "/proc"
+
+pid_t
+pidinfo(pid_t pid, char **name);
+
+int
+prociter(int (*proch)(pid_t pid, pid_t ppid, char *name, void *data),
+         void *data);
diff --git a/geo-replication/src/set_geo_rep_pem_keys.sh b/geo-replication/src/set_geo_rep_pem_keys.sh
new file mode 100755
index 00000000000..8a43fa39d1f
--- /dev/null
+++ b/geo-replication/src/set_geo_rep_pem_keys.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Script to copy the pem keys from the user's home directory
+# to $GLUSTERD_WORKDIR/geo-replication and then copy
+# the keys to other nodes in the cluster and add them to the
+# respective authorized keys. The script takes as argument the
+# user name and assumes that the user will be present in all
+# the nodes in the cluster. Not to be used for root user
+
+function main()
+{
+    user=$1
+    master_vol=$2
+    slave_vol=$3
+    GLUSTERD_WORKDIR=$(gluster system:: getwd)
+
+    if [ "$user" == "" ];  then
+        echo "Please enter the user's name"
+        exit 1;
+    fi
+
+    if [ "$master_vol" == "" ]; then
+        echo "Invalid master volume name"
+        exit 1;
+    fi
+
+    if [ "$slave_vol" == "" ]; then
+        echo "Invalid slave volume name"
+        exit 1;
+    fi
+
+    COMMON_SECRET_PEM_PUB=${master_vol}_${slave_vol}_common_secret.pem.pub
+
+    if [ "$user" == "root" ]; then
+        echo "This script is not needed for root"
+        exit 1;
+    fi
+
+    home_dir=`getent passwd $user | cut -d ':' -f 6`;
+
+    if [ "$home_dir" == "" ]; then
+        echo "No user $user found"
+        exit 1;
+    fi
+
+    if [ -f $home_dir/${COMMON_SECRET_PEM_PUB} ]; then
+        cp $home_dir/${COMMON_SECRET_PEM_PUB} ${GLUSTERD_WORKDIR}/geo-replication/
+        gluster system:: copy file /geo-replication/${COMMON_SECRET_PEM_PUB}
+        gluster system:: execute add_secret_pub $user geo-replication/${master_vol}_${slave_vol}_common_secret.pem.pub
+        gluster vol set ${slave_vol} features.read-only on
+    else
+        echo "$home_dir/common_secret.pem.pub not present. Please run geo-replication command on master with push-pem option to generate the file"
+        exit 1;
+    fi
+    exit 0;
+}
+
+main "$@";
diff --git a/geo-replication/syncdaemon/Makefile.am b/geo-replication/syncdaemon/Makefile.am
new file mode 100644
index 00000000000..d70e3368faf
--- /dev/null
+++ b/geo-replication/syncdaemon/Makefile.am
@@ -0,0 +1,8 @@
+syncdaemondir = $(GLUSTERFS_LIBEXECDIR)/python/syncdaemon
+
+syncdaemon_PYTHON = rconf.py gsyncd.py __init__.py master.py README.md repce.py \
+	resource.py syncdutils.py monitor.py libcxattr.py gsyncdconfig.py \
+	libgfchangelog.py gsyncdstatus.py conf.py logutils.py \
+	subcmds.py argsupgrade.py py2py3.py
+
+CLEANFILES =
diff --git a/geo-replication/syncdaemon/README.md b/geo-replication/syncdaemon/README.md
new file mode 100644
index 00000000000..5ab785ae669
--- /dev/null
+++ b/geo-replication/syncdaemon/README.md
@@ -0,0 +1,57 @@
+gsycnd, the Gluster Syncdaemon
+==============================
+
+REQUIREMENTS
+------------
+
+_gsyncd_ is a program which can operate either in _master_ or in _slave_ mode.
+Requirements are categorized according to this.
+
+* supported OS is GNU/Linux
+* Python >= 2.5, or 2.4 with Ctypes (see below) (both)
+* OpenSSH >= 4.0 (master) / SSH2 compliant sshd (eg. openssh) (slave)
+* rsync (both)
+* glusterfs: with marker and changelog support (master & slave);
+* FUSE: glusterfs fuse module with auxiliary gfid based access support
+
+INSTALLATION
+------------
+
+As of now, the supported way of operation is running from the source directory or using the RPMs given.
+
+
+CONFIGURATION
+-------------
+
+gsyncd tunables are a subset of the long command-line options; for listing them,
+type
+
+    gsyncd.py --help
+
+and see the long options up to "--config-file". (The leading double dash should be omitted;
+interim underscores and dashes are interchangeable.) The set of options bear some resemblance
+to those of glusterfs and rsync.
+
+The config file format matches the following syntax:
+
+      <option1>: <value1>
+      <option2>: <value2>
+      # comment
+
+By default (unless specified by the option `-c`), gsyncd looks for config file at _conf/gsyncd_template.conf_
+in the source tree.
+
+USAGE
+-----
+
+gsyncd is a utilitly for continuous mirroring, ie. it mirrors master to slave incrementally.
+Assume we have a gluster volume _pop_ at localhost. We try to set up the mirroring for volume
+_pop_ using gsyncd for gluster volume _moz_ on remote machine/cluster @ example.com. The
+respective gsyncd invocations are (demoing some syntax sugaring):
+
+`gsyncd.py :pop example.com::moz`
+
+gsyncd has to be available on both sides; it's location on the remote side has to be specified
+via the "--remote-gsyncd" option (or "remote-gsyncd" config file parameter). (This option can also be
+used for setting options on the remote side, although the suggested mode of operation is to
+set parameters like log file / pid file in the configuration file.)
diff --git a/geo-replication/syncdaemon/__codecheck.py b/geo-replication/syncdaemon/__codecheck.py
new file mode 100644
index 00000000000..9437147f7d9
--- /dev/null
+++ b/geo-replication/syncdaemon/__codecheck.py
@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+import os
+import os.path
+import sys
+import tempfile
+import shutil
+
+ipd = tempfile.mkdtemp(prefix='codecheck-aux')
+
+try:
+    # add a fake ipaddr module, we don't want to
+    # deal with the real one (just test our code)
+    f = open(os.path.join(ipd, 'ipaddr.py'), 'w')
+    f.write("""
+class IPAddress(object):
+    pass
+class IPNetwork(list):
+    pass
+""")
+    f.close()
+    sys.path.append(ipd)
+
+    fl = os.listdir(os.path.dirname(sys.argv[0]) or '.')
+    fl.sort()
+    for f in fl:
+        if f[-3:] != '.py' or f[0] == '_':
+            continue
+        m = f[:-3]
+        sys.stdout.write('importing %s ...' % m)
+        __import__(m)
+        print(' OK.')
+
+    def sys_argv_set(a):
+        sys.argv = sys.argv[:1] + a
+
+    gsyncd = sys.modules['gsyncd']
+    for a in [['--help'], ['--version'],
+              ['--canonicalize-escape-url', '/foo']]:
+        print(('>>> invoking program with args: %s' % ' '.join(a)))
+        pid = os.fork()
+        if not pid:
+            sys_argv_set(a)
+            gsyncd.main()
+        _, r = os.waitpid(pid, 0)
+        if r:
+            raise RuntimeError('invocation failed')
+finally:
+    shutil.rmtree(ipd)
diff --git a/geo-replication/syncdaemon/__init__.py b/geo-replication/syncdaemon/__init__.py
new file mode 100644
index 00000000000..b4648b69645
--- /dev/null
+++ b/geo-replication/syncdaemon/__init__.py
@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
diff --git a/geo-replication/syncdaemon/argsupgrade.py b/geo-replication/syncdaemon/argsupgrade.py
new file mode 100644
index 00000000000..7af40633ef8
--- /dev/null
+++ b/geo-replication/syncdaemon/argsupgrade.py
@@ -0,0 +1,359 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+# Converts old style args into new style args
+
+from __future__ import print_function
+import sys
+from argparse import ArgumentParser
+import socket
+import os
+
+from syncdutils import GsyncdError
+from conf import GLUSTERD_WORKDIR
+
+
+def gethostbyname(hnam):
+    """gethostbyname wrapper"""
+    try:
+        return socket.gethostbyname(hnam)
+    except socket.gaierror:
+        ex = sys.exc_info()[1]
+        raise GsyncdError("failed to resolve %s: %s" %
+                          (hnam, ex.strerror))
+
+
+def slave_url(urldata):
+    urldata = urldata.replace("ssh://", "")
+    host, vol = urldata.split("::")
+    vol = vol.split(":")[0]
+    return "%s::%s" % (host, vol)
+
+
+def init_gsyncd_template_conf():
+    path = GLUSTERD_WORKDIR + "/geo-replication/gsyncd_template.conf"
+    dname = os.path.dirname(path)
+    if not os.path.exists(dname):
+        try:
+            os.mkdir(dname)
+        except OSError:
+            pass
+
+    if not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def init_gsyncd_session_conf(master, slave):
+    slave = slave_url(slave)
+    master = master.strip(":")
+    slavehost, slavevol = slave.split("::")
+    slavehost = slavehost.split("@")[-1]
+
+    # Session Config File
+    path = "%s/geo-replication/%s_%s_%s/gsyncd.conf" % (
+        GLUSTERD_WORKDIR, master, slavehost, slavevol)
+
+    if os.path.exists(os.path.dirname(path)) and not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def init_gsyncd_conf(path):
+    dname = os.path.dirname(path)
+    if not os.path.exists(dname):
+        try:
+            os.mkdir(dname)
+        except OSError:
+            pass
+
+    if os.path.exists(dname) and not os.path.exists(path):
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+
+
+def upgrade():
+    # Create dummy template conf(empty), hack to avoid glusterd
+    # fail when it does stat to check the existence.
+    init_gsyncd_template_conf()
+
+    inet6 = False
+    if "--inet6" in sys.argv:
+        inet6 = True
+
+    if "--monitor" in sys.argv:
+        # python gsyncd.py --path=/bricks/b1
+        # --monitor -c gsyncd.conf
+        # --iprefix=/var :gv1
+        # --glusterd-uuid=f26ac7a8-eb1b-4ea7-959c-80b27d3e43d0
+        # f241::gv2
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--glusterd-uuid")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        p.add_argument("--path", action="append")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # Overwrite the sys.argv after rearrange
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+        sys.argv = [
+            sys.argv[0],
+            "monitor",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--local-node-id",
+            pargs.glusterd_uuid
+        ]
+    elif "--status-get" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2
+        # --status-get --path /bricks/b1
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--path")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        sys.argv = [
+            sys.argv[0],
+            "status",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--local-path",
+            pargs.path
+        ]
+    elif "--canonicalize-url" in sys.argv:
+        # This can accept multiple URLs and converts each URL to the
+        # format ssh://USER@IP:gluster://127.0.0.1:VOLUME
+        # This format not used in gsyncd, but added for glusterd compatibility
+        p = ArgumentParser()
+        p.add_argument("--canonicalize-url", nargs="+")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        for url in pargs.canonicalize_url:
+            host, vol = url.split("::")
+            host = host.replace("ssh://", "")
+            remote_addr = host
+            if "@" not in remote_addr:
+                remote_addr = "root@" + remote_addr
+
+            user, hname = remote_addr.split("@")
+
+            if not inet6:
+                hname = gethostbyname(hname)
+
+            print(("ssh://%s@%s:gluster://127.0.0.1:%s" % (
+                user, hname, vol)))
+
+        sys.exit(0)
+    elif "--normalize-url" in sys.argv:
+        # Adds schema prefix as ssh://
+        # This format not used in gsyncd, but added for glusterd compatibility
+        p = ArgumentParser()
+        p.add_argument("--normalize-url")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+        print(("ssh://%s" % slave_url(pargs.normalize_url)))
+        sys.exit(0)
+    elif "--config-get-all" in sys.argv:
+        #  -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-get-all
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        sys.argv = [
+            sys.argv[0],
+            "config-get",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--show-defaults",
+            "--use-underscore"
+        ]
+    elif "--verify" in sys.argv and "spawning" in sys.argv:
+        # Just checks that able to spawn gsyncd or not
+        sys.exit(0)
+    elif "--slavevoluuid-get" in sys.argv:
+        # --slavevoluuid-get f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--slavevoluuid-get")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+        host, vol = pargs.slavevoluuid_get.split("::")
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "voluuidget",
+            host,
+            vol
+        ]
+    elif "--config-set-rx" in sys.argv:
+        # Not required since default conf is not generated
+        # and custom conf generated only when required
+        # -c gsyncd.conf --config-set-rx remote-gsyncd
+        # /usr/local/libexec/glusterfs/gsyncd . .
+        # Touch the gsyncd.conf file and create session
+        # directory if required
+        p = ArgumentParser()
+        p.add_argument("-c", dest="config_file")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # If not template conf then it is trying to create
+        # session config, create a empty file instead
+        if pargs.config_file.endswith("gsyncd.conf"):
+            init_gsyncd_conf(pargs.config_file)
+        sys.exit(0)
+    elif "--create" in sys.argv:
+        # To update monitor status file
+        # --create Created -c gsyncd.conf
+        # --iprefix=/var :gv1 f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--create")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "monitor-status",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            pargs.create
+        ]
+    elif "--config-get" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-get pid-file
+        p = ArgumentParser()
+        p.add_argument("--config-get")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-get",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--only-value",
+            "--show-defaults",
+            "--name",
+            pargs.config_get.replace("_", "-")
+        ]
+    elif "--config-set" in sys.argv:
+        # ignore session-owner
+        if "session-owner" in sys.argv:
+            sys.exit(0)
+
+        # --path=/bricks/b1  -c gsyncd.conf :gv1 f241::gv2
+        # --config-set log_level DEBUG
+        p = ArgumentParser()
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--config-set", action='store_true')
+        p.add_argument("name")
+        p.add_argument("--value")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-set",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            "--name=%s" % pargs.name,
+            "--value=%s" % pargs.value
+        ]
+    elif "--config-check" in sys.argv:
+        # --config-check georep_session_working_dir
+        p = ArgumentParser()
+        p.add_argument("--config-check")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-check",
+            pargs.config_check.replace("_", "-")
+        ]
+    elif "--config-del" in sys.argv:
+        # -c gsyncd.conf --iprefix=/var :gv1 f241::gv2 --config-del log_level
+        p = ArgumentParser()
+        p.add_argument("--config-del")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("-c")
+        p.add_argument("--iprefix")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "config-reset",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave),
+            pargs.config_del.replace("_", "-")
+        ]
+    elif "--delete" in sys.argv:
+        # --delete -c gsyncd.conf --iprefix=/var
+        # --path-list=--path=/bricks/b1  :gv1 f241::gv2
+        p = ArgumentParser()
+        p.add_argument("--reset-sync-time", action="store_true")
+        p.add_argument("--path-list")
+        p.add_argument("master")
+        p.add_argument("slave")
+        p.add_argument("--iprefix")
+        p.add_argument("-c")
+        pargs = p.parse_known_args(sys.argv[1:])[0]
+
+        init_gsyncd_session_conf(pargs.master, pargs.slave)
+
+        paths = pargs.path_list.split("--path=")
+        paths = ["--path=%s" % x.strip() for x in paths if x.strip() != ""]
+
+        # Modified sys.argv
+        sys.argv = [
+            sys.argv[0],
+            "delete",
+            pargs.master.strip(":"),
+            slave_url(pargs.slave)
+        ]
+        sys.argv += paths
+
+        if pargs.reset_sync_time:
+            sys.argv.append("--reset-sync-time")
+
+    if inet6:
+        # Add `--inet6` as first argument
+        sys.argv = [sys.argv[0], "--inet6"] + sys.argv[1:]
diff --git a/geo-replication/syncdaemon/conf.py.in b/geo-replication/syncdaemon/conf.py.in
new file mode 100644
index 00000000000..2042fa9cdfb
--- /dev/null
+++ b/geo-replication/syncdaemon/conf.py.in
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+GLUSTERFS_LIBEXECDIR = '@GLUSTERFS_LIBEXECDIR@'
+GLUSTERD_WORKDIR = "@GLUSTERD_WORKDIR@"
+
+LOCALSTATEDIR = "@localstatedir@"
+UUID_FILE = "@GLUSTERD_WORKDIR@/glusterd.info"
+GLUSTERFS_CONFDIR = "@SYSCONF_DIR@/glusterfs"
+GCONF_VERSION = 4.0
diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py
new file mode 100644
index 00000000000..257ed72c6ae
--- /dev/null
+++ b/geo-replication/syncdaemon/gsyncd.py
@@ -0,0 +1,325 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from argparse import ArgumentParser
+import time
+import os
+from errno import EEXIST
+import sys
+import logging
+
+from logutils import setup_logging
+import gsyncdconfig as gconf
+from rconf import rconf
+import subcmds
+from conf import GLUSTERD_WORKDIR, GLUSTERFS_CONFDIR, GCONF_VERSION
+from syncdutils import (set_term_handler, finalize, lf,
+                        log_raise_exception, FreeObject, escape)
+import argsupgrade
+
+
+GSYNCD_VERSION = "gsyncd.py %s.0" % GCONF_VERSION
+
+
+def main():
+    rconf.starttime = time.time()
+
+    # If old Glusterd sends commands in old format, below function
+    # converts the sys.argv to new format. This conversion is added
+    # temporarily for backward compatibility. This can be removed
+    # once integrated with Glusterd2
+    # This modifies sys.argv globally, so rest of the code works as usual
+    argsupgrade.upgrade()
+
+    # Default argparse version handler prints to stderr, which is fixed in
+    # 3.x series but not in 2.x, using custom parser to fix this issue
+    if "--version" in sys.argv:
+        print(GSYNCD_VERSION)
+        sys.exit(0)
+
+    parser = ArgumentParser()
+    parser.add_argument("--inet6", action="store_true")
+    sp = parser.add_subparsers(dest="subcmd")
+
+    # Monitor Status File update
+    p = sp.add_parser("monitor-status")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("status", help="Update Monitor Status")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Monitor
+    p = sp.add_parser("monitor")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--pause-on-start",
+                   action="store_true",
+                   help="Start with Paused state")
+    p.add_argument("--local-node-id", help="Local Node ID")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--use-gconf-volinfo", action="store_true")
+
+    # Worker
+    p = sp.add_parser("worker")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("--local-path", help="Local Brick Path")
+    p.add_argument("--feedback-fd", type=int,
+                   help="feedback fd between monitor and worker")
+    p.add_argument("--local-node", help="Local master node")
+    p.add_argument("--local-node-id", help="Local Node ID")
+    p.add_argument("--subvol-num", type=int, help="Subvolume number")
+    p.add_argument("--is-hottier", action="store_true",
+                   help="Is this brick part of hot tier")
+    p.add_argument("--resource-remote",
+                   help="Remote node to connect to Slave Volume")
+    p.add_argument("--resource-remote-id",
+                   help="Remote node ID to connect to Slave Volume")
+    p.add_argument("--slave-id", help="Slave Volume ID")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Slave
+    p = sp.add_parser("slave")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave details user@host::vol format")
+    p.add_argument("--session-owner")
+    p.add_argument("--master-brick",
+                   help="Master brick which is connected to the Slave")
+    p.add_argument("--master-node",
+                   help="Master node which is connected to the Slave")
+    p.add_argument("--master-node-id",
+                   help="Master node ID which is connected to the Slave")
+    p.add_argument("--local-node", help="Local Slave node")
+    p.add_argument("--local-node-id", help="Local Slave ID")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # All configurations which are configured via "slave-" options
+    # DO NOT add default values for these configurations, default values
+    # will be picked from template config file
+    p.add_argument("--slave-timeout", type=int,
+                   help="Timeout to end gsyncd at Slave side")
+    p.add_argument("--use-rsync-xattrs", action="store_true")
+    p.add_argument("--slave-log-level", help="Slave Gsyncd Log level")
+    p.add_argument("--slave-gluster-log-level",
+                   help="Slave Gluster mount Log level")
+    p.add_argument("--slave-gluster-command-dir",
+                   help="Directory where Gluster binaries exist on slave")
+    p.add_argument("--slave-access-mount", action="store_true",
+                   help="Do not lazy umount the slave volume")
+    p.add_argument("--master-dist-count", type=int,
+                   help="Master Distribution count")
+
+    # Status
+    p = sp.add_parser("status")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--local-path", help="Local Brick Path")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--json", action="store_true")
+
+    # Config-check
+    p = sp.add_parser("config-check")
+    p.add_argument("name", help="Config Name")
+    p.add_argument("--value", help="Config Value")
+    p.add_argument("--debug", action="store_true")
+
+    # Config-get
+    p = sp.add_parser("config-get")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("--name", help="Config Name")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+    p.add_argument("--show-defaults", action="store_true")
+    p.add_argument("--only-value", action="store_true")
+    p.add_argument("--use-underscore", action="store_true")
+    p.add_argument("--json", action="store_true")
+
+    # Config-set
+    p = sp.add_parser("config-set")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-n", "--name", help="Config Name")
+    p.add_argument("-v", "--value", help="Config Value")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # Config-reset
+    p = sp.add_parser("config-reset")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("name", help="Config Name")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument("--debug", action="store_true")
+
+    # voluuidget
+    p = sp.add_parser("voluuidget")
+    p.add_argument("host", help="Hostname")
+    p.add_argument("volname", help="Volume Name")
+    p.add_argument("--debug", action="store_true")
+
+    # Delete
+    p = sp.add_parser("delete")
+    p.add_argument("master", help="Master Volume Name")
+    p.add_argument("slave", help="Slave")
+    p.add_argument("-c", "--config-file", help="Config File")
+    p.add_argument('--path', dest='paths', action="append")
+    p.add_argument("--reset-sync-time", action="store_true",
+                   help="Reset Sync Time")
+    p.add_argument("--debug", action="store_true")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Extra template values, All arguments are already part of template
+    # variables, use this for adding extra variables
+    extra_tmpl_args = {}
+
+    # Add First/Primary Slave host, user and volume
+    if getattr(args, "slave", None) is not None:
+        hostdata, slavevol = args.slave.split("::")
+        hostdata = hostdata.split("@")
+        slavehost = hostdata[-1]
+        slaveuser = "root"
+        if len(hostdata) == 2:
+            slaveuser = hostdata[0]
+        extra_tmpl_args["primary_slave_host"] = slavehost
+        extra_tmpl_args["slaveuser"] = slaveuser
+        extra_tmpl_args["slavevol"] = slavevol
+
+    # Add Bricks encoded path
+    if getattr(args, "local_path", None) is not None:
+        extra_tmpl_args["local_id"] = escape(args.local_path)
+
+    # Add Master Bricks encoded path(For Slave)
+    if getattr(args, "master_brick", None) is not None:
+        extra_tmpl_args["master_brick_id"] = escape(args.master_brick)
+
+    # Load configurations
+    config_file = getattr(args, "config_file", None)
+
+    # Subcmd accepts config file argument but not passed
+    # Set default path for config file in that case
+    # If an subcmd accepts config file then it also accepts
+    # master and Slave arguments.
+    if config_file is None and hasattr(args, "config_file") \
+        and args.subcmd != "slave":
+        config_file = "%s/geo-replication/%s_%s_%s/gsyncd.conf" % (
+            GLUSTERD_WORKDIR,
+            args.master,
+            extra_tmpl_args["primary_slave_host"],
+            extra_tmpl_args["slavevol"])
+
+    # If Config file path not exists, log error and continue using default conf
+    config_file_error_msg = None
+    if config_file is not None and not os.path.exists(config_file):
+        # Logging not yet initialized, create the error message to
+        # log later and reset the config_file to None
+        config_file_error_msg = lf(
+            "Session config file not exists, using the default config",
+            path=config_file)
+        config_file = None
+
+    rconf.config_file = config_file
+
+    # Override gconf values from argument values only if it is slave gsyncd
+    override_from_args = False
+    if args.subcmd == "slave":
+        override_from_args = True
+
+    if config_file is not None and \
+       args.subcmd in ["monitor", "config-get", "config-set", "config-reset"]:
+        ret = gconf.is_config_file_old(config_file, args.master, extra_tmpl_args["slavevol"])
+        if ret is not None:
+           gconf.config_upgrade(config_file, ret)
+
+    # Load Config file
+    gconf.load(GLUSTERFS_CONFDIR + "/gsyncd.conf",
+               config_file,
+               vars(args),
+               extra_tmpl_args,
+               override_from_args)
+
+    # Default label to print in log file
+    label = args.subcmd
+    if args.subcmd in ("worker"):
+        # If Worker, then add brick path also to label
+        label = "%s %s" % (args.subcmd, args.local_path)
+    elif args.subcmd == "slave":
+        # If Slave add Master node and Brick details
+        label = "%s %s%s" % (args.subcmd, args.master_node, args.master_brick)
+
+    # Setup Logger
+    # Default log file
+    log_file = gconf.get("cli-log-file")
+    log_level = gconf.get("cli-log-level")
+    if getattr(args, "master", None) is not None and \
+       getattr(args, "slave", None) is not None:
+        log_file = gconf.get("log-file")
+        log_level = gconf.get("log-level")
+
+    # Use different log file location for Slave log file
+    if args.subcmd == "slave":
+        log_file = gconf.get("slave-log-file")
+        log_level = gconf.get("slave-log-level")
+
+    if args.debug:
+        log_file = "-"
+        log_level = "DEBUG"
+
+    # Create Logdir if not exists
+    try:
+        if log_file != "-":
+            os.mkdir(os.path.dirname(log_file))
+    except OSError as e:
+        if e.errno != EEXIST:
+            raise
+
+    setup_logging(
+        log_file=log_file,
+        level=log_level,
+        label=label
+    )
+
+    if config_file_error_msg is not None:
+        logging.warn(config_file_error_msg)
+
+    # Log message for loaded config file
+    if config_file is not None:
+        logging.debug(lf("Using session config file", path=config_file))
+
+    set_term_handler()
+    excont = FreeObject(exval=0)
+
+    # Gets the function name based on the input argument. For example
+    # if subcommand passed as argument is monitor then it looks for
+    # function with name "subcmd_monitor" in subcmds file
+    func = getattr(subcmds, "subcmd_" + args.subcmd.replace("-", "_"), None)
+
+    try:
+        try:
+            if func is not None:
+                rconf.args = args
+                func(args)
+        except:
+            log_raise_exception(excont)
+    finally:
+        finalize(exval=excont.exval)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/geo-replication/syncdaemon/gsyncdconfig.py b/geo-replication/syncdaemon/gsyncdconfig.py
new file mode 100644
index 00000000000..8848071997a
--- /dev/null
+++ b/geo-replication/syncdaemon/gsyncdconfig.py
@@ -0,0 +1,485 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+try:
+    from ConfigParser import RawConfigParser, NoSectionError
+except ImportError:
+    from configparser import RawConfigParser, NoSectionError
+import os
+import shutil
+from string import Template
+from datetime import datetime
+from threading import Lock
+
+
+# Global object which can be used in other modules
+# once load_config is called
+_gconf = {}
+
+
+class GconfNotConfigurable(Exception):
+    pass
+
+
+class GconfInvalidValue(Exception):
+    pass
+
+
+class Gconf(object):
+    def __init__(self, default_conf_file, custom_conf_file=None,
+                 args={}, extra_tmpl_args={}, override_from_args=False):
+        self.lock = Lock()
+        self.default_conf_file = default_conf_file
+        self.custom_conf_file = custom_conf_file
+        self.tmp_conf_file = None
+        self.gconf = {}
+        self.gconfdata = {}
+        self.gconf_typecast = {}
+        self.template_conf = []
+        self.non_configurable_configs = []
+        self.prev_mtime = 0
+        if custom_conf_file is not None:
+            self.tmp_conf_file = custom_conf_file + ".tmp"
+
+        self.session_conf_items = []
+        self.args = args
+        self.extra_tmpl_args = extra_tmpl_args
+        self.override_from_args = override_from_args
+        # Store default values only if overwritten, Only for JSON/CLI output
+        self.default_values = {}
+        self._load()
+
+    def _tmpl_substitute(self):
+        tmpl_values = {}
+        for k, v in self.gconf.items():
+            tmpl_values[k.replace("-", "_")] = v
+
+        # override the config file values with the one user passed
+        for k, v in self.args.items():
+            # override the existing value only if set by user
+            if v is not None:
+                tmpl_values[k] = v
+
+        for k, v in self.extra_tmpl_args.items():
+            tmpl_values[k] = v
+
+        for k, v in self.gconf.items():
+            if k in self.template_conf and \
+               (isinstance(v, str) or isinstance(v, unicode)):
+                self.gconf[k] = Template(v).safe_substitute(tmpl_values)
+
+    def _do_typecast(self):
+        for k, v in self.gconf.items():
+            cast_func = globals().get(
+                "to_" + self.gconf_typecast.get(k, "string"), None)
+            if cast_func is not None:
+                self.gconf[k] = cast_func(v)
+                if self.default_values.get(k, None) is not None:
+                    self.default_values[k] = cast_func(v)
+
+    def reset(self, name):
+        # If custom conf file is not set then it is only read only configs
+        if self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        # If a config can not be modified
+        if name != "all" and not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        cnf = RawConfigParser()
+        with open(self.custom_conf_file) as f:
+            cnf.readfp(f)
+
+            # Nothing to Reset, Not configured
+            if name != "all":
+                if not cnf.has_option("vars", name):
+                    return True
+
+                # Remove option from custom conf file
+                cnf.remove_option("vars", name)
+            else:
+                # Remove and add empty section, do not disturb if config file
+                # already has any other section
+                try:
+                    cnf.remove_section("vars")
+                except NoSectionError:
+                    pass
+
+                cnf.add_section("vars")
+
+        with open(self.tmp_conf_file, "w") as fw:
+            cnf.write(fw)
+
+        os.rename(self.tmp_conf_file, self.custom_conf_file)
+
+        self.reload()
+
+        return True
+
+    def set(self, name, value):
+        if self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        if not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        if not self._is_valid_value(name, value):
+            raise GconfInvalidValue()
+
+        curr_val = self.gconf.get(name, None)
+        if curr_val == value:
+            return True
+
+        cnf = RawConfigParser()
+        with open(self.custom_conf_file) as f:
+            cnf.readfp(f)
+
+        if not cnf.has_section("vars"):
+            cnf.add_section("vars")
+
+        cnf.set("vars", name, value)
+        with open(self.tmp_conf_file, "w") as fw:
+            cnf.write(fw)
+
+        os.rename(self.tmp_conf_file, self.custom_conf_file)
+
+        self.reload()
+
+        return True
+
+    def check(self, name, value=None, with_conffile=True):
+        if with_conffile and self.custom_conf_file is None:
+            raise GconfNotConfigurable()
+
+        if not self._is_configurable(name):
+            raise GconfNotConfigurable()
+
+        if value is not None and not self._is_valid_value(name, value):
+            raise GconfInvalidValue()
+
+
+    def _load_with_lock(self):
+        with self.lock:
+            self._load()
+
+    def _load(self):
+        self.gconf = {}
+        self.template_conf = []
+        self.gconf_typecast = {}
+        self.non_configurable_configs = []
+        self.session_conf_items = []
+        self.default_values = {}
+
+        conf = RawConfigParser()
+        # Default Template config file
+        with open(self.default_conf_file) as f:
+            conf.readfp(f)
+
+        # Custom Config file
+        if self.custom_conf_file is not None:
+            with open(self.custom_conf_file) as f:
+                conf.readfp(f)
+
+        # Get version from default conf file
+        self.version = conf.get("__meta__", "version")
+
+        # Populate default values
+        for sect in conf.sections():
+            if sect in ["__meta__", "vars"]:
+                continue
+
+            # Collect list of available options with help details
+            self.gconfdata[sect] = {}
+            for k, v in conf.items(sect):
+                self.gconfdata[sect][k] = v.strip()
+
+            # Collect the Type cast information
+            if conf.has_option(sect, "type"):
+                self.gconf_typecast[sect] = conf.get(sect, "type")
+
+            # Prepare list of configurable conf
+            if conf.has_option(sect, "configurable"):
+                if conf.get(sect, "configurable").lower() == "false":
+                    self.non_configurable_configs.append(sect)
+
+            # if it is a template conf value which needs to be substituted
+            if conf.has_option(sect, "template"):
+                if conf.get(sect, "template").lower().strip() == "true":
+                    self.template_conf.append(sect)
+
+            # Set default values
+            if conf.has_option(sect, "value"):
+                self.gconf[sect] = conf.get(sect, "value").strip()
+
+        # Load the custom conf elements and overwrite
+        if conf.has_section("vars"):
+            for k, v in conf.items("vars"):
+                self.session_conf_items.append(k)
+                self.default_values[k] = self.gconf.get(k, "")
+                self.gconf[k] = v.strip()
+
+        # Overwrite the Slave configurations which are sent as
+        # arguments to gsyncd slave
+        if self.override_from_args:
+            for k, v in self.args.items():
+                k = k.replace("_", "-")
+                if k.startswith("slave-") and k in self.gconf:
+                    self.gconf[k] = v
+
+        self._tmpl_substitute()
+        self._do_typecast()
+
+    def reload(self, with_lock=True):
+        if self._is_config_changed():
+            if with_lock:
+                self._load_with_lock()
+            else:
+                self._load()
+
+    def get(self, name, default_value=None, with_lock=True):
+        if with_lock:
+            with self.lock:
+                return self.gconf.get(name, default_value)
+        else:
+            return self.gconf.get(name, default_value)
+
+    def getall(self, show_defaults=False, show_non_configurable=False):
+        cnf = {}
+        if not show_defaults:
+            for k in self.session_conf_items:
+                if k not in self.non_configurable_configs:
+                    dv = self.default_values.get(k, "")
+                    cnf[k] = {
+                        "value": self.get(k),
+                        "default": dv,
+                        "configurable": True,
+                        "modified": False if dv == "" else True
+                    }
+            return cnf
+
+        # Show all configs including defaults
+        for k, v in self.gconf.items():
+            configurable = False if k in self.non_configurable_configs \
+                           else True
+            dv = self.default_values.get(k, "")
+            modified = False if dv == "" else True
+            if show_non_configurable:
+                cnf[k] = {
+                    "value": v,
+                    "default": dv,
+                    "configurable": configurable,
+                    "modified": modified
+                }
+            else:
+                if k not in self.non_configurable_configs:
+                    cnf[k] = {
+                        "value": v,
+                        "default": dv,
+                        "configurable": configurable,
+                        "modified": modified
+                    }
+
+        return cnf
+
+    def getr(self, name, default_value=None):
+        with self.lock:
+            self.reload(with_lock=False)
+            return self.get(name, default_value, with_lock=False)
+
+    def get_help(self, name=None):
+        pass
+
+    def _is_configurable(self, name):
+        item = self.gconfdata.get(name, None)
+        if item is None:
+            return False
+
+        return item.get("configurable", True)
+
+    def _is_valid_value(self, name, value):
+        item = self.gconfdata.get(name, None)
+        if item is None:
+            return False
+
+        # If validation func not defined
+        if item.get("validation", None) is None:
+            return True
+
+        # minmax validation
+        if item["validation"] == "minmax":
+            return validate_minmax(value, item["min"], item["max"])
+
+        if item["validation"] == "choice":
+            return validate_choice(value, item["allowed_values"])
+
+        if item["validation"] == "bool":
+            return validate_bool(value)
+
+        if item["validation"] == "execpath":
+            return validate_execpath(value)
+
+        if item["validation"] == "unixtime":
+            return validate_unixtime(value)
+
+        if item["validation"] == "int":
+            return validate_int(value)
+
+        return False
+
+    def _is_config_changed(self):
+        if self.custom_conf_file is not None and \
+           os.path.exists(self.custom_conf_file):
+                st = os.lstat(self.custom_conf_file)
+                if st.st_mtime > self.prev_mtime:
+                    self.prev_mtime = st.st_mtime
+                    return True
+
+        return False
+
+def is_config_file_old(config_file, mastervol, slavevol):
+    cnf = RawConfigParser()
+    cnf.read(config_file)
+    session_section = "peers %s %s" % (mastervol, slavevol)
+    try:
+        return dict(cnf.items(session_section))
+    except NoSectionError:
+        return None
+
+def config_upgrade(config_file, ret):
+    config_file_backup = os.path.join(os.path.dirname(config_file), "gsyncd.conf.bkp")
+
+    #copy old config file in a backup file
+    shutil.copyfile(config_file, config_file_backup)
+
+    #write a new config file
+    config = RawConfigParser()
+    config.add_section('vars')
+
+    for key, value in ret.items():
+        #handle option name changes
+        if key == "use_tarssh":
+            new_key = "sync-method"
+            if value == "true":
+                new_value = "tarssh"
+            else:
+                new_value = "rsync"
+            config.set('vars', new_key, new_value)
+        elif key == "timeout":
+            new_key = "slave-timeout"
+            config.set('vars', new_key, value)
+        #for changes like: ignore_deletes to ignore-deletes
+        else:
+            new_key = key.replace("_", "-")
+            config.set('vars', new_key, value)
+
+    with open(config_file, 'w') as configfile:
+        config.write(configfile)
+
+
+def validate_int(value):
+    try:
+        _ = int(value)
+        return True
+    except ValueError:
+        return False
+
+
+def validate_unixtime(value):
+    try:
+        y = datetime.fromtimestamp(int(value)).strftime("%Y")
+        if y == "1970":
+            return False
+
+        return True
+    except ValueError:
+        return False
+
+
+def validate_minmax(value, minval, maxval):
+    try:
+        value = int(value)
+        minval = int(minval)
+        maxval = int(maxval)
+        return value >= minval and value <= maxval
+    except ValueError:
+        return False
+
+
+def validate_choice(value, allowed_values):
+    allowed_values = allowed_values.split(",")
+    allowed_values = [v.strip() for v in allowed_values]
+
+    return value in allowed_values
+
+
+def validate_bool(value):
+    return value in ["true", "false"]
+
+
+def validate_execpath(value):
+    return os.path.isfile(value) and os.access(value, os.X_OK)
+
+
+def validate_filepath(value):
+    return os.path.isfile(value)
+
+
+def validate_path(value):
+    return os.path.exists(value)
+
+
+def to_int(value):
+    return int(value)
+
+
+def to_float(value):
+    return float(value)
+
+
+def to_bool(value):
+    if isinstance(value, bool):
+        return value
+    return True if value in ["true", "True"] else False
+
+
+def get(name, default_value=None):
+    return _gconf.get(name, default_value)
+
+
+def getall(show_defaults=False, show_non_configurable=False):
+    return _gconf.getall(show_defaults=show_defaults,
+                         show_non_configurable=show_non_configurable)
+
+
+def getr(name, default_value=None):
+    return _gconf.getr(name, default_value)
+
+
+def load(default_conf, custom_conf=None, args={}, extra_tmpl_args={},
+         override_from_args=False):
+    global _gconf
+    _gconf = Gconf(default_conf, custom_conf, args, extra_tmpl_args,
+                   override_from_args)
+
+
+def setconfig(name, value):
+    global _gconf
+    _gconf.set(name, value)
+
+
+def resetconfig(name):
+    global _gconf
+    _gconf.reset(name)
+
+
+def check(name, value=None, with_conffile=True):
+    global _gconf
+    _gconf.check(name, value=value, with_conffile=with_conffile)
diff --git a/geo-replication/syncdaemon/gsyncdstatus.py b/geo-replication/syncdaemon/gsyncdstatus.py
new file mode 100644
index 00000000000..1a655ff8887
--- /dev/null
+++ b/geo-replication/syncdaemon/gsyncdstatus.py
@@ -0,0 +1,419 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+import fcntl
+import os
+import tempfile
+try:
+    import urllib.parse as urllib
+except ImportError:
+    import urllib
+import json
+import time
+from datetime import datetime
+from errno import EACCES, EAGAIN, ENOENT
+import logging
+
+from syncdutils import (EVENT_GEOREP_ACTIVE, EVENT_GEOREP_PASSIVE, gf_event,
+                        EVENT_GEOREP_CHECKPOINT_COMPLETED, lf)
+
+DEFAULT_STATUS = "N/A"
+MONITOR_STATUS = ("Created", "Started", "Paused", "Stopped")
+STATUS_VALUES = (DEFAULT_STATUS,
+                 "Initializing...",
+                 "Active",
+                 "Passive",
+                 "Faulty")
+
+CRAWL_STATUS_VALUES = (DEFAULT_STATUS,
+                       "Hybrid Crawl",
+                       "History Crawl",
+                       "Changelog Crawl")
+
+
+def human_time(ts):
+    try:
+        return datetime.fromtimestamp(float(ts)).strftime("%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return DEFAULT_STATUS
+
+
+def human_time_utc(ts):
+    try:
+        return datetime.utcfromtimestamp(
+            float(ts)).strftime("%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return DEFAULT_STATUS
+
+
+def get_default_values():
+    return {
+        "slave_node": DEFAULT_STATUS,
+        "worker_status": DEFAULT_STATUS,
+        "last_synced": 0,
+        "last_synced_entry": 0,
+        "crawl_status": DEFAULT_STATUS,
+        "entry": 0,
+        "data": 0,
+        "meta": 0,
+        "failures": 0,
+        "checkpoint_completed": DEFAULT_STATUS,
+        "checkpoint_time": 0,
+        "checkpoint_completion_time": 0}
+
+
+class LockedOpen(object):
+
+    def __init__(self, filename, *args, **kwargs):
+        self.filename = filename
+        self.open_args = args
+        self.open_kwargs = kwargs
+        self.fileobj = None
+
+    def __enter__(self):
+        """
+        If two processes compete to update a file, The first process
+        gets the lock and the second process is blocked in the fcntl.flock()
+        call. When first process replaces the file and releases the lock,
+        the already open file descriptor in the second process now points
+        to a  "ghost" file(not reachable by any path name) with old contents.
+        To avoid that conflict, check the fd already opened is same or
+        not. Open new one if not same
+        """
+        f = open(self.filename, *self.open_args, **self.open_kwargs)
+        while True:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            fnew = open(self.filename, *self.open_args, **self.open_kwargs)
+            if os.path.sameopenfile(f.fileno(), fnew.fileno()):
+                fnew.close()
+                break
+            else:
+                f.close()
+                f = fnew
+        self.fileobj = f
+        return f
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        fcntl.flock(self.fileobj, fcntl.LOCK_UN)
+        self.fileobj.close()
+
+
+def set_monitor_status(status_file, status):
+    fd = os.open(status_file, os.O_CREAT | os.O_RDWR)
+    os.close(fd)
+    with LockedOpen(status_file, 'r+'):
+        with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(status_file),
+                                         delete=False) as tf:
+            tf.write(status)
+            tempname = tf.name
+
+        os.rename(tempname, status_file)
+        dirfd = os.open(os.path.dirname(os.path.abspath(status_file)),
+                        os.O_DIRECTORY)
+        os.fsync(dirfd)
+        os.close(dirfd)
+
+
+class GeorepStatus(object):
+    def __init__(self, monitor_status_file, master_node, brick, master_node_id,
+                 master, slave, monitor_pid_file=None):
+        self.master = master
+        slv_data = slave.split("::")
+        self.slave_host = slv_data[0]
+        self.slave_volume = slv_data[1].split(":")[0]  # Remove Slave UUID
+        self.work_dir = os.path.dirname(monitor_status_file)
+        self.monitor_status_file = monitor_status_file
+        self.filename = os.path.join(self.work_dir,
+                                     "brick_%s.status"
+                                     % urllib.quote_plus(brick))
+
+        fd = os.open(self.filename, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+        fd = os.open(self.monitor_status_file, os.O_CREAT | os.O_RDWR)
+        os.close(fd)
+        self.master_node = master_node
+        self.master_node_id = master_node_id
+        self.brick = brick
+        self.default_values = get_default_values()
+        self.monitor_pid_file = monitor_pid_file
+
+    def send_event(self, event_type, **kwargs):
+        gf_event(event_type,
+                 master_volume=self.master,
+                 master_node=self.master_node,
+                 master_node_id=self.master_node_id,
+                 slave_host=self.slave_host,
+                 slave_volume=self.slave_volume,
+                 brick_path=self.brick,
+                 **kwargs)
+
+    def _update(self, mergerfunc):
+        data = self.default_values
+        with LockedOpen(self.filename, 'r+') as f:
+            try:
+                data.update(json.load(f))
+            except ValueError:
+                pass
+
+            data = mergerfunc(data)
+            # If Data is not changed by merger func
+            if not data:
+                return False
+
+            with tempfile.NamedTemporaryFile(
+                    'w',
+                    dir=os.path.dirname(self.filename),
+                    delete=False) as tf:
+                tf.write(data)
+                tempname = tf.name
+
+            os.rename(tempname, self.filename)
+            dirfd = os.open(os.path.dirname(os.path.abspath(self.filename)),
+                            os.O_DIRECTORY)
+            os.fsync(dirfd)
+            os.close(dirfd)
+            return True
+
+    def reset_on_worker_start(self):
+        def merger(data):
+            data["slave_node"] = DEFAULT_STATUS
+            data["crawl_status"] = DEFAULT_STATUS
+            data["entry"] = 0
+            data["data"] = 0
+            data["meta"] = 0
+            return json.dumps(data)
+
+        self._update(merger)
+
+    def set_field(self, key, value):
+        def merger(data):
+            # Current data and prev data is same
+            if data[key] == value:
+                return {}
+
+            data[key] = value
+            return json.dumps(data)
+
+        return self._update(merger)
+
+    def trigger_gf_event_checkpoint_completion(self, checkpoint_time,
+                                               checkpoint_completion_time):
+        self.send_event(EVENT_GEOREP_CHECKPOINT_COMPLETED,
+                        checkpoint_time=checkpoint_time,
+                        checkpoint_completion_time=checkpoint_completion_time)
+
+    def set_last_synced(self, value, checkpoint_time):
+        def merger(data):
+            data["last_synced"] = value[0]
+
+            # If checkpoint is not set or reset
+            # or if last set checkpoint is changed
+            if checkpoint_time == 0 or \
+               checkpoint_time != data["checkpoint_time"]:
+                data["checkpoint_time"] = 0
+                data["checkpoint_completion_time"] = 0
+                data["checkpoint_completed"] = "No"
+
+            # If checkpoint is completed and not marked as completed
+            # previously then update the checkpoint completed time
+            if checkpoint_time > 0 and checkpoint_time <= value[0]:
+                if data["checkpoint_completed"] == "No":
+                    curr_time = int(time.time())
+                    data["checkpoint_time"] = checkpoint_time
+                    data["checkpoint_completion_time"] = curr_time
+                    data["checkpoint_completed"] = "Yes"
+                    logging.info(lf("Checkpoint completed",
+                                    checkpoint_time=human_time_utc(
+                                        checkpoint_time),
+                                    completion_time=human_time_utc(curr_time)))
+                    self.trigger_gf_event_checkpoint_completion(
+                        checkpoint_time, curr_time)
+
+            return json.dumps(data)
+
+        self._update(merger)
+
+    def set_worker_status(self, status):
+        if self.set_field("worker_status", status):
+            logging.info(lf("Worker Status Change",
+                            status=status))
+
+    def set_worker_crawl_status(self, status):
+        if self.set_field("crawl_status", status):
+            logging.info(lf("Crawl Status Change",
+                            status=status))
+
+    def set_slave_node(self, slave_node):
+        def merger(data):
+            data["slave_node"] = slave_node
+            return json.dumps(data)
+
+        self._update(merger)
+
+    def inc_value(self, key, value):
+        def merger(data):
+            data[key] = data.get(key, 0) + value
+            return json.dumps(data)
+
+        self._update(merger)
+
+    def dec_value(self, key, value):
+        def merger(data):
+            data[key] = data.get(key, 0) - value
+            if data[key] < 0:
+                data[key] = 0
+            return json.dumps(data)
+
+        self._update(merger)
+
+    def set_active(self):
+        if self.set_field("worker_status", "Active"):
+            logging.info(lf("Worker Status Change",
+                            status="Active"))
+            self.send_event(EVENT_GEOREP_ACTIVE)
+
+    def set_passive(self):
+        if self.set_field("worker_status", "Passive"):
+            logging.info(lf("Worker Status Change",
+                            status="Passive"))
+            self.send_event(EVENT_GEOREP_PASSIVE)
+
+    def get_monitor_status(self):
+        data = ""
+        with open(self.monitor_status_file, "r") as f:
+            data = f.read().strip()
+        return data
+
+    def get_status(self, checkpoint_time=0):
+        """
+        Monitor Status --->        Created    Started  Paused      Stopped
+        ----------------------------------------------------------------------
+        slave_node                 N/A        VALUE    VALUE       N/A
+        status                     Created    VALUE    Paused      Stopped
+        last_synced                N/A        VALUE    VALUE       VALUE
+        last_synced_entry          N/A        VALUE    VALUE       VALUE
+        crawl_status               N/A        VALUE    N/A         N/A
+        entry                      N/A        VALUE    N/A         N/A
+        data                       N/A        VALUE    N/A         N/A
+        meta                       N/A        VALUE    N/A         N/A
+        failures                   N/A        VALUE    VALUE       VALUE
+        checkpoint_completed       N/A        VALUE    VALUE       VALUE
+        checkpoint_time            N/A        VALUE    VALUE       VALUE
+        checkpoint_completed_time  N/A        VALUE    VALUE       VALUE
+        """
+        data = self.default_values
+        with open(self.filename) as f:
+            try:
+                data.update(json.load(f))
+            except ValueError:
+                pass
+        monitor_status = self.get_monitor_status()
+
+        # Verifying whether monitor process running and adjusting status
+        if monitor_status in ["Started", "Paused"]:
+            try:
+                with open(self.monitor_pid_file, "r+") as f:
+                    fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    monitor_status = "Stopped"
+            except (IOError, OSError) as e:
+                # If pid file not exists, either monitor died or Geo-rep
+                # not even started once
+                if e.errno == ENOENT:
+                    monitor_status = "Stopped"
+                elif e.errno in (EACCES, EAGAIN):
+                    # cannot grab. so, monitor process still running..move on
+                    pass
+                else:
+                    raise
+
+        if monitor_status in ["Created", "Paused", "Stopped"]:
+            data["worker_status"] = monitor_status
+
+        if monitor_status == "":
+            data["worker_status"] = "Stopped"
+
+        # Checkpoint adjustments
+        if checkpoint_time == 0:
+            data["checkpoint_completed"] = DEFAULT_STATUS
+            data["checkpoint_time"] = DEFAULT_STATUS
+            data["checkpoint_completion_time"] = DEFAULT_STATUS
+        else:
+            if checkpoint_time != data["checkpoint_time"]:
+                if checkpoint_time <= data["last_synced"]:
+                    data["checkpoint_completed"] = "Yes"
+                    data["checkpoint_time"] = checkpoint_time
+                    data["checkpoint_completion_time"] = data["last_synced"]
+                else:
+                    data["checkpoint_completed"] = "No"
+                    data["checkpoint_time"] = checkpoint_time
+                    data["checkpoint_completion_time"] = DEFAULT_STATUS
+
+        if data["checkpoint_time"] not in [0, DEFAULT_STATUS]:
+            chkpt_time = data["checkpoint_time"]
+            data["checkpoint_time"] = human_time(chkpt_time)
+            data["checkpoint_time_utc"] = human_time_utc(chkpt_time)
+
+        if data["checkpoint_completion_time"] not in [0, DEFAULT_STATUS]:
+            chkpt_completion_time = data["checkpoint_completion_time"]
+            data["checkpoint_completion_time"] = human_time(
+                chkpt_completion_time)
+            data["checkpoint_completion_time_utc"] = human_time_utc(
+                chkpt_completion_time)
+
+        if data["last_synced"] == 0:
+            data["last_synced"] = DEFAULT_STATUS
+            data["last_synced_utc"] = DEFAULT_STATUS
+        else:
+            last_synced = data["last_synced"]
+            data["last_synced"] = human_time(last_synced)
+            data["last_synced_utc"] = human_time_utc(last_synced)
+
+        if data["worker_status"] != "Active":
+            data["last_synced"] = DEFAULT_STATUS
+            data["last_synced_utc"] = DEFAULT_STATUS
+            data["crawl_status"] = DEFAULT_STATUS
+            data["entry"] = DEFAULT_STATUS
+            data["data"] = DEFAULT_STATUS
+            data["meta"] = DEFAULT_STATUS
+            data["failures"] = DEFAULT_STATUS
+            data["checkpoint_completed"] = DEFAULT_STATUS
+            data["checkpoint_time"] = DEFAULT_STATUS
+            data["checkpoint_completed_time"] = DEFAULT_STATUS
+            data["checkpoint_time_utc"] = DEFAULT_STATUS
+            data["checkpoint_completion_time_utc"] = DEFAULT_STATUS
+
+        if data["worker_status"] not in ["Active", "Passive"]:
+            data["slave_node"] = DEFAULT_STATUS
+
+        if data.get("last_synced_utc", 0) == 0:
+            data["last_synced_utc"] = DEFAULT_STATUS
+
+        if data.get("checkpoint_completion_time_utc", 0) == 0:
+            data["checkpoint_completion_time_utc"] = DEFAULT_STATUS
+
+        if data.get("checkpoint_time_utc", 0) == 0:
+            data["checkpoint_time_utc"] = DEFAULT_STATUS
+
+        return data
+
+    def print_status(self, checkpoint_time=0, json_output=False):
+        status_out = self.get_status(checkpoint_time)
+        if json_output:
+            out = {}
+            # Convert all values as string
+            for k, v in status_out.items():
+                out[k] = str(v)
+            print(json.dumps(out))
+            return
+
+        for key, value in status_out.items():
+            print(("%s: %s" % (key, value)))
diff --git a/geo-replication/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py
new file mode 100644
index 00000000000..e6406c36bd7
--- /dev/null
+++ b/geo-replication/syncdaemon/libcxattr.py
@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+from ctypes import CDLL, get_errno
+from py2py3 import (bytearray_to_str, gr_create_string_buffer,
+                    gr_query_xattr, gr_lsetxattr, gr_lremovexattr)
+
+
+class Xattr(object):
+
+    """singleton that wraps the extended attributes system
+       interface for python using ctypes
+
+       Just implement it to the degree we need it, in particular
+       - we need just the l*xattr variants, ie. we never want symlinks to be
+         followed
+       - don't need size discovery for getxattr, as we always know the exact
+         sizes we expect
+    """
+
+    libc = CDLL("libc.so.6", use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _query_xattr(cls, path, siz, syscall, *a):
+        if siz:
+            buf = gr_create_string_buffer(siz)
+        else:
+            buf = None
+        ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
+        if ret == -1:
+            cls.raise_oserr()
+        if siz:
+            # py2 and py3 compatibility. Convert bytes array
+            # to string
+            result = bytearray_to_str(buf.raw)
+            return result[:ret]
+        else:
+            return ret
+
+    @classmethod
+    def lgetxattr(cls, path, attr, siz=0):
+        return gr_query_xattr(cls, path, siz, 'lgetxattr', attr)
+
+    @classmethod
+    def lgetxattr_buf(cls, path, attr):
+        """lgetxattr variant with size discovery"""
+        size = cls.lgetxattr(path, attr)
+        if size == -1:
+            cls.raise_oserr()
+        if size == 0:
+            return ''
+        return cls.lgetxattr(path, attr, size)
+
+    @classmethod
+    def llistxattr(cls, path, siz=0):
+        ret = gr_query_xattr(cls, path, siz, 'llistxattr')
+        if isinstance(ret, str):
+            ret = ret.strip('\0')
+            ret = ret.split('\0') if ret else []
+        return ret
+
+    @classmethod
+    def lsetxattr(cls, path, attr, val):
+        ret = gr_lsetxattr(cls, path, attr, val)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def lremovexattr(cls, path, attr):
+        ret = gr_lremovexattr(cls, path, attr)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def llistxattr_buf(cls, path):
+        """listxattr variant with size discovery"""
+        try:
+            # Assuming no more than 100 xattrs in a file/directory and
+            # each xattr key length will be less than 256 bytes
+            # llistxattr will be called with bigger size so that
+            # listxattr will not fail with ERANGE. OSError will be
+            # raised if fails even with the large size specified.
+            size = 256 * 100
+            return cls.llistxattr(path, size)
+        except OSError:
+            # If fixed length failed for getting list of xattrs then
+            # use the llistxattr call to get the size and use that
+            # size to get the list of xattrs.
+            size = cls.llistxattr(path)
+            if size == -1:
+                cls.raise_oserr()
+            if size == 0:
+                return []
+
+            return cls.llistxattr(path, size)
diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py
new file mode 100644
index 00000000000..a3bda7282c0
--- /dev/null
+++ b/geo-replication/syncdaemon/libgfchangelog.py
@@ -0,0 +1,143 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+from ctypes import CDLL, RTLD_GLOBAL, get_errno, byref, c_ulong
+from ctypes.util import find_library
+from syncdutils import ChangelogException, ChangelogHistoryNotAvailable
+from py2py3 import (gr_cl_history_changelog, gr_cl_done,
+                    gr_create_string_buffer, gr_cl_register,
+                    gr_cl_history_done, bytearray_to_str)
+
+
+libgfc = CDLL(
+    find_library("gfchangelog"),
+    mode=RTLD_GLOBAL,
+    use_errno=True
+)
+
+
+def _raise_changelog_err():
+    errn = get_errno()
+    raise ChangelogException(errn, os.strerror(errn))
+
+
+def _init():
+    if libgfc.gf_changelog_init(None) == -1:
+        _raise_changelog_err()
+
+
+def register(brick, path, log_file, log_level, retries=0):
+    _init()
+
+    ret = gr_cl_register(libgfc, brick, path, log_file, log_level, retries)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def scan():
+    ret = libgfc.gf_changelog_scan()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def startfresh():
+    ret = libgfc.gf_changelog_start_fresh()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def getchanges():
+    def clsort(cfile):
+        return cfile.split('.')[-1]
+
+    changes = []
+    buf = gr_create_string_buffer(4096)
+    call = libgfc.gf_changelog_next_change
+
+    while True:
+        ret = call(buf, 4096)
+        if ret in (0, -1):
+            break
+
+        # py2 and py3 compatibility
+        result = bytearray_to_str(buf.raw[:ret - 1])
+        changes.append(result)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+    # cleanup tracker
+    startfresh()
+
+    return sorted(changes, key=clsort)
+
+
+def done(clfile):
+    ret = gr_cl_done(libgfc, clfile)
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def history_scan():
+    ret = libgfc.gf_history_changelog_scan()
+    if ret == -1:
+        _raise_changelog_err()
+
+    return ret
+
+
+def history_changelog(changelog_path, start, end, num_parallel):
+    actual_end = c_ulong()
+    ret = gr_cl_history_changelog(libgfc, changelog_path, start, end,
+                                  num_parallel, byref(actual_end))
+    if ret == -1:
+        _raise_changelog_err()
+
+    if ret == -2:
+        raise ChangelogHistoryNotAvailable()
+
+    return (ret, actual_end.value)
+
+
+def history_startfresh():
+    ret = libgfc.gf_history_changelog_start_fresh()
+    if ret == -1:
+        _raise_changelog_err()
+
+
+def history_getchanges():
+    def clsort(cfile):
+        return cfile.split('.')[-1]
+
+    changes = []
+    buf = gr_create_string_buffer(4096)
+    call = libgfc.gf_history_changelog_next_change
+
+    while True:
+        ret = call(buf, 4096)
+        if ret in (0, -1):
+            break
+
+        # py2 and py3 compatibility
+        result = bytearray_to_str(buf.raw[:ret - 1])
+        changes.append(result)
+
+    if ret == -1:
+        _raise_changelog_err()
+
+    return sorted(changes, key=clsort)
+
+
+def history_done(clfile):
+    ret = gr_cl_history_done(libgfc, clfile)
+    if ret == -1:
+        _raise_changelog_err()
diff --git a/geo-replication/syncdaemon/logutils.py b/geo-replication/syncdaemon/logutils.py
new file mode 100644
index 00000000000..01ae7852f23
--- /dev/null
+++ b/geo-replication/syncdaemon/logutils.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+import logging
+from logging import Logger, handlers
+import sys
+import time
+
+
+class GLogger(Logger):
+
+    """Logger customizations for gsyncd.
+
+    It implements a log format similar to that of glusterfs.
+    """
+
+    def makeRecord(self, name, level, *a):
+        rv = Logger.makeRecord(self, name, level, *a)
+        rv.nsecs = (rv.created - int(rv.created)) * 1000000
+        fr = sys._getframe(4)
+        callee = fr.f_locals.get('self')
+        if callee:
+            ctx = str(type(callee)).split("'")[1].split('.')[-1]
+        else:
+            ctx = '<top>'
+        if not hasattr(rv, 'funcName'):
+            rv.funcName = fr.f_code.co_name
+        rv.lvlnam = logging.getLevelName(level)[0]
+        rv.ctx = ctx
+        return rv
+
+
+LOGFMT = ("[%(asctime)s.%(nsecs)d] %(lvlnam)s [%(module)s{0}"
+          ":%(lineno)s:%(funcName)s] %(ctx)s: %(message)s")
+
+
+def setup_logging(level="INFO", label="", log_file=""):
+    if label:
+        label = "(" + label + ")"
+
+    filename = None
+    stream = None
+    if log_file:
+        if log_file in ('-', '/dev/stderr'):
+            stream = sys.stderr
+        elif log_file == '/dev/stdout':
+            stream = sys.stdout
+        else:
+            filename = log_file
+
+    datefmt = "%Y-%m-%d %H:%M:%S"
+    fmt = LOGFMT.format(label)
+    logging.root = GLogger("root", level)
+    logging.setLoggerClass(GLogger)
+    logging.Formatter.converter = time.gmtime  # Log in GMT/UTC time
+    logging.getLogger().handlers = []
+    logging.getLogger().setLevel(level)
+
+    if filename is not None:
+        logging_handler = handlers.WatchedFileHandler(filename)
+        formatter = logging.Formatter(fmt=fmt,
+                                      datefmt=datefmt)
+        logging_handler.setFormatter(formatter)
+        logging.getLogger().addHandler(logging_handler)
+    else:
+        logging.basicConfig(stream=stream,
+                            format=fmt,
+                            datefmt=datefmt,
+                            level=level)
diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py
new file mode 100644
index 00000000000..9501aeae6b5
--- /dev/null
+++ b/geo-replication/syncdaemon/master.py
@@ -0,0 +1,2020 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+import time
+import stat
+import logging
+import fcntl
+import string
+import errno
+import tarfile
+from errno import ENOENT, ENODATA, EEXIST, EACCES, EAGAIN, ESTALE, EINTR
+from threading import Condition, Lock
+from datetime import datetime
+
+import gsyncdconfig as gconf
+import libgfchangelog
+from rconf import rconf
+from syncdutils import (Thread, GsyncdError, escape_space_newline,
+                        unescape_space_newline, gauxpfx, escape,
+                        lstat, errno_wrap, FreeObject, lf, matching_disk_gfid,
+                        NoStimeAvailable, PartialHistoryAvailable,
+                        host_brick_split)
+
+URXTIME = (-1, 0)
+
+# Default rollover time set in changelog translator
+# changelog rollover time is hardcoded here to avoid the
+# xsync usage when crawling switch happens from history
+# to changelog. If rollover time increased in translator
+# then geo-rep can enter into xsync crawl after history
+# crawl before starting live changelog crawl.
+CHANGELOG_ROLLOVER_TIME = 15
+
+# Utility functions to help us to get to closer proximity
+# of the DRY principle (no, don't look for elevated or
+# perspectivistic things here)
+
+
+def _xtime_now():
+    t = time.time()
+    sec = int(t)
+    nsec = int((t - sec) * 1000000)
+    return (sec, nsec)
+
+
+def _volinfo_hook_relax_foreign(self):
+    volinfo_sys = self.get_sys_volinfo()
+    fgn_vi = volinfo_sys[self.KFGN]
+    if fgn_vi:
+        expiry = fgn_vi['timeout'] - int(time.time()) + 1
+        logging.info(lf('foreign volume info found, waiting for expiry',
+                        expiry=expiry))
+        time.sleep(expiry)
+        volinfo_sys = self.get_sys_volinfo()
+    return volinfo_sys
+
+
+def edct(op, **ed):
+    dct = {}
+    dct['op'] = op
+    # This is used in automatic gfid conflict resolution.
+    # When marked True, it's skipped during re-processing.
+    dct['skip_entry'] = False
+    for k in ed:
+        if k == 'stat':
+            st = ed[k]
+            dst = dct['stat'] = {}
+            if st:
+                dst['uid'] = st.st_uid
+                dst['gid'] = st.st_gid
+                dst['mode'] = st.st_mode
+                dst['atime'] = st.st_atime
+                dst['mtime'] = st.st_mtime
+        else:
+            dct[k] = ed[k]
+    return dct
+
+
+# The API!
+
+def gmaster_builder(excrawl=None):
+    """produce the GMaster class variant corresponding
+       to sync mode"""
+    this = sys.modules[__name__]
+    modemixin = gconf.get("special-sync-mode")
+    if not modemixin:
+        modemixin = 'normal'
+
+    if gconf.get("change-detector") == 'xsync':
+        changemixin = 'xsync'
+    elif excrawl:
+        changemixin = excrawl
+    else:
+        changemixin = gconf.get("change-detector")
+
+    logging.debug(lf('setting up change detection mode',
+                     mode=changemixin))
+    modemixin = getattr(this, modemixin.capitalize() + 'Mixin')
+    crawlmixin = getattr(this, 'GMaster' + changemixin.capitalize() + 'Mixin')
+
+    if gconf.get("use-rsync-xattrs"):
+        sendmarkmixin = SendmarkRsyncMixin
+    else:
+        sendmarkmixin = SendmarkNormalMixin
+
+    if gconf.get("ignore-deletes"):
+        purgemixin = PurgeNoopMixin
+    else:
+        purgemixin = PurgeNormalMixin
+
+    if gconf.get("sync-method") == "tarssh":
+        syncengine = TarSSHEngine
+    else:
+        syncengine = RsyncEngine
+
+    class _GMaster(crawlmixin, modemixin, sendmarkmixin,
+                   purgemixin, syncengine):
+        pass
+
+    return _GMaster
+
+
+# Mixin classes that implement the data format
+# and logic particularities of the certain
+# sync modes
+
+class NormalMixin(object):
+
+    """normal geo-rep behavior"""
+
+    minus_infinity = URXTIME
+
+    # following staticmethods ideally would be
+    # methods of an xtime object (in particular,
+    # implementing the hooks needed for comparison
+    # operators), but at this point we don't yet
+    # have a dedicated xtime class
+
+    @staticmethod
+    def serialize_xtime(xt):
+        return "%d.%d" % tuple(xt)
+
+    @staticmethod
+    def deserialize_xtime(xt):
+        return tuple(int(x) for x in xt.split("."))
+
+    @staticmethod
+    def native_xtime(xt):
+        return xt
+
+    @staticmethod
+    def xtime_geq(xt0, xt1):
+        return xt0 >= xt1
+
+    def make_xtime_opts(self, is_master, opts):
+        if 'create' not in opts:
+            opts['create'] = is_master
+        if 'default_xtime' not in opts:
+            opts['default_xtime'] = URXTIME
+
+    def xtime_low(self, rsc, path, **opts):
+        if rsc == self.master:
+            xt = rsc.server.xtime(path, self.uuid)
+        else:
+            xt = rsc.server.stime(path, self.uuid)
+            if isinstance(xt, int) and xt == ENODATA:
+                xt = rsc.server.xtime(path, self.uuid)
+                if not isinstance(xt, int):
+                    self.slave.server.set_stime(path, self.uuid, xt)
+        if isinstance(xt, int) and xt != ENODATA:
+            return xt
+        if xt == ENODATA or xt < self.volmark:
+            if opts['create']:
+                xt = _xtime_now()
+                rsc.server.aggregated.set_xtime(path, self.uuid, xt)
+            else:
+                zero_zero = (0, 0)
+                if xt != zero_zero:
+                    xt = opts['default_xtime']
+        return xt
+
+    def keepalive_payload_hook(self, timo, gap):
+        # first grab a reference as self.volinfo
+        # can be changed in main thread
+        vi = self.volinfo
+        if vi:
+            # then have a private copy which we can mod
+            vi = vi.copy()
+            vi['timeout'] = int(time.time()) + timo
+        else:
+            # send keep-alive more frequently to
+            # avoid a delay in announcing our volume info
+            # to slave if it becomes established in the
+            # meantime
+            gap = min(10, gap)
+        return (vi, gap)
+
+    def volinfo_hook(self):
+        return self.get_sys_volinfo()
+
+    def xtime_reversion_hook(self, path, xtl, xtr):
+        if xtr > xtl:
+            raise GsyncdError("timestamp corruption for " + path)
+
+    def need_sync(self, e, xte, xtrd):
+        return xte > xtrd
+
+    def set_slave_xtime(self, path, mark):
+        self.slave.server.set_stime(path, self.uuid, mark)
+        # self.slave.server.set_xtime_remote(path, self.uuid, mark)
+
+
+class PartialMixin(NormalMixin):
+
+    """a variant tuned towards operation with a master
+       that has partial info of the slave (brick typically)"""
+
+    def xtime_reversion_hook(self, path, xtl, xtr):
+        pass
+
+
+class RecoverMixin(NormalMixin):
+
+    """a variant that differs from normal in terms
+       of ignoring non-indexed files"""
+
+    @staticmethod
+    def make_xtime_opts(is_master, opts):
+        if 'create' not in opts:
+            opts['create'] = False
+        if 'default_xtime' not in opts:
+            opts['default_xtime'] = URXTIME
+
+    def keepalive_payload_hook(self, timo, gap):
+        return (None, gap)
+
+    def volinfo_hook(self):
+        return _volinfo_hook_relax_foreign(self)
+
+# Further mixins for certain tunable behaviors
+
+
+class SendmarkNormalMixin(object):
+
+    def sendmark_regular(self, *a, **kw):
+        return self.sendmark(*a, **kw)
+
+
+class SendmarkRsyncMixin(object):
+
+    def sendmark_regular(self, *a, **kw):
+        pass
+
+
+class PurgeNormalMixin(object):
+
+    def purge_missing(self, path, names):
+        self.slave.server.purge(path, names)
+
+
+class PurgeNoopMixin(object):
+
+    def purge_missing(self, path, names):
+        pass
+
+
+class TarSSHEngine(object):
+
+    """Sync engine that uses tar(1) piped over ssh(1)
+       for data transfers. Good for lots of small files.
+    """
+
+    def a_syncdata(self, files):
+        logging.debug(lf("Files", files=files))
+
+        for f in files:
+            pb = self.syncer.add(f)
+
+            def regjob(se, xte, pb):
+                rv = pb.wait()
+                if rv[0]:
+                    logging.debug(lf('synced', file=se))
+                    return True
+                else:
+                    # stat check for file presence
+                    st = lstat(se)
+                    if isinstance(st, int):
+                        # file got unlinked in the interim
+                        self.unlinked_gfids.add(se)
+                        return True
+
+            self.add_job(self.FLAT_DIR_HIERARCHY, 'reg', regjob, f, None, pb)
+
+    def syncdata_wait(self):
+        if self.wait(self.FLAT_DIR_HIERARCHY, None):
+            return True
+
+    def syncdata(self, files):
+        self.a_syncdata(files)
+        self.syncdata_wait()
+
+
+class RsyncEngine(object):
+
+    """Sync engine that uses rsync(1) for data transfers"""
+
+    def a_syncdata(self, files):
+        logging.debug(lf("files", files=files))
+
+        for f in files:
+            logging.debug(lf('candidate for syncing', file=f))
+            pb = self.syncer.add(f)
+
+            def regjob(se, xte, pb):
+                rv = pb.wait()
+                if rv[0]:
+                    logging.debug(lf('synced', file=se))
+                    return True
+                else:
+                    # stat to check if the file exist
+                    st = lstat(se)
+                    if isinstance(st, int):
+                        # file got unlinked in the interim
+                        self.unlinked_gfids.add(se)
+                        return True
+
+            self.add_job(self.FLAT_DIR_HIERARCHY, 'reg', regjob, f, None, pb)
+
+    def syncdata_wait(self):
+        if self.wait(self.FLAT_DIR_HIERARCHY, None):
+            return True
+
+    def syncdata(self, files):
+        self.a_syncdata(files)
+        self.syncdata_wait()
+
+
+class GMasterCommon(object):
+
+    """abstract class impementling master role"""
+
+    KFGN = 0
+    KNAT = 1
+
+    def get_sys_volinfo(self):
+        """query volume marks on fs root
+
+        err out on multiple foreign masters
+        """
+        fgn_vis, nat_vi = (
+            self.master.server.aggregated.foreign_volume_infos(),
+            self.master.server.aggregated.native_volume_info())
+        fgn_vi = None
+        if fgn_vis:
+            if len(fgn_vis) > 1:
+                raise GsyncdError("cannot work with multiple foreign masters")
+            fgn_vi = fgn_vis[0]
+        return fgn_vi, nat_vi
+
+    @property
+    def uuid(self):
+        if self.volinfo:
+            return self.volinfo['uuid']
+
+    @property
+    def volmark(self):
+        if self.volinfo:
+            return self.volinfo['volume_mark']
+
+    def get_entry_stime(self):
+        data = self.slave.server.entry_stime(".", self.uuid)
+        if isinstance(data, int):
+            data = None
+        return data
+
+    def get_data_stime(self):
+        data = self.slave.server.stime(".", self.uuid)
+        if isinstance(data, int):
+            data = None
+        return data
+
+    def xtime(self, path, *a, **opts):
+        """get amended xtime
+
+        as of amending, we can create missing xtime, or
+        determine a valid value if what we get is expired
+        (as of the volume mark expiry); way of amendig
+        depends on @opts and on subject of query (master
+        or slave).
+        """
+        if a:
+            rsc = a[0]
+        else:
+            rsc = self.master
+        self.make_xtime_opts(rsc == self.master, opts)
+        return self.xtime_low(rsc, path, **opts)
+
+    def __init__(self, master, slave):
+        self.master = master
+        self.slave = slave
+        self.jobtab = {}
+        if gconf.get("sync-method") == "tarssh":
+            self.syncer = Syncer(slave, self.slave.tarssh, [2])
+        else:
+            # partial transfer (cf. rsync(1)), that's normal
+            self.syncer = Syncer(slave, self.slave.rsync, [23, 24])
+        # crawls vs. turns:
+        # - self.crawls is simply the number of crawl() invocations on root
+        # - one turn is a maximal consecutive sequence of crawls so that each
+        #   crawl in it detects a change to be synced
+        # - self.turns is the number of turns since start
+        # - self.total_turns is a limit so that if self.turns reaches it, then
+        #   we exit (for diagnostic purposes)
+        # so, eg., if the master fs changes unceasingly, self.turns will remain
+        # 0.
+        self.crawls = 0
+        self.turns = 0
+        self.total_turns = rconf.turns
+        self.crawl_start = datetime.now()
+        self.lastreport = {'crawls': 0, 'turns': 0, 'time': 0}
+        self.start = None
+        self.change_seen = None
+        # the actual volinfo we make use of
+        self.volinfo = None
+        self.terminate = False
+        self.sleep_interval = 1
+        self.unlinked_gfids = set()
+
+    def init_keep_alive(cls):
+        """start the keep-alive thread """
+        timo = gconf.get("slave-timeout", 0)
+        if timo > 0:
+            def keep_alive():
+                while True:
+                    vi, gap = cls.keepalive_payload_hook(timo, timo * 0.5)
+                    cls.slave.server.keep_alive(vi)
+                    time.sleep(gap)
+            t = Thread(target=keep_alive)
+            t.start()
+
+    def mgmt_lock(self):
+
+        """Take management volume lock """
+        if rconf.mgmt_lock_fd:
+            try:
+                fcntl.lockf(rconf.mgmt_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                return True
+            except:
+                ex = sys.exc_info()[1]
+                if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
+                    return False
+                raise
+
+        fd = None
+        bname = str(self.uuid) + "_" + rconf.args.slave_id + "_subvol_" \
+            + str(rconf.args.subvol_num) + ".lock"
+        mgmt_lock_dir = os.path.join(gconf.get("meta-volume-mnt"), "geo-rep")
+        path = os.path.join(mgmt_lock_dir, bname)
+        logging.debug(lf("lock file path", path=path))
+        try:
+            fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno == ENOENT:
+                logging.info("Creating geo-rep directory in meta volume...")
+                try:
+                    os.makedirs(mgmt_lock_dir)
+                except OSError:
+                    ex = sys.exc_info()[1]
+                    if ex.errno == EEXIST:
+                        pass
+                    else:
+                        raise
+                fd = os.open(path, os.O_CREAT | os.O_RDWR)
+            else:
+                raise
+        try:
+            fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # Save latest FD for future use
+            rconf.mgmt_lock_fd = fd
+        except:
+            ex = sys.exc_info()[1]
+            if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
+                # cannot grab, it's taken
+                rconf.mgmt_lock_fd = fd
+                return False
+            raise
+
+        return True
+
+    def should_crawl(self):
+        if not gconf.get("use-meta-volume"):
+            return rconf.args.local_node_id in self.master.server.node_uuid()
+
+        if not os.path.ismount(gconf.get("meta-volume-mnt")):
+            logging.error("Meta-volume is not mounted. Worker Exiting...")
+            sys.exit(1)
+        return self.mgmt_lock()
+
+    def register(self):
+        self.register()
+
+    def crawlwrap(self, oneshot=False, register_time=None):
+        if oneshot:
+            # it's important to do this during the oneshot crawl as
+            # for a passive gsyncd (ie. in a replicate scenario)
+            # the keepalive thread would keep the connection alive.
+            self.init_keep_alive()
+
+        # If crawlwrap is called when partial history available,
+        # then it sets register_time which is the time when geo-rep
+        # worker registered to changelog consumption. Since nsec is
+        # not considered in register time, there are chances of skipping
+        # changes detection in xsync crawl. This limit will be reset when
+        # crawlwrap is called again.
+        self.live_changelog_start_time = None
+        if register_time:
+            self.live_changelog_start_time = (register_time, 0)
+
+        # no need to maintain volinfo state machine.
+        # in a cascading setup, each geo-replication session is
+        # independent (ie. 'volume-mark' and 'xtime' are not
+        # propagated). This is because the slave's xtime is now
+        # stored on the master itself. 'volume-mark' just identifies
+        # that we are in a cascading setup and need to enable
+        # 'geo-replication.ignore-pid-check' option.
+        volinfo_sys = self.volinfo_hook()
+        self.volinfo = volinfo_sys[self.KNAT]
+        inter_master = volinfo_sys[self.KFGN]
+        logging.debug("%s master with volume id %s ..." %
+                      (inter_master and "intermediate" or "primary",
+                       self.uuid))
+        rconf.volume_id = self.uuid
+        if self.volinfo:
+            if self.volinfo['retval']:
+                logging.warn(lf("master cluster's info may not be valid",
+                                error=self.volinfo['retval']))
+        else:
+            raise GsyncdError("master volinfo unavailable")
+        self.lastreport['time'] = time.time()
+
+        t0 = time.time()
+        crawl = self.should_crawl()
+        while not self.terminate:
+            if self.start:
+                logging.debug("... crawl #%d done, took %.6f seconds" %
+                              (self.crawls, time.time() - self.start))
+            self.start = time.time()
+            should_display_info = self.start - self.lastreport['time'] >= 60
+            if should_display_info:
+                logging.debug("%d crawls, %d turns",
+                              self.crawls - self.lastreport['crawls'],
+                              self.turns - self.lastreport['turns'])
+                self.lastreport.update(crawls=self.crawls,
+                                       turns=self.turns,
+                                       time=self.start)
+            t1 = time.time()
+            if int(t1 - t0) >= gconf.get("replica-failover-interval"):
+                crawl = self.should_crawl()
+                t0 = t1
+            self.update_worker_remote_node()
+            if not crawl:
+                self.status.set_passive()
+                # bring up _this_ brick to the cluster stime
+                # which is min of cluster (but max of the replicas)
+                brick_stime = self.xtime('.', self.slave)
+                cluster_stime = self.master.server.aggregated.stime_mnt(
+                    '.', '.'.join([str(self.uuid), rconf.args.slave_id]))
+                logging.debug(lf("Crawl info",
+                                 cluster_stime=cluster_stime,
+                                 brick_stime=brick_stime))
+
+                if not isinstance(cluster_stime, int):
+                    if brick_stime < cluster_stime:
+                        self.slave.server.set_stime(
+                            self.FLAT_DIR_HIERARCHY, self.uuid, cluster_stime)
+                        self.upd_stime(cluster_stime)
+                        # Purge all changelogs available in processing dir
+                        # less than cluster_stime
+                        proc_dir = os.path.join(self.tempdir,
+                                                ".processing")
+
+                        if os.path.exists(proc_dir):
+                            to_purge = [f for f in os.listdir(proc_dir)
+                                        if (f.startswith("CHANGELOG.") and
+                                            int(f.split('.')[-1]) <
+                                            cluster_stime[0])]
+                            for f in to_purge:
+                                os.remove(os.path.join(proc_dir, f))
+
+                time.sleep(5)
+                continue
+
+            self.status.set_active()
+            self.crawl()
+
+            if oneshot:
+                return
+            time.sleep(self.sleep_interval)
+
+    @staticmethod
+    def humantime(*tpair):
+        """format xtime-like (sec, nsec) pair to human readable format"""
+        ts = datetime.fromtimestamp(float('.'.join(str(n) for n in tpair))).\
+            strftime("%Y-%m-%d %H:%M:%S")
+        if len(tpair) > 1:
+            ts += '.' + str(tpair[1])
+        return ts
+
+    def _crawl_time_format(self, crawl_time):
+        # Ex: 5 years, 4 days, 20:23:10
+        years, days = divmod(crawl_time.days, 365.25)
+        years = int(years)
+        days = int(days)
+
+        date = ""
+        m, s = divmod(crawl_time.seconds, 60)
+        h, m = divmod(m, 60)
+
+        if years != 0:
+            date += "%s %s " % (years, "year" if years == 1 else "years")
+        if days != 0:
+            date += "%s %s " % (days, "day" if days == 1 else "days")
+
+        date += "%s:%s:%s" % (string.zfill(h, 2),
+                              string.zfill(m, 2), string.zfill(s, 2))
+        return date
+
+    def add_job(self, path, label, job, *a, **kw):
+        """insert @job function to job table at @path with @label"""
+        if self.jobtab.get(path) is None:
+            self.jobtab[path] = []
+        self.jobtab[path].append((label, a, lambda: job(*a, **kw)))
+
+    def add_failjob(self, path, label):
+        """invoke .add_job with a job that does nothing just fails"""
+        logging.debug('salvaged: ' + label)
+        self.add_job(path, label, lambda: False)
+
+    def wait(self, path, *args):
+        """perform jobs registered for @path
+
+        Reset jobtab entry for @path,
+        determine success as the conjunction of
+        success of all the jobs. In case of
+        success, call .sendmark on @path
+        """
+        jobs = self.jobtab.pop(path, [])
+        succeed = True
+        for j in jobs:
+            ret = j[-1]()
+            if not ret:
+                succeed = False
+        if succeed and not args[0] is None:
+            self.sendmark(path, *args)
+        return succeed
+
+    def sendmark(self, path, mark, adct=None):
+        """update slave side xtime for @path to master side xtime
+
+        also can send a setattr payload (see Server.setattr).
+        """
+        if adct:
+            self.slave.server.setattr(path, adct)
+        self.set_slave_xtime(path, mark)
+
+
+class XCrawlMetadata(object):
+    def __init__(self, st_uid, st_gid, st_mode, st_atime, st_mtime):
+        self.st_uid = int(st_uid)
+        self.st_gid = int(st_gid)
+        self.st_mode = int(st_mode)
+        self.st_atime = float(st_atime)
+        self.st_mtime = float(st_mtime)
+
+
+class GMasterChangelogMixin(GMasterCommon):
+
+    """ changelog based change detection and syncing """
+
+    # index for change type and entry
+    IDX_START = 0
+    IDX_END = 2
+    UNLINK_ENTRY = 2
+
+    POS_GFID = 0
+    POS_TYPE = 1
+    POS_ENTRY1 = -1
+
+    TYPE_META = "M "
+    TYPE_GFID = "D "
+    TYPE_ENTRY = "E "
+
+    MAX_EF_RETRIES = 10
+    MAX_OE_RETRIES = 10
+
+    # flat directory hierarchy for gfid based access
+    FLAT_DIR_HIERARCHY = '.'
+
+    CHANGELOG_CONN_RETRIES = 5
+
+    def init_fop_batch_stats(self):
+        self.batch_stats = {
+            "CREATE": 0,
+            "MKNOD": 0,
+            "UNLINK": 0,
+            "MKDIR": 0,
+            "RMDIR": 0,
+            "LINK": 0,
+            "SYMLINK": 0,
+            "RENAME": 0,
+            "SETATTR": 0,
+            "SETXATTR": 0,
+            "XATTROP": 0,
+            "DATA": 0,
+            "ENTRY_SYNC_TIME": 0,
+            "META_SYNC_TIME": 0,
+            "DATA_START_TIME": 0
+        }
+
+    def update_fop_batch_stats(self, ty):
+        if ty in ['FSETXATTR']:
+            ty = 'SETXATTR'
+        self.batch_stats[ty] = self.batch_stats.get(ty, 0) + 1
+
+    def archive_and_purge_changelogs(self, changelogs):
+        # Creates tar file instead of tar.gz, since changelogs will
+        # be appended to existing tar. archive name is
+        # archive_<YEAR><MONTH>.tar
+        archive_name = "archive_%s.tar" % datetime.today().strftime(
+            gconf.get("changelog-archive-format"))
+
+        try:
+            tar = tarfile.open(os.path.join(self.processed_changelogs_dir,
+                                            archive_name),
+                               "a")
+        except tarfile.ReadError:
+            tar = tarfile.open(os.path.join(self.processed_changelogs_dir,
+                                            archive_name),
+                               "w")
+
+        for f in changelogs:
+            try:
+                f = os.path.basename(f)
+                tar.add(os.path.join(self.processed_changelogs_dir, f),
+                        arcname=os.path.basename(f))
+            except:
+                exc = sys.exc_info()[1]
+                if ((isinstance(exc, OSError) or
+                     isinstance(exc, IOError)) and exc.errno == ENOENT):
+                    continue
+                else:
+                    tar.close()
+                    raise
+        tar.close()
+
+        for f in changelogs:
+            try:
+                f = os.path.basename(f)
+                os.remove(os.path.join(self.processed_changelogs_dir, f))
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    continue
+                else:
+                    raise
+
+    def setup_working_dir(self):
+        workdir = os.path.join(gconf.get("working-dir"),
+                               escape(rconf.args.local_path))
+        logging.debug('changelog working dir %s' % workdir)
+        return workdir
+
+    def log_failures(self, failures, entry_key, gfid_prefix, log_prefix):
+        num_failures = 0
+        for failure in failures:
+            st = lstat(os.path.join(gfid_prefix, failure[0][entry_key]))
+            if not isinstance(st, int):
+                num_failures += 1
+                logging.error(lf('%s FAILED' % log_prefix,
+                                 data=failure))
+                if failure[0]['op'] == 'MKDIR':
+                    raise GsyncdError("The above directory failed to sync."
+                                      " Please fix it to proceed further.")
+
+        self.status.inc_value("failures", num_failures)
+
+    def fix_possible_entry_failures(self, failures, retry_count, entries):
+        pfx = gauxpfx()
+        fix_entry_ops = []
+        failures1 = []
+        remove_gfids = set()
+        for failure in failures:
+            if failure[2]['name_mismatch']:
+                pbname = failure[2]['slave_entry']
+            elif failure[2]['dst']:
+                pbname = failure[0]['entry1']
+            else:
+                pbname = failure[0]['entry']
+
+            op = failure[0]['op']
+            # name exists but gfid is different
+            if failure[2]['gfid_mismatch'] or failure[2]['name_mismatch']:
+                slave_gfid = failure[2]['slave_gfid']
+                st = lstat(os.path.join(pfx, slave_gfid))
+                # Takes care of scenarios with no hardlinks
+                if isinstance(st, int) and st == ENOENT:
+                    logging.debug(lf('Entry not present on master. Fixing gfid '
+                                    'mismatch in slave. Deleting the entry',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    # Add deletion to fix_entry_ops list
+                    if failure[2]['slave_isdir']:
+                        fix_entry_ops.append(
+                            edct('RMDIR',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+                    else:
+                        fix_entry_ops.append(
+                            edct('UNLINK',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+                    remove_gfids.add(slave_gfid)
+                    if op in ['RENAME']:
+                        # If renamed gfid doesn't exists on master, remove
+                        # rename entry and unlink src on slave
+                        st = lstat(os.path.join(pfx, failure[0]['gfid']))
+                        if isinstance(st, int) and st == ENOENT:
+                            logging.debug("Unlink source %s" % repr(failure))
+                            remove_gfids.add(failure[0]['gfid'])
+                            fix_entry_ops.append(
+                                edct('UNLINK',
+                                     gfid=failure[0]['gfid'],
+                                     entry=failure[0]['entry']))
+                # Takes care of scenarios of hardlinks/renames on master
+                elif not isinstance(st, int):
+                    if matching_disk_gfid(slave_gfid, pbname):
+                        # Safe to ignore the failure as master contains same
+                        # file with same gfid. Remove entry from entries list
+                        logging.debug(lf('Fixing gfid mismatch in slave. '
+                                        ' Safe to ignore, take out entry',
+                                        retry_count=retry_count,
+                                        entry=repr(failure)))
+                        remove_gfids.add(failure[0]['gfid'])
+                        if op == 'RENAME':
+                            fix_entry_ops.append(
+                                edct('UNLINK',
+                                     gfid=failure[0]['gfid'],
+                                     entry=failure[0]['entry']))
+                    # The file exists on master but with different name.
+                    # Probably renamed and got missed during xsync crawl.
+                    elif failure[2]['slave_isdir']:
+                        realpath = os.readlink(os.path.join(
+                                               rconf.args.local_path,
+                                               ".glusterfs",
+                                               slave_gfid[0:2],
+                                               slave_gfid[2:4],
+                                               slave_gfid))
+                        dst_entry = os.path.join(pfx, realpath.split('/')[-2],
+                                                 realpath.split('/')[-1])
+                        src_entry = pbname
+                        logging.debug(lf('Fixing dir name/gfid mismatch in '
+                                        'slave', retry_count=retry_count,
+                                        entry=repr(failure)))
+                        if src_entry == dst_entry:
+                            # Safe to ignore the failure as master contains
+                            # same directory as in slave with same gfid.
+                            # Remove the failure entry from entries list
+                            logging.debug(lf('Fixing dir name/gfid mismatch'
+                                            ' in slave. Safe to ignore, '
+                                            'take out entry',
+                                            retry_count=retry_count,
+                                            entry=repr(failure)))
+                            try:
+                                entries.remove(failure[0])
+                            except ValueError:
+                                pass
+                        else:
+                            rename_dict = edct('RENAME', gfid=slave_gfid,
+                                               entry=src_entry,
+                                               entry1=dst_entry, stat=st,
+                                               link=None)
+                            logging.debug(lf('Fixing dir name/gfid mismatch'
+                                            ' in slave. Renaming',
+                                            retry_count=retry_count,
+                                            entry=repr(rename_dict)))
+                            fix_entry_ops.append(rename_dict)
+                    else:
+                        # A hardlink file exists with different name or
+                        # renamed file exists and we are sure from
+                        # matching_disk_gfid check that the entry doesn't
+                        # exist with same gfid so we can safely delete on slave
+                        logging.debug(lf('Fixing file gfid mismatch in slave. '
+                                        'Hardlink/Rename Case. Deleting entry',
+                                        retry_count=retry_count,
+                                        entry=repr(failure)))
+                        fix_entry_ops.append(
+                            edct('UNLINK',
+                                 gfid=failure[2]['slave_gfid'],
+                                 entry=pbname))
+            elif failure[1] == ENOENT:
+                if op in ['RENAME']:
+                    pbname = failure[0]['entry1']
+                else:
+                    pbname = failure[0]['entry']
+
+                pargfid = pbname.split('/')[1]
+                st = lstat(os.path.join(pfx, pargfid))
+                # Safe to ignore the failure as master doesn't contain
+                # parent directory.
+                if isinstance(st, int):
+                    logging.debug(lf('Fixing ENOENT error in slave. Parent '
+                                    'does not exist on master. Safe to '
+                                    'ignore, take out entry',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    try:
+                        entries.remove(failure[0])
+                    except ValueError:
+                        pass
+                else:
+                    logging.debug(lf('Fixing ENOENT error in slave. Create '
+                                    'parent directory on slave.',
+                                    retry_count=retry_count,
+                                    entry=repr(failure)))
+                    realpath = os.readlink(os.path.join(rconf.args.local_path,
+                                                        ".glusterfs",
+                                                        pargfid[0:2],
+                                                        pargfid[2:4],
+                                                        pargfid))
+                    dir_entry = os.path.join(pfx, realpath.split('/')[-2],
+                                             realpath.split('/')[-1])
+                    fix_entry_ops.append(
+                        edct('MKDIR', gfid=pargfid, entry=dir_entry,
+                             mode=st.st_mode, uid=st.st_uid, gid=st.st_gid))
+
+        logging.debug("remove_gfids: %s" % repr(remove_gfids))
+        if remove_gfids:
+            for e in entries:
+                if e['op'] in ['MKDIR', 'MKNOD', 'CREATE', 'RENAME'] \
+                   and e['gfid'] in remove_gfids:
+                    logging.debug("Removed entry op from retrial list: entry: %s" % repr(e))
+                    e['skip_entry'] = True
+
+        if fix_entry_ops:
+            # Process deletions of entries whose gfids are mismatched
+            failures1 = self.slave.server.entry_ops(fix_entry_ops)
+
+        return (failures1, fix_entry_ops)
+
+    def handle_entry_failures(self, failures, entries):
+        retries = 0
+        pending_failures = False
+        failures1 = []
+        failures2 = []
+        entry_ops1 = []
+        entry_ops2 = []
+
+        if failures:
+            pending_failures = True
+            failures1 = failures
+            entry_ops1 = entries
+
+            while pending_failures and retries < self.MAX_EF_RETRIES:
+                retries += 1
+                (failures2, entry_ops2) = self.fix_possible_entry_failures(
+                    failures1, retries, entry_ops1)
+                if not failures2:
+                    pending_failures = False
+                    logging.info(lf('Successfully fixed entry ops with gfid '
+                                 'mismatch', retry_count=retries))
+                else:
+                    pending_failures = True
+                    failures1 = failures2
+                    entry_ops1 = entry_ops2
+
+            if pending_failures:
+                for failure in failures1:
+                    logging.error("Failed to fix entry ops %s", repr(failure))
+
+    def process_change(self, change, done, retry):
+        pfx = gauxpfx()
+        clist = []
+        entries = []
+        meta_gfid = set()
+        datas = set()
+
+        change_ts = change.split(".")[-1]
+
+        # Ignore entry ops which are already processed in Changelog modes
+        ignore_entry_ops = False
+        entry_stime = None
+        data_stime = None
+        if self.name in ["live_changelog", "history_changelog"]:
+            entry_stime = self.get_entry_stime()
+            data_stime = self.get_data_stime()
+
+        if entry_stime is not None and data_stime is not None:
+            # if entry_stime is not None but data_stime > entry_stime
+            # This situation is caused by the stime update of Passive worker
+            # Consider data_stime in this case.
+            if data_stime[0] > entry_stime[0]:
+                entry_stime = data_stime
+
+            # Compare the entry_stime with changelog file suffix
+            # if changelog time is less than entry_stime then ignore
+            if int(change_ts) <= entry_stime[0]:
+                ignore_entry_ops = True
+
+        try:
+            f = open(change, "r")
+            clist = f.readlines()
+            f.close()
+        except IOError:
+            raise
+
+        for e in clist:
+            e = e.strip()
+            et = e[self.IDX_START:self.IDX_END]   # entry type
+            ec = e[self.IDX_END:].split(' ')      # rest of the bits
+
+            # skip ENTRY operation if hot tier brick
+            if self.name == 'live_changelog' or \
+               self.name == 'history_changelog':
+                if rconf.args.is_hottier and et == self.TYPE_ENTRY:
+                    logging.debug(lf('skip ENTRY op if hot tier brick',
+                                     op=ec[self.POS_TYPE]))
+                    continue
+
+            # Data and Meta operations are decided while parsing
+            # UNLINK/RMDIR/MKNOD except that case ignore all the other
+            # entry ops if ignore_entry_ops is True.
+            # UNLINK/RMDIR/MKNOD entry_ops are ignored in the end
+            if ignore_entry_ops and et == self.TYPE_ENTRY and \
+               ec[self.POS_TYPE] not in ["UNLINK", "RMDIR", "MKNOD"]:
+                continue
+
+            if et == self.TYPE_ENTRY:
+                # extract information according to the type of
+                # the entry operation. create(), mkdir() and mknod()
+                # have mode, uid, gid information in the changelog
+                # itself, so no need to stat()...
+                ty = ec[self.POS_TYPE]
+
+                self.update_fop_batch_stats(ec[self.POS_TYPE])
+
+                # PARGFID/BNAME
+                en = unescape_space_newline(
+                    os.path.join(pfx, ec[self.POS_ENTRY1]))
+                # GFID of the entry
+                gfid = ec[self.POS_GFID]
+
+                if ty in ['UNLINK', 'RMDIR']:
+                    # The index of PARGFID/BNAME for UNLINK, RMDIR
+                    # is no more the last index. It varies based on
+                    # changelog.capture-del-path is enabled or not.
+                    en = unescape_space_newline(
+                        os.path.join(pfx, ec[self.UNLINK_ENTRY]))
+
+                    # Remove from DATA list, so that rsync will
+                    # not fail
+                    pt = os.path.join(pfx, ec[0])
+                    st = lstat(pt)
+                    if pt in datas and isinstance(st, int):
+                        # file got unlinked, May be historical Changelog
+                        datas.remove(pt)
+
+                    if ty in ['RMDIR'] and not isinstance(st, int):
+                        logging.info(lf('Ignoring rmdir. Directory present in '
+                                        'master', gfid=gfid, pgfid_bname=en))
+                        continue
+
+                    if not gconf.get("ignore-deletes"):
+                        if not ignore_entry_ops:
+                            entries.append(edct(ty, gfid=gfid, entry=en))
+                elif ty in ['CREATE', 'MKDIR', 'MKNOD']:
+                    # Special case: record mknod as link
+                    if ty in ['MKNOD']:
+                        mode = int(ec[2])
+                        if mode & 0o1000:
+                                # Avoid stat'ing the file as it
+                                # may be deleted in the interim
+                                st = FreeObject(st_mode=int(ec[2]),
+                                                st_uid=int(ec[3]),
+                                                st_gid=int(ec[4]),
+                                                st_atime=0,
+                                                st_mtime=0)
+
+                                # So, it may be deleted, but still we are
+                                # append LINK? Because, the file will be
+                                # CREATED if source not exists.
+                                entries.append(edct('LINK', stat=st, entry=en,
+                                               gfid=gfid))
+
+                                # Here, we have the assumption that only
+                                # tier-gfid.linkto causes this mknod. Add data
+                                datas.add(os.path.join(pfx, ec[0]))
+                                continue
+
+                    # stat info. present in the changelog itself
+                    entries.append(edct(ty, gfid=gfid, entry=en,
+                                   mode=int(ec[2]),
+                                   uid=int(ec[3]), gid=int(ec[4])))
+                elif ty == "RENAME":
+                    go = os.path.join(pfx, gfid)
+                    st = lstat(go)
+                    if isinstance(st, int):
+                        st = {}
+
+                    rl = None
+                    if st and stat.S_ISLNK(st.st_mode):
+                        rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                        [ESTALE, EINTR])
+                        if isinstance(rl, int):
+                            rl = None
+
+                    e1 = unescape_space_newline(
+                        os.path.join(pfx, ec[self.POS_ENTRY1 - 1]))
+                    entries.append(edct(ty, gfid=gfid, entry=e1, entry1=en,
+                                        stat=st, link=rl))
+                    # If src doesn't exist while doing rename, destination
+                    # is created. If data is not followed by rename, this
+                    # remains zero byte file on slave. Hence add data entry
+                    # for renames
+                    datas.add(os.path.join(pfx, gfid))
+                else:
+                    # stat() to get mode and other information
+                    if not matching_disk_gfid(gfid, en):
+                        logging.debug(lf('Ignoring entry, purged in the '
+                                      'interim', file=en, gfid=gfid))
+                        continue
+
+                    go = os.path.join(pfx, gfid)
+                    st = lstat(go)
+                    if isinstance(st, int):
+                        logging.debug(lf('Ignoring entry, purged in the '
+                                      'interim', file=en, gfid=gfid))
+                        continue
+
+                    if ty == 'LINK':
+                        rl = None
+                        if st and stat.S_ISLNK(st.st_mode):
+                            rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                            [ESTALE, EINTR])
+                            if isinstance(rl, int):
+                                rl = None
+                        entries.append(edct(ty, stat=st, entry=en, gfid=gfid,
+                                       link=rl))
+                        # If src doesn't exist while doing link, destination
+                        # is created based on file type. If data is not
+                        # followed by link, this remains zero byte file on
+                        # slave. Hence add data entry for links
+                        if rl is None:
+                            datas.add(os.path.join(pfx, gfid))
+                    elif ty == 'SYMLINK':
+                        rl = errno_wrap(os.readlink, [en], [ENOENT],
+                                        [ESTALE, EINTR])
+                        if isinstance(rl, int):
+                            continue
+
+                        entries.append(
+                            edct(ty, stat=st, entry=en, gfid=gfid, link=rl))
+                    else:
+                        logging.warn(lf('ignoring op',
+                                        gfid=gfid,
+                                        type=ty))
+            elif et == self.TYPE_GFID:
+                # If self.unlinked_gfids is available, then that means it is
+                # retrying the changelog second time. Do not add the GFID's
+                # to rsync job if failed previously but unlinked in master
+                if self.unlinked_gfids and \
+                   os.path.join(pfx, ec[0]) in self.unlinked_gfids:
+                    logging.debug("ignoring data, since file purged interim")
+                else:
+                    datas.add(os.path.join(pfx, ec[0]))
+            elif et == self.TYPE_META:
+                self.update_fop_batch_stats(ec[self.POS_TYPE])
+                if ec[1] == 'SETATTR':  # only setattr's for now...
+                    if len(ec) == 5:
+                        # In xsync crawl, we already have stat data
+                        # avoid doing stat again
+                        meta_gfid.add((os.path.join(pfx, ec[0]),
+                                       XCrawlMetadata(st_uid=ec[2],
+                                                      st_gid=ec[3],
+                                                      st_mode=ec[4],
+                                                      st_atime=ec[5],
+                                                      st_mtime=ec[6])))
+                    else:
+                        meta_gfid.add((os.path.join(pfx, ec[0]), ))
+                elif ec[1] in ['SETXATTR', 'XATTROP', 'FXATTROP']:
+                    # To sync xattr/acls use rsync/tar, --xattrs and --acls
+                    # switch to rsync and tar
+                    if not gconf.get("sync-method") == "tarssh" and \
+                       (gconf.get("sync-xattrs") or gconf.get("sync-acls")):
+                        datas.add(os.path.join(pfx, ec[0]))
+            else:
+                logging.warn(lf('got invalid fop type',
+                                type=et))
+        logging.debug('entries: %s' % repr(entries))
+
+        # Increment counters for Status
+        self.files_in_batch += len(datas)
+        self.status.inc_value("data", len(datas))
+
+        self.batch_stats["DATA"] += self.files_in_batch - \
+            self.batch_stats["SETXATTR"] - \
+            self.batch_stats["XATTROP"]
+
+        entry_start_time = time.time()
+        # sync namespace
+        if entries and not ignore_entry_ops:
+            # Increment counters for Status
+            self.status.inc_value("entry", len(entries))
+
+            failures = self.slave.server.entry_ops(entries)
+
+            if gconf.get("gfid-conflict-resolution"):
+                count = 0
+                if failures:
+                    logging.info(lf('Entry ops failed with gfid mismatch',
+                                count=len(failures)))
+                while failures and count < self.MAX_OE_RETRIES:
+                    count += 1
+                    self.handle_entry_failures(failures, entries)
+                    logging.info(lf('Retry original entries', count=count))
+                    failures = self.slave.server.entry_ops(entries)
+                    if not failures:
+                        logging.info("Successfully fixed all entry ops with "
+                                     "gfid mismatch")
+                        break
+
+            self.log_failures(failures, 'gfid', gauxpfx(), 'ENTRY')
+            self.status.dec_value("entry", len(entries))
+
+            # Update Entry stime in Brick Root only in case of Changelog mode
+            if self.name in ["live_changelog", "history_changelog"]:
+                entry_stime_to_update = (int(change_ts) - 1, 0)
+                self.upd_entry_stime(entry_stime_to_update)
+                self.status.set_field("last_synced_entry",
+                                      entry_stime_to_update[0])
+
+        self.batch_stats["ENTRY_SYNC_TIME"] += time.time() - entry_start_time
+
+        if ignore_entry_ops:
+            # Book keeping, to show in logs the range of Changelogs skipped
+            self.num_skipped_entry_changelogs += 1
+            if self.skipped_entry_changelogs_first is None:
+                self.skipped_entry_changelogs_first = change_ts
+
+            self.skipped_entry_changelogs_last = change_ts
+
+        meta_start_time = time.time()
+        # sync metadata
+        if meta_gfid:
+            meta_entries = []
+            for go in meta_gfid:
+                if len(go) > 1:
+                    st = go[1]
+                else:
+                    st = lstat(go[0])
+                if isinstance(st, int):
+                    logging.debug(lf('file got purged in the interim',
+                                     file=go[0]))
+                    continue
+                meta_entries.append(edct('META', go=go[0], stat=st))
+            if meta_entries:
+                self.status.inc_value("meta", len(meta_entries))
+                failures = self.slave.server.meta_ops(meta_entries)
+                self.log_failures(failures, 'go', '', 'META')
+                self.status.dec_value("meta", len(meta_entries))
+
+        self.batch_stats["META_SYNC_TIME"] += time.time() - meta_start_time
+
+        if self.batch_stats["DATA_START_TIME"] == 0:
+            self.batch_stats["DATA_START_TIME"] = time.time()
+
+        # sync data
+        if datas:
+            self.a_syncdata(datas)
+            self.datas_in_batch.update(datas)
+
+    def process(self, changes, done=1):
+        tries = 0
+        retry = False
+        self.unlinked_gfids = set()
+        self.files_in_batch = 0
+        self.datas_in_batch = set()
+        # Error log disabled till the last round
+        self.syncer.disable_errorlog()
+        self.skipped_entry_changelogs_first = None
+        self.skipped_entry_changelogs_last = None
+        self.num_skipped_entry_changelogs = 0
+        self.batch_start_time = time.time()
+        self.init_fop_batch_stats()
+
+        while True:
+            # first, fire all changelog transfers in parallel. entry and
+            # metadata are performed synchronously, therefore in serial.
+            # However at the end of each changelog, data is synchronized
+            # with syncdata_async() - which means it is serial w.r.t
+            # entries/metadata of that changelog but happens in parallel
+            # with data of other changelogs.
+
+            if retry:
+                if tries == (gconf.get("max-rsync-retries") - 1):
+                    # Enable Error logging if it is last retry
+                    self.syncer.enable_errorlog()
+
+                # Remove Unlinked GFIDs from Queue
+                for unlinked_gfid in self.unlinked_gfids:
+                    if unlinked_gfid in self.datas_in_batch:
+                        self.datas_in_batch.remove(unlinked_gfid)
+
+                # Retry only Sync. Do not retry entry ops
+                if self.datas_in_batch:
+                    self.a_syncdata(self.datas_in_batch)
+            else:
+                for change in changes:
+                    logging.debug(lf('processing change',
+                                     changelog=change))
+                    self.process_change(change, done, retry)
+                    if not retry:
+                        # number of changelogs processed in the batch
+                        self.turns += 1
+
+            # Now we wait for all the data transfers fired off in the above
+            # step to complete. Note that this is not ideal either. Ideally
+            # we want to trigger the entry/meta-data transfer of the next
+            # batch while waiting for the data transfer of the current batch
+            # to finish.
+
+            # Note that the reason to wait for the data transfer (vs doing it
+            # completely in the background and call the changelog_done()
+            # asynchronously) is because this waiting acts as a "backpressure"
+            # and prevents a spiraling increase of wait stubs from consuming
+            # unbounded memory and resources.
+
+            # update the slave's time with the timestamp of the _last_
+            # changelog file time suffix. Since, the changelog prefix time
+            # is the time when the changelog was rolled over, introduce a
+            # tolerance of 1 second to counter the small delta b/w the
+            # marker update and gettimeofday().
+            # NOTE: this is only for changelog mode, not xsync.
+
+            # @change is the last changelog (therefore max time for this batch)
+            if self.syncdata_wait():
+                self.unlinked_gfids = set()
+                if done:
+                    xtl = (int(change.split('.')[-1]) - 1, 0)
+                    self.upd_stime(xtl)
+                    list(map(self.changelog_done_func, changes))
+                    self.archive_and_purge_changelogs(changes)
+
+                # Reset Data counter after sync
+                self.status.dec_value("data", self.files_in_batch)
+                self.files_in_batch = 0
+                self.datas_in_batch = set()
+                break
+
+            # We do not know which changelog transfer failed, retry everything.
+            retry = True
+            tries += 1
+            if tries == gconf.get("max-rsync-retries"):
+                logging.error(lf('changelogs could not be processed '
+                                 'completely - moving on...',
+                                 files=list(map(os.path.basename, changes))))
+
+                # Reset data counter on failure
+                self.status.dec_value("data", self.files_in_batch)
+                self.files_in_batch = 0
+                self.datas_in_batch = set()
+
+                if done:
+                    xtl = (int(change.split('.')[-1]) - 1, 0)
+                    self.upd_stime(xtl)
+                    list(map(self.changelog_done_func, changes))
+                    self.archive_and_purge_changelogs(changes)
+                break
+            # it's either entry_ops() or Rsync that failed to do it's
+            # job. Mostly it's entry_ops() [which currently has a problem
+            # of failing to create an entry but failing to return an errno]
+            # Therefore we do not know if it's either Rsync or the freaking
+            # entry_ops() that failed... so we retry the _whole_ changelog
+            # again.
+            # TODO: remove entry retries when it's gets fixed.
+            logging.warn(lf('incomplete sync, retrying changelogs',
+                            files=list(map(os.path.basename, changes))))
+
+            # Reset the Data counter before Retry
+            self.status.dec_value("data", self.files_in_batch)
+            self.files_in_batch = 0
+            self.init_fop_batch_stats()
+            time.sleep(0.5)
+
+        # Log the Skipped Entry ops range if any
+        if self.skipped_entry_changelogs_first is not None and \
+           self.skipped_entry_changelogs_last is not None:
+            logging.info(lf("Skipping already processed entry ops",
+                            from_changelog=self.skipped_entry_changelogs_first,
+                            to_changelog=self.skipped_entry_changelogs_last,
+                            num_changelogs=self.num_skipped_entry_changelogs))
+
+        # Log Current batch details
+        if changes:
+            logging.info(
+                lf("Entry Time Taken",
+                   UNL=self.batch_stats["UNLINK"],
+                   RMD=self.batch_stats["RMDIR"],
+                   CRE=self.batch_stats["CREATE"],
+                   MKN=self.batch_stats["MKNOD"],
+                   MKD=self.batch_stats["MKDIR"],
+                   REN=self.batch_stats["RENAME"],
+                   LIN=self.batch_stats["LINK"],
+                   SYM=self.batch_stats["SYMLINK"],
+                   duration="%.4f" % self.batch_stats["ENTRY_SYNC_TIME"]))
+
+            logging.info(
+                lf("Data/Metadata Time Taken",
+                   SETA=self.batch_stats["SETATTR"],
+                   meta_duration="%.4f" % self.batch_stats["META_SYNC_TIME"],
+                   SETX=self.batch_stats["SETXATTR"],
+                   XATT=self.batch_stats["XATTROP"],
+                   DATA=self.batch_stats["DATA"],
+                   data_duration="%.4f" % (
+                       time.time() - self.batch_stats["DATA_START_TIME"])))
+
+            logging.info(
+                lf("Batch Completed",
+                   mode=self.name,
+                   duration="%.4f" % (time.time() - self.batch_start_time),
+                   changelog_start=changes[0].split(".")[-1],
+                   changelog_end=changes[-1].split(".")[-1],
+                   num_changelogs=len(changes),
+                   stime=self.get_data_stime(),
+                   entry_stime=self.get_entry_stime()))
+
+    def upd_entry_stime(self, stime):
+        self.slave.server.set_entry_stime(self.FLAT_DIR_HIERARCHY,
+                                          self.uuid,
+                                          stime)
+
+    def upd_stime(self, stime, path=None):
+        if not path:
+            path = self.FLAT_DIR_HIERARCHY
+        if not stime == URXTIME:
+            self.sendmark(path, stime)
+
+        # Update last_synced_time in status file based on stime
+        # only update stime if stime xattr set to Brick root
+        if path == self.FLAT_DIR_HIERARCHY:
+            chkpt_time = gconf.getr("checkpoint")
+            checkpoint_time = 0
+            if chkpt_time is not None:
+                checkpoint_time = int(chkpt_time)
+
+            self.status.set_last_synced(stime, checkpoint_time)
+
+    def update_worker_remote_node(self):
+        node = rconf.args.resource_remote
+        node_data = node.split("@")
+        node = node_data[-1]
+        remote_node_ip, _ = host_brick_split(node)
+        self.status.set_slave_node(remote_node_ip)
+
+    def changelogs_batch_process(self, changes):
+        changelogs_batches = []
+        current_size = 0
+        for c in changes:
+            si = os.lstat(c).st_size
+            if (si + current_size) > gconf.get("changelog-batch-size"):
+                # Create new batch if single Changelog file greater than
+                # Max Size! or current batch size exceeds Max size
+                changelogs_batches.append([c])
+                current_size = si
+            else:
+                # Append to last batch, if No batches available Create one
+                current_size += si
+                if not changelogs_batches:
+                    changelogs_batches.append([c])
+                else:
+                    changelogs_batches[-1].append(c)
+
+        for batch in changelogs_batches:
+            logging.debug(lf('processing changes',
+                             batch=batch))
+            self.process(batch)
+
+    def crawl(self):
+        self.status.set_worker_crawl_status("Changelog Crawl")
+        changes = []
+        # get stime (from the brick) and purge changelogs
+        # that are _historical_ to that time.
+        data_stime = self.get_data_stime()
+
+        libgfchangelog.scan()
+        self.crawls += 1
+        changes = libgfchangelog.getchanges()
+        if changes:
+            if data_stime:
+                logging.info(lf("slave's time",
+                                stime=data_stime))
+                processed = [x for x in changes
+                             if int(x.split('.')[-1]) < data_stime[0]]
+                for pr in processed:
+                    logging.debug(
+                        lf('skipping already processed change',
+                           changelog=os.path.basename(pr)))
+                    self.changelog_done_func(pr)
+                    changes.remove(pr)
+                self.archive_and_purge_changelogs(processed)
+
+        self.changelogs_batch_process(changes)
+
+    def register(self, register_time, status):
+        self.sleep_interval = gconf.get("change-interval")
+        self.changelog_done_func = libgfchangelog.done
+        self.tempdir = self.setup_working_dir()
+        self.processed_changelogs_dir = os.path.join(self.tempdir,
+                                                     ".processed")
+        self.name = "live_changelog"
+        self.status = status
+
+
+class GMasterChangeloghistoryMixin(GMasterChangelogMixin):
+    def register(self, register_time, status):
+        self.changelog_register_time = register_time
+        self.history_crawl_start_time = register_time
+        self.changelog_done_func = libgfchangelog.history_done
+        self.history_turns = 0
+        self.tempdir = self.setup_working_dir()
+        self.processed_changelogs_dir = os.path.join(self.tempdir,
+                                                     ".history/.processed")
+        self.name = "history_changelog"
+        self.status = status
+
+    def crawl(self):
+        self.history_turns += 1
+        self.status.set_worker_crawl_status("History Crawl")
+        data_stime = self.get_data_stime()
+
+        end_time = int(time.time())
+
+        #as start of historical crawl marks Geo-rep worker restart
+        if gconf.get("ignore-deletes"):
+            logging.info(lf('ignore-deletes config option is set',
+                         stime=data_stime))
+
+        logging.info(lf('starting history crawl',
+                        turns=self.history_turns,
+                        stime=data_stime,
+                        etime=end_time,
+                        entry_stime=self.get_entry_stime()))
+
+        if not data_stime or data_stime == URXTIME:
+            raise NoStimeAvailable()
+
+        # Changelogs backend path is hardcoded as
+        # <BRICK_PATH>/.glusterfs/changelogs, if user configured to different
+        # location then consuming history will not work(Known issue as of now)
+        changelog_path = os.path.join(rconf.args.local_path,
+                                      ".glusterfs/changelogs")
+        ret, actual_end = libgfchangelog.history_changelog(
+            changelog_path,
+            data_stime[0],
+            end_time,
+            gconf.get("sync-jobs"))
+
+        # scan followed by getchanges till scan returns zero.
+        # history_scan() is blocking call, till it gets the number
+        # of changelogs to process. Returns zero when no changelogs
+        # to be processed. returns positive value as number of changelogs
+        # to be processed, which will be fetched using
+        # history_getchanges()
+        while libgfchangelog.history_scan() > 0:
+            self.crawls += 1
+
+            changes = libgfchangelog.history_getchanges()
+            if changes:
+                if data_stime:
+                    logging.info(lf("slave's time",
+                                    stime=data_stime))
+                    processed = [x for x in changes
+                                 if int(x.split('.')[-1]) < data_stime[0]]
+                    for pr in processed:
+                        logging.debug(lf('skipping already processed change',
+                                         changelog=os.path.basename(pr)))
+                        self.changelog_done_func(pr)
+                        changes.remove(pr)
+
+            self.changelogs_batch_process(changes)
+
+        history_turn_time = int(time.time()) - self.history_crawl_start_time
+
+        logging.info(lf('finished history crawl',
+                        endtime=actual_end,
+                        stime=self.get_data_stime(),
+                        entry_stime=self.get_entry_stime()))
+
+        # If TS returned from history_changelog is < register_time
+        # then FS crawl may be required, since history is only available
+        # till TS returned from history_changelog
+        if actual_end < self.changelog_register_time:
+            if self.history_turns < 2:
+                sleep_time = 1
+                if history_turn_time < CHANGELOG_ROLLOVER_TIME:
+                    sleep_time = CHANGELOG_ROLLOVER_TIME - history_turn_time
+                time.sleep(sleep_time)
+                self.history_crawl_start_time = int(time.time())
+                self.crawl()
+            else:
+                # This exception will be caught in resource.py and
+                # fallback to xsync for the small gap.
+                raise PartialHistoryAvailable(str(actual_end))
+
+
+class GMasterXsyncMixin(GMasterChangelogMixin):
+
+    """
+    This crawl needs to be xtime based (as of now
+    it's not. this is because we generate CHANGELOG
+    file during each crawl which is then processed
+    by process_change()).
+    For now it's used as a one-shot initial sync
+    mechanism and only syncs directories, regular
+    files, hardlinks and symlinks.
+    """
+
+    XSYNC_MAX_ENTRIES = 1 << 13
+
+    def register(self, register_time=None, status=None):
+        self.status = status
+        self.counter = 0
+        self.comlist = []
+        self.stimes = []
+        self.sleep_interval = 60
+        self.tempdir = self.setup_working_dir()
+        logging.info(lf('Working dir',
+                        path=self.tempdir))
+        self.tempdir = os.path.join(self.tempdir, 'xsync')
+        self.processed_changelogs_dir = self.tempdir
+        self.name = "xsync"
+        try:
+            os.makedirs(self.tempdir)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno == EEXIST and os.path.isdir(self.tempdir):
+                pass
+            else:
+                raise
+        # Purge stale unprocessed xsync changelogs
+        for f in os.listdir(self.tempdir):
+            if f.startswith("XSYNC-CHANGELOG"):
+                os.remove(os.path.join(self.tempdir, f))
+
+
+    def crawl(self):
+        """
+        event dispatcher thread
+
+        this thread dispatches either changelog or synchronizes stime.
+        additionally terminates itself on receiving a 'finale' event
+        """
+        def Xsyncer():
+            self.Xcrawl()
+        t = Thread(target=Xsyncer)
+        t.start()
+        logging.info(lf('starting hybrid crawl',
+                        stime=self.get_data_stime()))
+        self.status.set_worker_crawl_status("Hybrid Crawl")
+        while True:
+            try:
+                item = self.comlist.pop(0)
+                if item[0] == 'finale':
+                    logging.info(lf('finished hybrid crawl',
+                                    stime=self.get_data_stime()))
+                    break
+                elif item[0] == 'xsync':
+                    logging.info(lf('processing xsync changelog',
+                                    path=item[1]))
+                    self.process([item[1]], 0)
+                    self.archive_and_purge_changelogs([item[1]])
+                elif item[0] == 'stime':
+                    logging.debug(lf('setting slave time',
+                                     time=item[1]))
+                    self.upd_stime(item[1][1], item[1][0])
+                else:
+                    logging.warn(lf('unknown tuple in comlist',
+                                    entry=item))
+            except IndexError:
+                time.sleep(1)
+
+    def write_entry_change(self, prefix, data=[]):
+        if not getattr(self, "fh", None):
+            self.open()
+
+        self.fh.write("%s %s\n" % (prefix, ' '.join(data)))
+
+    def open(self):
+        try:
+            self.xsync_change = os.path.join(
+                self.tempdir, 'XSYNC-CHANGELOG.' + str(int(time.time())))
+            self.fh = open(self.xsync_change, 'w')
+        except IOError:
+            raise
+
+    def close(self):
+        if getattr(self, "fh", None):
+            self.fh.flush()
+            os.fsync(self.fh.fileno())
+            self.fh.close()
+            self.fh = None
+
+    def fname(self):
+        return self.xsync_change
+
+    def put(self, mark, item):
+        self.comlist.append((mark, item))
+
+    def sync_xsync(self, last):
+        """schedule a processing of changelog"""
+        self.close()
+        if self.counter > 0:
+            self.put('xsync', self.fname())
+        self.counter = 0
+        if not last:
+            time.sleep(1)  # make sure changelogs are 1 second apart
+
+    def sync_stime(self, stime=None, last=False):
+        """schedule a stime synchronization"""
+        if stime:
+            self.put('stime', stime)
+        if last:
+            self.put('finale', None)
+
+    def sync_done(self, stime=[], last=False):
+        self.sync_xsync(last)
+        if stime:
+            # Send last as True only for last stime entry
+            for st in stime[:-1]:
+                self.sync_stime(st, False)
+
+            if stime and stime[-1]:
+                self.sync_stime(stime[-1], last)
+
+    def is_sticky(self, path, mo):
+        """check for DHTs linkto sticky bit file"""
+        sticky = False
+        if mo & 0o1000:
+            sticky = self.master.server.linkto_check(path)
+        return sticky
+
+    def Xcrawl(self, path='.', xtr_root=None):
+        """
+        generate a CHANGELOG file consumable by process_change.
+
+        slave's xtime (stime) is _cached_ for comparisons across
+        the filesystem tree, but set after directory synchronization.
+        """
+        if path == '.':
+            self.crawls += 1
+        if not xtr_root:
+            # get the root stime and use it for all comparisons
+            xtr_root = self.xtime('.', self.slave)
+            if isinstance(xtr_root, int):
+                if xtr_root != ENOENT:
+                    logging.warn(lf("slave cluster not returning the "
+                                    "xtime for root",
+                                    error=xtr_root))
+                xtr_root = self.minus_infinity
+        xtl = self.xtime(path)
+        if isinstance(xtl, int):
+            logging.warn("master cluster's xtime not found")
+        xtr = self.xtime(path, self.slave)
+        if isinstance(xtr, int):
+            if xtr != ENOENT:
+                logging.warn(lf("slave cluster not returning the "
+                                "xtime for dir",
+                                path=path,
+                                error=xtr))
+            xtr = self.minus_infinity
+        xtr = max(xtr, xtr_root)
+        zero_zero = (0, 0)
+        if xtr_root == zero_zero:
+            xtr = self.minus_infinity
+        if not self.need_sync(path, xtl, xtr):
+            if path == '.':
+                self.sync_done([(path, xtl)], True)
+            return
+        self.xtime_reversion_hook(path, xtl, xtr)
+        logging.debug("entering " + path)
+        dem = self.master.server.entries(path)
+        pargfid = self.master.server.gfid(path)
+        if isinstance(pargfid, int):
+            logging.warn(lf('skipping directory',
+                            path=path))
+        for e in dem:
+            bname = e
+            e = os.path.join(path, e)
+            xte = self.xtime(e)
+            if isinstance(xte, int):
+                logging.warn(lf("irregular xtime",
+                                path=e,
+                                error=errno.errorcode[xte]))
+                continue
+            if not self.need_sync(e, xte, xtr):
+                continue
+            st = self.master.server.lstat(e)
+            if isinstance(st, int):
+                logging.warn(lf('got purged in the interim',
+                                path=e))
+                continue
+            if self.is_sticky(e, st.st_mode):
+                logging.debug(lf('ignoring sticky bit file',
+                                 path=e))
+                continue
+            gfid = self.master.server.gfid(e)
+            if isinstance(gfid, int):
+                logging.warn(lf('skipping entry',
+                                path=e))
+                continue
+            mo = st.st_mode
+            self.counter += 1 if ((stat.S_ISDIR(mo) or
+                                   stat.S_ISLNK(mo) or
+                                   stat.S_ISREG(mo))) else 0
+            if self.counter == self.XSYNC_MAX_ENTRIES:
+                self.sync_done(self.stimes, False)
+                self.stimes = []
+            if stat.S_ISDIR(mo):
+                self.write_entry_change("E",
+                                        [gfid, 'MKDIR', str(mo),
+                                         str(0), str(0), escape_space_newline(
+                                             os.path.join(pargfid, bname))])
+                self.write_entry_change("M", [gfid, "SETATTR", str(st.st_uid),
+                                              str(st.st_gid), str(st.st_mode),
+                                              str(st.st_atime),
+                                              str(st.st_mtime)])
+                self.Xcrawl(e, xtr_root)
+                stime_to_update = xte
+                # Live Changelog Start time indicates that from that time
+                # onwards Live changelogs are available. If we update stime
+                # greater than live_changelog_start time then Geo-rep will
+                # skip those changelogs as already processed. But Xsync
+                # actually failed to sync the deletes and Renames. Update
+                # stime as min(Live_changelogs_time, Actual_stime) When it
+                # switches to Changelog mode, it syncs Deletes and Renames.
+                if self.live_changelog_start_time:
+                    stime_to_update = min(self.live_changelog_start_time, xte)
+                self.stimes.append((e, stime_to_update))
+            elif stat.S_ISLNK(mo):
+                self.write_entry_change(
+                    "E", [gfid, 'SYMLINK', escape_space_newline(
+                        os.path.join(pargfid, bname))])
+            elif stat.S_ISREG(mo):
+                nlink = st.st_nlink
+                nlink -= 1  # fixup backend stat link count
+                # if a file has a hardlink, create a Changelog entry as
+                # 'LINK' so the slave side will decide if to create the
+                # new entry, or to create link.
+                if nlink == 1:
+                    self.write_entry_change("E",
+                                            [gfid, 'MKNOD', str(mo),
+                                             str(0), str(0),
+                                             escape_space_newline(
+                                                 os.path.join(
+                                                     pargfid, bname))])
+                else:
+                    self.write_entry_change(
+                        "E", [gfid, 'LINK', escape_space_newline(
+                            os.path.join(pargfid, bname))])
+                self.write_entry_change("D", [gfid])
+        if path == '.':
+            stime_to_update = xtl
+            if self.live_changelog_start_time:
+                stime_to_update = min(self.live_changelog_start_time, xtl)
+            self.stimes.append((path, stime_to_update))
+            self.sync_done(self.stimes, True)
+
+
+class BoxClosedErr(Exception):
+    pass
+
+
+class PostBox(list):
+
+    """synchronized collection for storing things thought of as "requests" """
+
+    def __init__(self, *a):
+        list.__init__(self, *a)
+        # too bad Python stdlib does not have read/write locks...
+        # it would suffivce to grab the lock in .append as reader, in .close as
+        # writer
+        self.lever = Condition()
+        self.open = True
+        self.done = False
+
+    def wait(self):
+        """wait on requests to be processed"""
+        self.lever.acquire()
+        if not self.done:
+            self.lever.wait()
+        self.lever.release()
+        return self.result
+
+    def wakeup(self, data):
+        """wake up requestors with the result"""
+        self.result = data
+        self.lever.acquire()
+        self.done = True
+        self.lever.notifyAll()
+        self.lever.release()
+
+    def append(self, e):
+        """post a request"""
+        self.lever.acquire()
+        if not self.open:
+            raise BoxClosedErr
+        list.append(self, e)
+        self.lever.release()
+
+    def close(self):
+        """prohibit the posting of further requests"""
+        self.lever.acquire()
+        self.open = False
+        self.lever.release()
+
+
+class Syncer(object):
+
+    """a staged queue to relay rsync requests to rsync workers
+
+    By "staged queue" its meant that when a consumer comes to the
+    queue, it takes _all_ entries, leaving the queue empty.
+    (I don't know if there is an official term for this pattern.)
+
+    The queue uses a PostBox to accumulate incoming items.
+    When a consumer (rsync worker) comes, a new PostBox is
+    set up and the old one is passed on to the consumer.
+
+    Instead of the simplistic scheme of having one big lock
+    which synchronizes both the addition of new items and
+    PostBox exchanges, use a separate lock to arbitrate consumers,
+    and rely on PostBox's synchronization mechanisms take
+    care about additions.
+
+    There is a corner case racy situation, producers vs. consumers,
+    which is not handled by this scheme: namely, when the PostBox
+    exchange occurs in between being passed to the producer for posting
+    and the post placement. But that's what Postbox.close is for:
+    such a posting will find the PostBox closed, in which case
+    the producer can re-try posting against the actual PostBox of
+    the queue.
+
+    To aid accumlation of items in the PostBoxen before grabbed
+    by an rsync worker, the worker goes to sleep a bit after
+    each completed syncjob.
+    """
+
+    def __init__(self, slave, sync_engine, resilient_errnos=[]):
+        """spawn worker threads"""
+        self.log_err = False
+        self.slave = slave
+        self.lock = Lock()
+        self.pb = PostBox()
+        self.sync_engine = sync_engine
+        self.errnos_ok = resilient_errnos
+        for i in range(gconf.get("sync-jobs")):
+            t = Thread(target=self.syncjob, args=(i + 1, ))
+            t.start()
+
+    def syncjob(self, job_id):
+        """the life of a worker"""
+        while True:
+            pb = None
+            while True:
+                self.lock.acquire()
+                if self.pb:
+                    pb, self.pb = self.pb, PostBox()
+                self.lock.release()
+                if pb:
+                    break
+                time.sleep(0.5)
+            pb.close()
+            start = time.time()
+            po = self.sync_engine(pb, self.log_err)
+            logging.info(lf("Sync Time Taken",
+                            job=job_id,
+                            num_files=len(pb),
+                            return_code=po.returncode,
+                            duration="%.4f" % (time.time() - start)))
+
+            if po.returncode == 0:
+                ret = (True, 0)
+            elif po.returncode in self.errnos_ok:
+                ret = (False, po.returncode)
+            else:
+                po.errfail()
+            pb.wakeup(ret)
+
+    def add(self, e):
+        while True:
+            pb = self.pb
+            try:
+                pb.append(e)
+                return pb
+            except BoxClosedErr:
+                pass
+
+    def enable_errorlog(self):
+        self.log_err = True
+
+    def disable_errorlog(self):
+        self.log_err = False
diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py
new file mode 100644
index 00000000000..6aa7b9dfc99
--- /dev/null
+++ b/geo-replication/syncdaemon/monitor.py
@@ -0,0 +1,395 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+import time
+import signal
+import logging
+import xml.etree.ElementTree as XET
+from threading import Lock
+from errno import ECHILD, ESRCH
+import random
+
+from resource import SSH
+import gsyncdconfig as gconf
+import libgfchangelog
+from rconf import rconf
+from syncdutils import (select, waitpid, errno_wrap, lf, grabpidfile,
+                        set_term_handler, GsyncdError,
+                        Thread, finalize, Volinfo, VolinfoFromGconf,
+                        gf_event, EVENT_GEOREP_FAULTY, get_up_nodes,
+                        unshare_propagation_supported)
+from gsyncdstatus import GeorepStatus, set_monitor_status
+import py2py3
+from py2py3 import pipe
+
+ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
+
+
+def get_subvol_num(brick_idx, vol, hot):
+    tier = vol.is_tier()
+    disperse_count = vol.disperse_count(tier, hot)
+    replica_count = vol.replica_count(tier, hot)
+    distribute_count = vol.distribution_count(tier, hot)
+    gconf.setconfig("master-distribution-count", distribute_count)
+
+    if (tier and not hot):
+        brick_idx = brick_idx - vol.get_hot_bricks_count(tier)
+
+    subvol_size = disperse_count if disperse_count > 0 else replica_count
+    cnt = int((brick_idx + 1) / subvol_size)
+    rem = (brick_idx + 1) % subvol_size
+    if rem > 0:
+        cnt = cnt + 1
+
+    if (tier and hot):
+        return "hot_" + str(cnt)
+    elif (tier and not hot):
+        return "cold_" + str(cnt)
+    else:
+        return str(cnt)
+
+
+class Monitor(object):
+
+    """class which spawns and manages gsyncd workers"""
+
+    ST_INIT = 'Initializing...'
+    ST_STARTED = 'Started'
+    ST_STABLE = 'Active'
+    ST_FAULTY = 'Faulty'
+    ST_INCON = 'inconsistent'
+    _ST_ORD = [ST_STABLE, ST_INIT, ST_FAULTY, ST_INCON]
+
+    def __init__(self):
+        self.lock = Lock()
+        self.state = {}
+        self.status = {}
+
+    @staticmethod
+    def terminate():
+        # relax one SIGTERM by setting a handler that sets back
+        # standard handler
+        set_term_handler(lambda *a: set_term_handler())
+        # give a chance to graceful exit
+        errno_wrap(os.kill, [-os.getpid(), signal.SIGTERM], [ESRCH])
+
+    def monitor(self, w, argv, cpids, slave_vol, slave_host, master,
+                suuid, slavenodes):
+        """the monitor loop
+
+        Basic logic is a blantantly simple blunt heuristics:
+        if spawned client survives 60 secs, it's considered OK.
+        This servers us pretty well as it's not vulneralbe to
+        any kind of irregular behavior of the child...
+
+        ... well, except for one: if children is hung up on
+        waiting for some event, it can survive aeons, still
+        will be defunct. So we tweak the above logic to
+        expect the worker to send us a signal within 60 secs
+        (in the form of closing its end of a pipe). The worker
+        does this when it's done with the setup stage
+        ready to enter the service loop (note it's the setup
+        stage which is vulnerable to hangs -- the full
+        blown worker blows up on EPIPE if the net goes down,
+        due to the keep-alive thread)
+        """
+        if not self.status.get(w[0]['dir'], None):
+            self.status[w[0]['dir']] = GeorepStatus(gconf.get("state-file"),
+                                                    w[0]['host'],
+                                                    w[0]['dir'],
+                                                    w[0]['uuid'],
+                                                    master,
+                                                    "%s::%s" % (slave_host,
+                                                                slave_vol))
+        ret = 0
+
+        def nwait(p, o=0):
+            try:
+                p2, r = waitpid(p, o)
+                if not p2:
+                    return
+                return r
+            except OSError as e:
+                # no child process, this happens if the child process
+                # already died and has been cleaned up
+                if e.errno == ECHILD:
+                    return -1
+                else:
+                    raise
+
+        def exit_signalled(s):
+            """ child terminated due to receipt of SIGUSR1 """
+            return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
+
+        def exit_status(s):
+            if os.WIFEXITED(s):
+                return os.WEXITSTATUS(s)
+            return 1
+
+        conn_timeout = gconf.get("connection-timeout")
+        while ret in (0, 1):
+            remote_user, remote_host = w[1][0].split("@")
+            remote_id = w[1][1]
+            # Check the status of the connected slave node
+            # If the connected slave node is down then try to connect to
+            # different up node.
+            current_slave_host = remote_host
+            slave_up_hosts = get_up_nodes(slavenodes, gconf.get("ssh-port"))
+
+            if (current_slave_host, remote_id) not in slave_up_hosts:
+                if len(slave_up_hosts) > 0:
+                    remote_new = random.choice(slave_up_hosts)
+                    remote_host = "%s@%s" % (remote_user, remote_new[0])
+                    remote_id = remote_new[1]
+
+            # Spawn the worker in lock to avoid fd leak
+            self.lock.acquire()
+
+            self.status[w[0]['dir']].set_worker_status(self.ST_INIT)
+            logging.info(lf('starting gsyncd worker',
+                            brick=w[0]['dir'],
+                            slave_node=remote_host))
+
+            pr, pw = pipe()
+            cpid = os.fork()
+            if cpid == 0:
+                os.close(pr)
+
+                args_to_worker = argv + [
+                    'worker',
+                    rconf.args.master,
+                    rconf.args.slave,
+                    '--feedback-fd', str(pw),
+                    '--local-path', w[0]['dir'],
+                    '--local-node', w[0]['host'],
+                    '--local-node-id', w[0]['uuid'],
+                    '--slave-id', suuid,
+                    '--subvol-num', str(w[2]),
+                    '--resource-remote', remote_host,
+                    '--resource-remote-id', remote_id
+                ]
+
+                if rconf.args.config_file is not None:
+                    args_to_worker += ['-c', rconf.args.config_file]
+
+                if w[3]:
+                    args_to_worker.append("--is-hottier")
+
+                if rconf.args.debug:
+                    args_to_worker.append("--debug")
+
+                access_mount = gconf.get("access-mount")
+                if access_mount:
+                    os.execv(sys.executable, args_to_worker)
+                else:
+                    if unshare_propagation_supported():
+                        logging.debug("Worker would mount volume privately")
+                        unshare_cmd = ['unshare', '-m', '--propagation',
+                                       'private']
+                        cmd = unshare_cmd + args_to_worker
+                        os.execvp("unshare", cmd)
+                    else:
+                        logging.debug("Mount is not private. It would be lazy"
+                                      " umounted")
+                        os.execv(sys.executable, args_to_worker)
+
+            cpids.add(cpid)
+            os.close(pw)
+
+            self.lock.release()
+
+            t0 = time.time()
+            so = select((pr,), (), (), conn_timeout)[0]
+            os.close(pr)
+
+            if so:
+                ret = nwait(cpid, os.WNOHANG)
+
+                if ret is not None:
+                    logging.info(lf("worker died before establishing "
+                                    "connection",
+                                    brick=w[0]['dir']))
+                else:
+                    logging.debug("worker(%s) connected" % w[0]['dir'])
+                    while time.time() < t0 + conn_timeout:
+                        ret = nwait(cpid, os.WNOHANG)
+
+                        if ret is not None:
+                            logging.info(lf("worker died in startup phase",
+                                            brick=w[0]['dir']))
+                            break
+
+                        time.sleep(1)
+            else:
+                logging.info(
+                    lf("Worker not confirmed after wait, aborting it. "
+                       "Gsyncd invocation on remote slave via SSH or "
+                       "gluster master mount might have hung. Please "
+                       "check the above logs for exact issue and check "
+                       "master or slave volume for errors. Restarting "
+                       "master/slave volume accordingly might help.",
+                       brick=w[0]['dir'],
+                       timeout=conn_timeout))
+                errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
+                ret = nwait(cpid)
+            if ret is None:
+                ret = nwait(cpid)
+            if exit_signalled(ret):
+                ret = 0
+            else:
+                ret = exit_status(ret)
+                if ret in (0, 1):
+                    self.status[w[0]['dir']].set_worker_status(self.ST_FAULTY)
+                    gf_event(EVENT_GEOREP_FAULTY,
+                             master_volume=master.volume,
+                             master_node=w[0]['host'],
+                             master_node_id=w[0]['uuid'],
+                             slave_host=slave_host,
+                             slave_volume=slave_vol,
+                             current_slave_host=current_slave_host,
+                             brick_path=w[0]['dir'])
+            time.sleep(10)
+        self.status[w[0]['dir']].set_worker_status(self.ST_INCON)
+        return ret
+
+    def multiplex(self, wspx, suuid, slave_vol, slave_host, master, slavenodes):
+        argv = [os.path.basename(sys.executable), sys.argv[0]]
+
+        cpids = set()
+        ta = []
+        for wx in wspx:
+            def wmon(w):
+                cpid, _ = self.monitor(w, argv, cpids, slave_vol,
+                                       slave_host, master, suuid, slavenodes)
+                time.sleep(1)
+                self.lock.acquire()
+                for cpid in cpids:
+                    errno_wrap(os.kill, [cpid, signal.SIGKILL], [ESRCH])
+                self.lock.release()
+                finalize(exval=1)
+            t = Thread(target=wmon, args=[wx])
+            t.start()
+            ta.append(t)
+
+        # monitor status was being updated in each monitor thread. It
+        # should not be done as it can cause deadlock for a worker start.
+        # set_monitor_status uses flock to synchronize multple instances
+        # updating the file. Since each monitor thread forks worker,
+        # these processes can hold the reference to fd of status
+        # file causing deadlock to workers which starts later as flock
+        # will not be release until all references to same fd is closed.
+        # It will also cause fd leaks.
+
+        self.lock.acquire()
+        set_monitor_status(gconf.get("state-file"), self.ST_STARTED)
+        self.lock.release()
+        for t in ta:
+            t.join()
+
+
+def distribute(master, slave):
+    if rconf.args.use_gconf_volinfo:
+        mvol = VolinfoFromGconf(master.volume, master=True)
+    else:
+        mvol = Volinfo(master.volume, master.host, master=True)
+    logging.debug('master bricks: ' + repr(mvol.bricks))
+    prelude = []
+    slave_host = None
+    slave_vol = None
+
+    prelude = [gconf.get("ssh-command")] + \
+        gconf.get("ssh-options").split() + \
+        ["-p", str(gconf.get("ssh-port"))] + \
+        [slave.remote_addr]
+
+    logging.debug('slave SSH gateway: ' + slave.remote_addr)
+
+    if rconf.args.use_gconf_volinfo:
+        svol = VolinfoFromGconf(slave.volume, master=False)
+    else:
+        svol = Volinfo(slave.volume, "localhost", prelude, master=False)
+
+    sbricks = svol.bricks
+    suuid = svol.uuid
+    slave_host = slave.remote_addr.split('@')[-1]
+    slave_vol = slave.volume
+
+    # save this xattr for the session delete command
+    old_stime_xattr_prefix = gconf.get("stime-xattr-prefix", None)
+    new_stime_xattr_prefix = "trusted.glusterfs." + mvol.uuid + "." + \
+                             svol.uuid
+    if not old_stime_xattr_prefix or \
+       old_stime_xattr_prefix != new_stime_xattr_prefix:
+        gconf.setconfig("stime-xattr-prefix", new_stime_xattr_prefix)
+
+    logging.debug('slave bricks: ' + repr(sbricks))
+
+    slavenodes = set((b['host'], b["uuid"]) for b in sbricks)
+    rap = SSH.parse_ssh_address(slave)
+    slaves = [(rap['user'] + '@' + h[0], h[1]) for h in slavenodes]
+
+    workerspex = []
+    for idx, brick in enumerate(mvol.bricks):
+        if rconf.args.local_node_id == brick['uuid']:
+            is_hot = mvol.is_hot(":".join([brick['host'], brick['dir']]))
+            workerspex.append((brick,
+                               slaves[idx % len(slaves)],
+                               get_subvol_num(idx, mvol, is_hot),
+                               is_hot))
+    logging.debug('worker specs: ' + repr(workerspex))
+    return workerspex, suuid, slave_vol, slave_host, master, slavenodes
+
+
+def monitor(local, remote):
+    # Check if gsyncd restarted in pause state. If
+    # yes, send SIGSTOP to negative of monitor pid
+    # to go back to pause state.
+    if rconf.args.pause_on_start:
+        errno_wrap(os.kill, [-os.getpid(), signal.SIGSTOP], [ESRCH])
+
+    """oh yeah, actually Monitor is used as singleton, too"""
+    return Monitor().multiplex(*distribute(local, remote))
+
+
+def startup(go_daemon=True):
+    """set up logging, pidfile grabbing, daemonization"""
+    pid_file = gconf.get("pid-file")
+    if not grabpidfile():
+        sys.stderr.write("pidfile is taken, exiting.\n")
+        sys.exit(2)
+    rconf.pid_file_owned = True
+
+    if not go_daemon:
+        return
+
+    x, y = pipe()
+    cpid = os.fork()
+    if cpid:
+        os.close(x)
+        sys.exit()
+    os.close(y)
+    os.setsid()
+    dn = os.open(os.devnull, os.O_RDWR)
+    for f in (sys.stdin, sys.stdout, sys.stderr):
+        os.dup2(dn, f.fileno())
+
+    if not grabpidfile(pid_file + '.tmp'):
+        raise GsyncdError("cannot grab temporary pidfile")
+
+    os.rename(pid_file + '.tmp', pid_file)
+
+    # wait for parent to terminate
+    # so we can start up with
+    # no messing from the dirty
+    # ol' bustard
+    select((x,), (), ())
+    os.close(x)
diff --git a/geo-replication/syncdaemon/py2py3.py b/geo-replication/syncdaemon/py2py3.py
new file mode 100644
index 00000000000..f9c76e1b50a
--- /dev/null
+++ b/geo-replication/syncdaemon/py2py3.py
@@ -0,0 +1,184 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+# All python2/python3 compatibility routines
+
+import sys
+import os
+import stat
+import struct
+from syncdutils import umask
+from ctypes import create_string_buffer
+
+if sys.version_info >= (3,):
+    def pipe():
+        (r, w) = os.pipe()
+        os.set_inheritable(r, True)
+        os.set_inheritable(w, True)
+        return (r, w)
+
+    # Raw conversion of bytearray to string. Used in the cases where
+    # buffer is created by create_string_buffer which is a 8-bit char
+    # array and passed to syscalls to fetch results. Using encode/decode
+    # doesn't work as it converts to string altering the size.
+    def bytearray_to_str(byte_arr):
+        return ''.join([chr(b) for b in byte_arr])
+
+    # Raw conversion of string to bytes. This is required to convert
+    # back the string into bytearray(c char array) to use in struc
+    # pack/unpacking. Again encode/decode can't be used as it
+    # converts it alters size.
+    def str_to_bytearray(string):
+        return bytes([ord(c) for c in string])
+
+    def gr_create_string_buffer(size):
+        return create_string_buffer(b'\0', size)
+
+    def gr_query_xattr(cls, path, size, syscall, attr=None):
+        if attr:
+            return cls._query_xattr(path.encode(), size, syscall,
+                                    attr.encode())
+        else:
+            return cls._query_xattr(path.encode(), size, syscall)
+
+    def gr_lsetxattr(cls, path, attr, val):
+        return cls.libc.lsetxattr(path.encode(), attr.encode(), val,
+                                  len(val), 0)
+
+    def gr_lremovexattr(cls, path, attr):
+        return cls.libc.lremovexattr(path.encode(), attr.encode())
+
+    def gr_cl_register(libgfapi, brick, path, log_file, log_level, retries):
+        return libgfapi.gf_changelog_register(brick.encode(),
+                                              path.encode(),
+                                              log_file.encode(),
+                                              log_level, retries)
+
+    def gr_cl_done(libgfapi, clfile):
+        return libgfapi.gf_changelog_done(clfile.encode())
+
+    def gr_cl_history_changelog(libgfapi, changelog_path, start, end, num_parallel,
+                                actual_end):
+        return libgfapi.gf_history_changelog(changelog_path.encode(),
+                                             start, end, num_parallel,
+                                             actual_end)
+
+    def gr_cl_history_done(libgfapi, clfile):
+        return libgfapi.gf_history_changelog_done(clfile.encode())
+
+    # regular file
+
+    def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        return struct.pack(cls._fmt_mknod(blen),
+                           uid, gid, gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), 0, umask())
+
+    def entry_pack_reg_stat(cls, gf, bn, st):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        mo = st['mode']
+        return struct.pack(cls._fmt_mknod(blen),
+                           st['uid'], st['gid'],
+                           gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), 0, umask())
+    # mkdir
+
+    def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        return struct.pack(cls._fmt_mkdir(blen),
+                           uid, gid, gf.encode(), mo, bn_encoded,
+                           stat.S_IMODE(mo), umask())
+    # symlink
+
+    def entry_pack_symlink(cls, gf, bn, lnk, st):
+        bn_encoded = bn.encode()
+        blen = len(bn_encoded)
+        lnk_encoded = lnk.encode()
+        llen = len(lnk_encoded)
+        return struct.pack(cls._fmt_symlink(blen, llen),
+                           st['uid'], st['gid'],
+                           gf.encode(), st['mode'], bn_encoded,
+                           lnk_encoded)
+else:
+    def pipe():
+        (r, w) = os.pipe()
+        return (r, w)
+
+    # Raw conversion of bytearray to string
+    def bytearray_to_str(byte_arr):
+        return byte_arr
+
+    # Raw conversion of string to bytearray
+    def str_to_bytearray(string):
+        return string
+
+    def gr_create_string_buffer(size):
+        return create_string_buffer('\0', size)
+
+    def gr_query_xattr(cls, path, size, syscall, attr=None):
+        if attr:
+            return cls._query_xattr(path, size, syscall, attr)
+        else:
+            return cls._query_xattr(path, size, syscall)
+
+    def gr_lsetxattr(cls, path, attr, val):
+        return cls.libc.lsetxattr(path, attr, val, len(val), 0)
+
+    def gr_lremovexattr(cls, path, attr):
+        return cls.libc.lremovexattr(path, attr)
+
+    def gr_cl_register(libgfapi, brick, path, log_file, log_level, retries):
+        return libgfapi.gf_changelog_register(brick, path, log_file,
+                                              log_level, retries)
+
+    def gr_cl_done(libgfapi, clfile):
+        return libgfapi.gf_changelog_done(clfile)
+
+    def gr_cl_history_changelog(libgfapi, changelog_path, start, end, num_parallel,
+                                actual_end):
+        return libgfapi.gf_history_changelog(changelog_path, start, end,
+                                             num_parallel, actual_end)
+
+    def gr_cl_history_done(libgfapi, clfile):
+        return libgfapi.gf_history_changelog_done(clfile)
+
+    # regular file
+
+    def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+        blen = len(bn)
+        return struct.pack(cls._fmt_mknod(blen),
+                           uid, gid, gf, mo, bn,
+                           stat.S_IMODE(mo), 0, umask())
+
+    def entry_pack_reg_stat(cls, gf, bn, st):
+        blen = len(bn)
+        mo = st['mode']
+        return struct.pack(cls._fmt_mknod(blen),
+                           st['uid'], st['gid'],
+                           gf, mo, bn,
+                           stat.S_IMODE(mo), 0, umask())
+    # mkdir
+
+    def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+        blen = len(bn)
+        return struct.pack(cls._fmt_mkdir(blen),
+                           uid, gid, gf, mo, bn,
+                           stat.S_IMODE(mo), umask())
+    # symlink
+
+    def entry_pack_symlink(cls, gf, bn, lnk, st):
+        blen = len(bn)
+        llen = len(lnk)
+        return struct.pack(cls._fmt_symlink(blen, llen),
+                           st['uid'], st['gid'],
+                           gf, st['mode'], bn, lnk)
diff --git a/geo-replication/syncdaemon/rconf.py b/geo-replication/syncdaemon/rconf.py
new file mode 100644
index 00000000000..ff716ee4d6d
--- /dev/null
+++ b/geo-replication/syncdaemon/rconf.py
@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+
+class RConf(object):
+
+    """singleton class to store runtime globals
+       shared between gsyncd modules"""
+
+    ssh_ctl_dir = None
+    ssh_ctl_args = None
+    cpid = None
+    pid_file_owned = False
+    log_exit = False
+    permanent_handles = []
+    log_metadata = {}
+    mgmt_lock_fd = None
+    args = None
+    turns = 0
+    mountbroker = False
+    mount_point = None
+    mbr_umount_cmd = []
+
+rconf = RConf()
diff --git a/geo-replication/syncdaemon/repce.py b/geo-replication/syncdaemon/repce.py
new file mode 100644
index 00000000000..c622afa6373
--- /dev/null
+++ b/geo-replication/syncdaemon/repce.py
@@ -0,0 +1,253 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+import time
+import logging
+from threading import Condition
+try:
+    import _thread as thread
+except ImportError:
+    import thread
+try:
+    from queue import Queue
+except ImportError:
+    from Queue import Queue
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+from syncdutils import Thread, select, lf
+
+pickle_proto = 2
+repce_version = 1.0
+
+
+def ioparse(i, o):
+    if isinstance(i, int):
+        i = os.fdopen(i, 'rb')
+    # rely on duck typing for recognizing
+    # streams as that works uniformly
+    # in py2 and py3
+    if hasattr(o, 'fileno'):
+        o = o.fileno()
+    return (i, o)
+
+
+def send(out, *args):
+    """pickle args and write out wholly in one syscall
+
+    ie. not use the ability of pickle to dump directly to
+    a stream, as that would potentially mess up messages
+    by interleaving them
+    """
+    os.write(out, pickle.dumps(args, pickle_proto))
+
+
+def recv(inf):
+    """load an object from input stream
+    python2 and python3 compatibility, inf is sys.stdin
+    and is opened as text stream by default. Hence using the
+    buffer attribute in python3
+    """
+    if hasattr(inf, "buffer"):
+        return pickle.load(inf.buffer)
+    else:
+        return pickle.load(inf)
+
+
+class RepceServer(object):
+
+    """RePCe is Hungarian for canola, http://hu.wikipedia.org/wiki/Repce
+
+    ... also our homebrewed RPC backend where the transport layer is
+    reduced to a pair of filehandles.
+
+    This is the server component.
+    """
+
+    def __init__(self, obj, i, o, wnum=6):
+        """register a backend object .obj to which incoming messages
+           are dispatched, also incoming/outcoming streams
+        """
+        self.obj = obj
+        self.inf, self.out = ioparse(i, o)
+        self.wnum = wnum
+        self.q = Queue()
+
+    def service_loop(self):
+        """fire up worker threads, get messages and dispatch among them"""
+        for i in range(self.wnum):
+            t = Thread(target=self.worker)
+            t.start()
+        try:
+            while True:
+                self.q.put(recv(self.inf))
+        except EOFError:
+            logging.info("terminating on reaching EOF.")
+
+    def worker(self):
+        """life of a worker
+
+        Get message, extract its id, method name and arguments
+        (kwargs not supported), call method on .obj.
+        Send back message id + return value.
+        If method call throws an exception, rescue it, and send
+        back the exception as result (with flag marking it as
+        exception).
+        """
+        while True:
+            in_data = self.q.get(True)
+            rid = in_data[0]
+            rmeth = in_data[1]
+            exc = False
+            if rmeth == '__repce_version__':
+                res = repce_version
+            else:
+                try:
+                    res = getattr(self.obj, rmeth)(*in_data[2:])
+                except:
+                    res = sys.exc_info()[1]
+                    exc = True
+                    logging.exception("call failed: ")
+            send(self.out, rid, exc, res)
+
+
+class RepceJob(object):
+
+    """class representing message status we can use
+    for waiting on reply"""
+
+    def __init__(self, cbk):
+        """
+        - .rid: (process-wise) unique id
+        - .cbk: what we do upon receiving reply
+        """
+        self.rid = (os.getpid(), thread.get_ident(), time.time())
+        self.cbk = cbk
+        self.lever = Condition()
+        self.done = False
+
+    def __repr__(self):
+        return ':'.join([str(x) for x in self.rid])
+
+    def wait(self):
+        self.lever.acquire()
+        if not self.done:
+            self.lever.wait()
+        self.lever.release()
+        return self.result
+
+    def wakeup(self, data):
+        self.result = data
+        self.lever.acquire()
+        self.done = True
+        self.lever.notify()
+        self.lever.release()
+
+
+class RepceClient(object):
+
+    """RePCe is Hungarian for canola, http://hu.wikipedia.org/wiki/Repce
+
+    ... also our homebrewed RPC backend where the transport layer is
+    reduced to a pair of filehandles.
+
+    This is the client component.
+    """
+
+    def __init__(self, i, o):
+        self.inf, self.out = ioparse(i, o)
+        self.jtab = {}
+        t = Thread(target=self.listen)
+        t.start()
+
+    def listen(self):
+        while True:
+            select((self.inf,), (), ())
+            rid, exc, res = recv(self.inf)
+            rjob = self.jtab.pop(rid)
+            if rjob.cbk:
+                rjob.cbk(rjob, [exc, res])
+
+    def push(self, meth, *args, **kw):
+        """wrap arguments in a RepceJob, send them to server
+           and return the RepceJob
+
+           @cbk to pass on RepceJob can be given as kwarg.
+        """
+        cbk = kw.get('cbk')
+        if not cbk:
+            def cbk(rj, res):
+                if res[0]:
+                    raise res[1]
+        rjob = RepceJob(cbk)
+        self.jtab[rjob.rid] = rjob
+        logging.debug("call %s %s%s ..." % (repr(rjob), meth, repr(args)))
+        send(self.out, rjob.rid, meth, *args)
+        return rjob
+
+    def __call__(self, meth, *args):
+        """RePCe client is callabe, calling it implements a synchronous
+        remote call.
+
+        We do a .push with a cbk which does a wakeup upon receiving answer,
+        then wait on the RepceJob.
+        """
+        rjob = self.push(
+            meth, *args, **{'cbk': lambda rj, res: rj.wakeup(res)})
+        exc, res = rjob.wait()
+        if exc:
+            logging.error(lf('call failed',
+                             call=repr(rjob),
+                             method=meth,
+                             error=str(type(res).__name__)))
+            raise res
+        logging.debug("call %s %s -> %s" % (repr(rjob), meth, repr(res)))
+        return res
+
+    class mprx(object):
+
+        """method proxy, standard trick to implement rubyesque
+        method_missing in Python
+
+        A class is a closure factory, you know what I mean, or go read
+        some SICP.
+        """
+
+        def __init__(self, ins, meth):
+            self.ins = ins
+            self.meth = meth
+
+        def __call__(self, *a):
+            return self.ins(self.meth, *a)
+
+    def __getattr__(self, meth):
+        """this implements transparent method dispatch to remote object,
+           so that you don't need to call the RepceClient instance like
+
+             rclient('how_old_are_you_if_born_in', 1979)
+
+           but you can make it into an ordinary method call like
+
+             rclient.how_old_are_you_if_born_in(1979)
+        """
+        return self.mprx(self, meth)
+
+    def __version__(self):
+        """used in handshake to verify compatibility"""
+        d = {'proto': self('__repce_version__')}
+        try:
+            d['object'] = self('version')
+        except AttributeError:
+            pass
+        return d
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
new file mode 100644
index 00000000000..f12c7ceaa36
--- /dev/null
+++ b/geo-replication/syncdaemon/resource.py
@@ -0,0 +1,1583 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import re
+import os
+import sys
+import stat
+import time
+import fcntl
+import types
+import struct
+import logging
+import tempfile
+import subprocess
+from errno import (EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EACCES,
+                   EISDIR, ENOTEMPTY, ESTALE, EINVAL, EBUSY, EPERM)
+import errno
+
+from rconf import rconf
+import gsyncdconfig as gconf
+import libgfchangelog
+
+import repce
+from repce import RepceServer, RepceClient
+from master import gmaster_builder
+import syncdutils
+from syncdutils import (GsyncdError, select, privileged, funcode,
+                        entry2pb, gauxpfx, errno_wrap, lstat,
+                        NoStimeAvailable, PartialHistoryAvailable,
+                        ChangelogException, ChangelogHistoryNotAvailable,
+                        get_changelog_log_level, get_rsync_version,
+                        GX_GFID_CANONICAL_LEN,
+                        gf_mount_ready, lf, Popen, sup,
+                        Xattr, matching_disk_gfid, get_gfid_from_mnt,
+                        unshare_propagation_supported, get_slv_dir_path)
+from gsyncdstatus import GeorepStatus
+from py2py3 import (pipe, str_to_bytearray, entry_pack_reg,
+                    entry_pack_reg_stat, entry_pack_mkdir,
+                    entry_pack_symlink)
+
+
+ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
+
+slv_volume = None
+slv_host = None
+
+
+class Server(object):
+
+    """singleton implemening those filesystem access primitives
+       which are needed for geo-replication functionality
+
+    (Singleton in the sense it's a class which has only static
+    and classmethods and is used directly, without instantiation.)
+    """
+
+    GX_NSPACE_PFX = (privileged() and "trusted" or "system")
+    GX_NSPACE = GX_NSPACE_PFX + ".glusterfs"
+    NTV_FMTSTR = "!" + "B" * 19 + "II"
+    FRGN_XTRA_FMT = "I"
+    FRGN_FMTSTR = NTV_FMTSTR + FRGN_XTRA_FMT
+
+    # for backend gfid fetch, do not use GX_NSPACE_PFX
+    GFID_XATTR = 'trusted.gfid'
+    GFID_FMTSTR = "!" + "B" * 16
+
+    local_path = ''
+
+    @classmethod
+    def _fmt_mknod(cls, l):
+        return "!II%dsI%dsIII" % (GX_GFID_CANONICAL_LEN, l + 1)
+
+    @classmethod
+    def _fmt_mkdir(cls, l):
+        return "!II%dsI%dsII" % (GX_GFID_CANONICAL_LEN, l + 1)
+
+    @classmethod
+    def _fmt_symlink(cls, l1, l2):
+        return "!II%dsI%ds%ds" % (GX_GFID_CANONICAL_LEN, l1 + 1, l2 + 1)
+
+    def _pathguard(f):
+        """decorator method that checks
+        the path argument of the decorated
+        functions to make sure it does not
+        point out of the managed tree
+        """
+
+        fc = funcode(f)
+        pi = list(fc.co_varnames).index('path')
+
+        def ff(*args):
+            path = args[pi]
+            ps = path.split('/')
+            if path[0] == '/' or '..' in ps:
+                raise ValueError('unsafe path')
+            args = list(args)
+            args[pi] = os.path.join(args[0].local_path, path)
+            return f(*args)
+        return ff
+
+    @classmethod
+    @_pathguard
+    def entries(cls, path):
+        """directory entries in an array"""
+        # prevent symlinks being followed
+        if not stat.S_ISDIR(os.lstat(path).st_mode):
+            raise OSError(ENOTDIR, os.strerror(ENOTDIR))
+        return os.listdir(path)
+
+    @classmethod
+    @_pathguard
+    def lstat(cls, path):
+        try:
+            return os.lstat(path)
+        except (IOError, OSError):
+            ex = sys.exc_info()[1]
+            if ex.errno == ENOENT:
+                return ex.errno
+            else:
+                raise
+
+    @classmethod
+    @_pathguard
+    def linkto_check(cls, path):
+        try:
+            return not (
+                Xattr.lgetxattr_buf(path,
+                                    'trusted.glusterfs.dht.linkto') == '')
+        except (IOError, OSError):
+            ex = sys.exc_info()[1]
+            if ex.errno in (ENOENT, ENODATA):
+                return False
+            else:
+                raise
+
+    @classmethod
+    @_pathguard
+    def gfid(cls, path):
+        buf = errno_wrap(Xattr.lgetxattr, [path, cls.GFID_XATTR, 16],
+                         [ENOENT], [ESTALE, ENODATA])
+        if buf == ENOENT:
+            return buf
+        else:
+            buf = str_to_bytearray(buf)
+            m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(
+                ['%02x' % x for x in struct.unpack(cls.GFID_FMTSTR, buf)]))
+            return '-'.join(m.groups())
+
+    @classmethod
+    @_pathguard
+    def purge(cls, path, entries=None):
+        """force-delete subtrees
+
+        If @entries is not specified, delete
+        the whole subtree under @path (including
+        @path).
+
+        Otherwise, @entries should be a
+        a sequence of children of @path, and
+        the effect is identical with a joint
+        @entries-less purge on them, ie.
+
+        for e in entries:
+            cls.purge(os.path.join(path, e))
+        """
+        me_also = entries is None
+        if not entries:
+            try:
+                # if it's a symlink, prevent
+                # following it
+                try:
+                    os.unlink(path)
+                    return
+                except OSError:
+                    ex = sys.exc_info()[1]
+                    if ex.errno == EISDIR:
+                        entries = os.listdir(path)
+                    else:
+                        raise
+            except OSError:
+                ex = sys.exc_info()[1]
+                if ex.errno in (ENOTDIR, ENOENT, ELOOP):
+                    try:
+                        os.unlink(path)
+                        return
+                    except OSError:
+                        ex = sys.exc_info()[1]
+                        if ex.errno == ENOENT:
+                            return
+                        raise
+                else:
+                    raise
+        for e in entries:
+            cls.purge(os.path.join(path, e))
+        if me_also:
+            os.rmdir(path)
+
+    @classmethod
+    @_pathguard
+    def _create(cls, path, ctor):
+        """path creation backend routine"""
+        try:
+            ctor(path)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno == EEXIST:
+                cls.purge(path)
+                return ctor(path)
+            raise
+
+    @classmethod
+    @_pathguard
+    def mkdir(cls, path):
+        cls._create(path, os.mkdir)
+
+    @classmethod
+    @_pathguard
+    def symlink(cls, lnk, path):
+        cls._create(path, lambda p: os.symlink(lnk, p))
+
+    @classmethod
+    @_pathguard
+    def xtime(cls, path, uuid):
+        """query xtime extended attribute
+
+        Return xtime of @path for @uuid as a pair of integers.
+        "Normal" errors due to non-existent @path or extended attribute
+        are tolerated and errno is returned in such a case.
+        """
+
+        try:
+            val = Xattr.lgetxattr(path,
+                                  '.'.join([cls.GX_NSPACE, uuid, 'xtime']),
+                                  8)
+            val = str_to_bytearray(val)
+            return struct.unpack('!II', val)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
+                return ex.errno
+            else:
+                raise
+
+    @classmethod
+    @_pathguard
+    def stime_mnt(cls, path, uuid):
+        """query xtime extended attribute
+
+        Return xtime of @path for @uuid as a pair of integers.
+        "Normal" errors due to non-existent @path or extended attribute
+        are tolerated and errno is returned in such a case.
+        """
+
+        try:
+            val = Xattr.lgetxattr(path,
+                                  '.'.join([cls.GX_NSPACE, uuid, 'stime']),
+                                  8)
+            val = str_to_bytearray(val)
+            return struct.unpack('!II', val)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
+                return ex.errno
+            else:
+                raise
+
+    @classmethod
+    @_pathguard
+    def stime(cls, path, uuid):
+        """query xtime extended attribute
+
+        Return xtime of @path for @uuid as a pair of integers.
+        "Normal" errors due to non-existent @path or extended attribute
+        are tolerated and errno is returned in such a case.
+        """
+
+        try:
+            val = Xattr.lgetxattr(path,
+                                  '.'.join([cls.GX_NSPACE, uuid, 'stime']),
+                                  8)
+            val = str_to_bytearray(val)
+            return struct.unpack('!II', val)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
+                return ex.errno
+            else:
+                raise
+
+    @classmethod
+    @_pathguard
+    def entry_stime(cls, path, uuid):
+        """
+        entry_stime xattr to reduce the number of retry of Entry changes when
+        Geo-rep worker crashes and restarts. entry_stime is updated after
+        processing every changelog file. On failure and restart, worker only
+        have to reprocess the last changelog for Entry ops.
+        Xattr Key: <PFX>.<MASTERVOL_UUID>.<SLAVEVOL_UUID>.entry_stime
+        """
+        try:
+            val = Xattr.lgetxattr(path,
+                                  '.'.join([cls.GX_NSPACE, uuid,
+                                            'entry_stime']),
+                                  8)
+            val = str_to_bytearray(val)
+            return struct.unpack('!II', val)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
+                return ex.errno
+            else:
+                raise
+
+    @classmethod
+    def node_uuid(cls, path='.'):
+        try:
+            uuid_l = Xattr.lgetxattr_buf(
+                path, '.'.join([cls.GX_NSPACE, 'node-uuid']))
+            return uuid_l[:-1].split(' ')
+        except OSError:
+            raise
+
+    @classmethod
+    @_pathguard
+    def set_stime(cls, path, uuid, mark):
+        """set @mark as stime for @uuid on @path"""
+        errno_wrap(Xattr.lsetxattr,
+                   [path,
+                    '.'.join([cls.GX_NSPACE, uuid, 'stime']),
+                    struct.pack('!II', *mark)],
+                   [ENOENT],
+                   [ESTALE, EINVAL])
+
+    @classmethod
+    @_pathguard
+    def set_entry_stime(cls, path, uuid, mark):
+        """set @mark as stime for @uuid on @path"""
+        errno_wrap(Xattr.lsetxattr,
+                   [path,
+                    '.'.join([cls.GX_NSPACE, uuid, 'entry_stime']),
+                    struct.pack('!II', *mark)],
+                   [ENOENT],
+                   [ESTALE, EINVAL])
+
+    @classmethod
+    @_pathguard
+    def set_xtime(cls, path, uuid, mark):
+        """set @mark as xtime for @uuid on @path"""
+        errno_wrap(Xattr.lsetxattr,
+                   [path,
+                    '.'.join([cls.GX_NSPACE, uuid, 'xtime']),
+                    struct.pack('!II', *mark)],
+                   [ENOENT],
+                   [ESTALE, EINVAL])
+
+    @classmethod
+    @_pathguard
+    def set_xtime_remote(cls, path, uuid, mark):
+        """
+        set @mark as xtime for @uuid on @path
+        the difference b/w this and set_xtime() being
+        set_xtime() being overloaded to set the xtime
+        on the brick (this method sets xtime on the
+        remote slave)
+        """
+        Xattr.lsetxattr(
+            path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']),
+            struct.pack('!II', *mark))
+
+    @classmethod
+    def entry_ops(cls, entries):
+        pfx = gauxpfx()
+        logging.debug('entries: %s' % repr(entries))
+        dist_count = rconf.args.master_dist_count
+
+        def entry_purge(op, entry, gfid, e, uid, gid):
+            # This is an extremely racy code and needs to be fixed ASAP.
+            # The GFID check here is to be sure that the pargfid/bname
+            # to be purged is the GFID gotten from the changelog.
+            # (a stat(changelog_gfid) would also be valid here)
+            # The race here is between the GFID check and the purge.
+
+            # If the entry or the gfid of the file to be deleted is not present
+            # on slave, we can ignore the unlink/rmdir
+            if isinstance(lstat(entry), int) or \
+               isinstance(lstat(os.path.join(pfx, gfid)), int):
+                return
+
+            if not matching_disk_gfid(gfid, entry):
+                collect_failure(e, EEXIST, uid, gid)
+                return
+
+            if op == 'UNLINK':
+                er = errno_wrap(os.unlink, [entry], [ENOENT, ESTALE], [EBUSY])
+                # EISDIR is safe error, ignore. This can only happen when
+                # unlink is sent from master while fixing gfid conflicts.
+                if er != EISDIR:
+                    return er
+
+            elif op == 'RMDIR':
+                er = errno_wrap(os.rmdir, [entry], [ENOENT, ESTALE,
+                                                    ENOTEMPTY], [EBUSY])
+                if er == ENOTEMPTY:
+                    return er
+
+        def collect_failure(e, cmd_ret, uid, gid, dst=False):
+            slv_entry_info = {}
+            slv_entry_info['gfid_mismatch'] = False
+            slv_entry_info['name_mismatch'] = False
+            slv_entry_info['dst'] = dst
+            slv_entry_info['slave_isdir'] = False
+            slv_entry_info['slave_name'] = None
+            slv_entry_info['slave_gfid'] = None
+            # We do this for failing fops on Slave
+            # Master should be logging this
+            if cmd_ret is None:
+                return False
+
+            if e.get("stat", {}):
+                # Copy actual UID/GID value back to entry stat
+                e['stat']['uid'] = uid
+                e['stat']['gid'] = gid
+
+            if cmd_ret in [EEXIST, ESTALE]:
+                if dst:
+                    en = e['entry1']
+                else:
+                    en = e['entry']
+                disk_gfid = get_gfid_from_mnt(en)
+                if isinstance(disk_gfid, str) and \
+                   e['gfid'] != disk_gfid:
+                    slv_entry_info['gfid_mismatch'] = True
+                    st = lstat(en)
+                    if not isinstance(st, int):
+                        if st and stat.S_ISDIR(st.st_mode):
+                            slv_entry_info['slave_isdir'] = True
+                            dir_name = get_slv_dir_path(slv_host, slv_volume,
+                                                        disk_gfid)
+                            slv_entry_info['slave_name'] = dir_name
+                        else:
+                            slv_entry_info['slave_isdir'] = False
+                    slv_entry_info['slave_gfid'] = disk_gfid
+                    failures.append((e, cmd_ret, slv_entry_info))
+                else:
+                    return False
+            else:
+                failures.append((e, cmd_ret, slv_entry_info))
+
+            return True
+
+        failures = []
+
+        def recursive_rmdir(gfid, entry, path):
+            """disk_gfid check added for original path for which
+            recursive_delete is called. This disk gfid check executed
+            before every Unlink/Rmdir. If disk gfid is not matching
+            with GFID from Changelog, that means other worker
+            deleted the directory. Even if the subdir/file present,
+            it belongs to different parent. Exit without performing
+            further deletes.
+            """
+            if not matching_disk_gfid(gfid, entry):
+                return
+
+            names = []
+            names = errno_wrap(os.listdir, [path], [ENOENT], [ESTALE, ENOTSUP])
+            if isinstance(names, int):
+                return
+
+            for name in names:
+                fullname = os.path.join(path, name)
+                if not matching_disk_gfid(gfid, entry):
+                    return
+                er = errno_wrap(os.remove, [fullname], [ENOENT, ESTALE,
+                                                        EISDIR], [EBUSY])
+
+                if er == EISDIR:
+                    recursive_rmdir(gfid, entry, fullname)
+
+            if not matching_disk_gfid(gfid, entry):
+                return
+
+            errno_wrap(os.rmdir, [path], [ENOENT, ESTALE], [EBUSY])
+
+        def rename_with_disk_gfid_confirmation(gfid, entry, en, uid, gid):
+            if not matching_disk_gfid(gfid, entry):
+                logging.error(lf("RENAME ignored: source entry does not match "
+                                 "with on-disk gfid",
+                                 source=entry,
+                                 gfid=gfid,
+                                 disk_gfid=get_gfid_from_mnt(entry),
+                                 target=en))
+                collect_failure(e, EEXIST, uid, gid)
+                return
+
+            cmd_ret = errno_wrap(os.rename,
+                                 [entry, en],
+                                 [ENOENT, EEXIST], [ESTALE, EBUSY])
+            collect_failure(e, cmd_ret, uid, gid)
+
+        for e in entries:
+            blob = None
+            op = e['op']
+            gfid = e['gfid']
+            entry = e['entry']
+            uid = 0
+            gid = 0
+
+            # Skip entry processing if it's marked true during gfid
+            # conflict resolution
+            if e['skip_entry']:
+                continue
+
+            if e.get("stat", {}):
+                # Copy UID/GID value and then reset to zero. Copied UID/GID
+                # will be used to run chown once entry is created.
+                uid = e['stat']['uid']
+                gid = e['stat']['gid']
+                e['stat']['uid'] = 0
+                e['stat']['gid'] = 0
+
+            (pg, bname) = entry2pb(entry)
+            if op in ['RMDIR', 'UNLINK']:
+                # Try once, if rmdir failed with ENOTEMPTY
+                # then delete recursively.
+                er = entry_purge(op, entry, gfid, e, uid, gid)
+                if isinstance(er, int):
+                    if er == ENOTEMPTY and op == 'RMDIR':
+                        # Retry if ENOTEMPTY, ESTALE
+                        er1 = errno_wrap(recursive_rmdir,
+                                         [gfid, entry,
+                                          os.path.join(pg, bname)],
+                                         [], [ENOTEMPTY, ESTALE, ENODATA])
+                        if not isinstance(er1, int):
+                            logging.debug("Removed %s => %s/%s recursively" %
+                                          (gfid, pg, bname))
+                        else:
+                            logging.warn(lf("Recursive remove failed",
+                                            gfid=gfid,
+                                            pgfid=pg,
+                                            bname=bname,
+                                            error=os.strerror(er1)))
+                    else:
+                        logging.warn(lf("Failed to remove",
+                                        gfid=gfid,
+                                        pgfid=pg,
+                                        bname=bname,
+                                        error=os.strerror(er)))
+            elif op in ['CREATE', 'MKNOD']:
+                slink = os.path.join(pfx, gfid)
+                st = lstat(slink)
+                # don't create multiple entries with same gfid
+                if isinstance(st, int):
+                    blob = entry_pack_reg(cls, gfid, bname,
+                                          e['mode'], e['uid'], e['gid'])
+                # Self healed hardlinks are recorded as MKNOD.
+                # So if the gfid already exists, it should be
+                # processed as hard link not mknod.
+                elif op in ['MKNOD']:
+                    cmd_ret = errno_wrap(os.link,
+                                         [slink, entry],
+                                         [ENOENT, EEXIST], [ESTALE])
+                    collect_failure(e, cmd_ret, uid, gid)
+            elif op == 'MKDIR':
+                en = e['entry']
+                slink = os.path.join(pfx, gfid)
+                st = lstat(slink)
+                # don't create multiple entries with same gfid
+                if isinstance(st, int):
+                    blob = entry_pack_mkdir(cls, gfid, bname,
+                                            e['mode'], e['uid'], e['gid'])
+                elif (isinstance(lstat(en), int) or
+                      not matching_disk_gfid(gfid, en)):
+                    # If gfid of a directory exists on slave but path based
+                    # create is getting EEXIST. This means the directory is
+                    # renamed in master but recorded as MKDIR during hybrid
+                    # crawl. Get the directory path by reading the backend
+                    # symlink and trying to rename to new name as said by
+                    # master.
+                    logging.info(lf("Special case: rename on mkdir",
+                                    gfid=gfid, entry=repr(entry)))
+                    src_entry = get_slv_dir_path(slv_host, slv_volume, gfid)
+                    if src_entry is None:
+                        collect_failure(e, ENOENT, uid, gid)
+                    if src_entry is not None and src_entry != entry:
+                        slv_entry_info = {}
+                        slv_entry_info['gfid_mismatch'] = False
+                        slv_entry_info['name_mismatch'] = True
+                        slv_entry_info['dst'] = False
+                        slv_entry_info['slave_isdir'] = True
+                        slv_entry_info['slave_gfid'] = gfid
+                        slv_entry_info['slave_entry'] = src_entry
+
+                        failures.append((e, EEXIST, slv_entry_info))
+            elif op == 'LINK':
+                slink = os.path.join(pfx, gfid)
+                st = lstat(slink)
+                if isinstance(st, int):
+                    (pg, bname) = entry2pb(entry)
+                    if stat.S_ISREG(e['stat']['mode']):
+                        blob = entry_pack_reg_stat(cls, gfid, bname, e['stat'])
+                    elif stat.S_ISLNK(e['stat']['mode']):
+                        blob = entry_pack_symlink(cls, gfid, bname, e['link'],
+                                                  e['stat'])
+                else:
+                    cmd_ret = errno_wrap(os.link,
+                                         [slink, entry],
+                                         [ENOENT, EEXIST], [ESTALE])
+                    collect_failure(e, cmd_ret, uid, gid)
+            elif op == 'SYMLINK':
+                en = e['entry']
+                st = lstat(entry)
+                if isinstance(st, int):
+                    blob = entry_pack_symlink(cls, gfid, bname, e['link'],
+                                              e['stat'])
+                elif not matching_disk_gfid(gfid, en):
+                    collect_failure(e, EEXIST, uid, gid)
+            elif op == 'RENAME':
+                en = e['entry1']
+                # The matching disk gfid check validates two things
+                #  1. Validates name is present, return false otherwise
+                #  2. Validates gfid is same, returns false otherwise
+                # So both validations are necessary to decide src doesn't
+                # exist. We can't rely on only gfid stat as hardlink could
+                # be present and we can't rely only on name as name could
+                # exist with different gfid.
+                if not matching_disk_gfid(gfid, entry):
+                    if e['stat'] and not stat.S_ISDIR(e['stat']['mode']):
+                        if stat.S_ISLNK(e['stat']['mode']):
+                            # src is not present, so don't sync symlink as
+                            # we don't know target. It's ok to ignore. If
+                            # it's unliked, it's fine. If it's renamed to
+                            # something else, it will be synced then.
+                            if e['link'] is not None:
+                                st1 = lstat(en)
+                                if isinstance(st1, int):
+                                    (pg, bname) = entry2pb(en)
+                                    blob = entry_pack_symlink(cls, gfid, bname,
+                                                              e['link'],
+                                                              e['stat'])
+                                elif not matching_disk_gfid(gfid, en):
+                                    collect_failure(e, EEXIST, uid, gid, True)
+                        else:
+                            slink = os.path.join(pfx, gfid)
+                            st = lstat(slink)
+                            # don't create multiple entries with same gfid
+                            if isinstance(st, int):
+                                (pg, bname) = entry2pb(en)
+                                blob = entry_pack_reg_stat(cls, gfid, bname,
+                                                           e['stat'])
+                            else:
+                                cmd_ret = errno_wrap(os.link, [slink, en],
+                                                    [ENOENT, EEXIST], [ESTALE])
+                                collect_failure(e, cmd_ret, uid, gid)
+                else:
+                    st = lstat(entry)
+                    st1 = lstat(en)
+                    if isinstance(st1, int):
+                        rename_with_disk_gfid_confirmation(gfid, entry, en,
+                                                           uid, gid)
+                    else:
+                        if st.st_ino == st1.st_ino:
+                            # we have a hard link, we can now unlink source
+                            try:
+                                errno_wrap(os.unlink, [entry],
+                                           [ENOENT, ESTALE], [EBUSY])
+                            except OSError as e:
+                                if e.errno == EISDIR:
+                                    try:
+                                        errno_wrap(os.rmdir, [entry],
+                                                   [ENOENT, ESTALE], [EBUSY])
+                                    except OSError as e:
+                                        if e.errno == ENOTEMPTY:
+                                            logging.error(
+                                                lf("Directory Rename failed. "
+                                                   "Both Old and New"
+                                                   " directories exists",
+                                                   old=entry,
+                                                   new=en))
+                                        else:
+                                            raise
+                                else:
+                                    raise
+                        elif not matching_disk_gfid(gfid, en) and dist_count > 1:
+                            collect_failure(e, EEXIST, uid, gid, True)
+                        else:
+                            # We are here which means matching_disk_gfid for
+                            # both source and destination has returned false
+                            # and distribution count for master vol is greater
+                            # then one. Which basically says both the source and
+                            # destination exist and not hardlinks.
+                            # So we are safe to go ahead with rename here.
+                            rename_with_disk_gfid_confirmation(gfid, entry, en,
+                                                               uid, gid)
+            if blob:
+                cmd_ret = errno_wrap(Xattr.lsetxattr,
+                                     [pg, 'glusterfs.gfid.newfile', blob],
+                                     [EEXIST, ENOENT, ESTALE],
+                                     [ESTALE, EINVAL, EBUSY])
+                collect_failure(e, cmd_ret, uid, gid)
+
+                # If UID/GID is different than zero that means we are trying
+                # create Entry with different UID/GID. Create Entry with
+                # UID:0 and GID:0, and then call chown to set UID/GID
+                if uid != 0 or gid != 0:
+                    path = os.path.join(pfx, gfid)
+                    cmd_ret = errno_wrap(os.lchown, [path, uid, gid], [ENOENT],
+                                         [ESTALE, EINVAL])
+                    collect_failure(e, cmd_ret, uid, gid)
+
+        return failures
+
+    @classmethod
+    def meta_ops(cls, meta_entries):
+        logging.debug('Meta-entries: %s' % repr(meta_entries))
+        failures = []
+        for e in meta_entries:
+            mode = e['stat']['mode']
+            uid = e['stat']['uid']
+            gid = e['stat']['gid']
+            atime = e['stat']['atime']
+            mtime = e['stat']['mtime']
+            go = e['go']
+            # Linux doesn't support chmod on symlink itself.
+            # It is always applied to the target file. So
+            # changelog would record target file's gfid
+            # and we are good. But 'chown' is supported on
+            # symlink file. So changelog would record symlink
+            # gfid in such cases. Since we do 'chown' 'chmod'
+            # 'utime' for each gfid recorded for metadata, and
+            # we know from changelog the metadata is on symlink's
+            # gfid or target file's gfid, we should be doing
+            # 'lchown' 'lchmod' 'utime with no-deference' blindly.
+            # But since 'lchmod' and 'utime with no de-reference' is
+            # not supported in python3, we have to rely on 'chmod'
+            # and 'utime with de-reference'. Hence avoiding 'chmod'
+            # and 'utime' if it's symlink file.
+
+            is_symlink = False
+            cmd_ret = errno_wrap(os.lchown, [go, uid, gid], [ENOENT],
+                                 [ESTALE, EINVAL])
+            if isinstance(cmd_ret, int):
+                continue
+
+            is_symlink = os.path.islink(go)
+
+            if not is_symlink:
+                cmd_ret = errno_wrap(os.chmod, [go, mode],
+                                     [ENOENT, EACCES, EPERM], [ESTALE, EINVAL])
+                if isinstance(cmd_ret, int):
+                    failures.append((e, cmd_ret, "chmod"))
+
+                cmd_ret = errno_wrap(os.utime, [go, (atime, mtime)],
+                                     [ENOENT, EACCES, EPERM], [ESTALE, EINVAL])
+                if isinstance(cmd_ret, int):
+                    failures.append((e, cmd_ret, "utime"))
+        return failures
+
+    @classmethod
+    @_pathguard
+    def setattr(cls, path, adct):
+        """set file attributes
+
+        @adct is a dict, where 'own', 'mode' and 'times'
+        keys are looked for and values used to perform
+        chown, chmod or utimes on @path.
+        """
+        own = adct.get('own')
+        if own:
+            os.lchown(path, *own)
+        mode = adct.get('mode')
+        if mode:
+            os.chmod(path, stat.S_IMODE(mode))
+        times = adct.get('times')
+        if times:
+            os.utime(path, times)
+
+    @staticmethod
+    def pid():
+        return os.getpid()
+
+    last_keep_alive = 0
+
+    @classmethod
+    def keep_alive(cls, dct):
+        """process keepalive messages.
+
+        Return keep-alive counter (number of received keep-alive
+        messages).
+
+        Now the "keep-alive" message can also have a payload which is
+        used to set a foreign volume-mark on the underlying file system.
+        """
+        if dct:
+            key = '.'.join([cls.GX_NSPACE, 'volume-mark', dct['uuid']])
+            val = struct.pack(cls.FRGN_FMTSTR,
+                              *(dct['version'] +
+                                tuple(int(x, 16)
+                                      for x in re.findall('(?:[\da-f]){2}',
+                                                          dct['uuid'])) +
+                                (dct['retval'],) + dct['volume_mark'][0:2] + (
+                                    dct['timeout'],)))
+            Xattr.lsetxattr('.', key, val)
+        cls.last_keep_alive += 1
+        return cls.last_keep_alive
+
+    @staticmethod
+    def version():
+        """version used in handshake"""
+        return 1.0
+
+
+class Mounter(object):
+
+    """Abstract base class for mounter backends"""
+
+    def __init__(self, params):
+        self.params = params
+        self.mntpt = None
+        self.umount_cmd = []
+
+    @classmethod
+    def get_glusterprog(cls):
+        gluster_cmd_dir = gconf.get("gluster-command-dir")
+        if rconf.args.subcmd == "slave":
+            gluster_cmd_dir = gconf.get("slave-gluster-command-dir")
+        return os.path.join(gluster_cmd_dir, cls.glusterprog)
+
+    def umount_l(self, d):
+        """perform lazy umount"""
+        po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        po.wait()
+        return po
+
+    @classmethod
+    def make_umount_argv(cls, d):
+        raise NotImplementedError
+
+    def make_mount_argv(self, label=None):
+        raise NotImplementedError
+
+    def cleanup_mntpt(self, *a):
+        pass
+
+    def handle_mounter(self, po):
+        po.wait()
+
+    def inhibit(self, label):
+        """inhibit a gluster filesystem
+
+        Mount glusterfs over a temporary mountpoint,
+        change into the mount, and lazy unmount the
+        filesystem.
+        """
+        mpi, mpo = pipe()
+        mh = Popen.fork()
+        if mh:
+            # Parent
+            os.close(mpi)
+            fcntl.fcntl(mpo, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+            d = None
+            margv = self.make_mount_argv(label)
+            if self.mntpt:
+                # mntpt is determined pre-mount
+                d = self.mntpt
+                mnt_msg = d + '\0'
+                encoded_msg = mnt_msg.encode()
+                os.write(mpo, encoded_msg)
+            po = Popen(margv, **self.mountkw)
+            self.handle_mounter(po)
+            po.terminate_geterr()
+            logging.debug('auxiliary glusterfs mount in place')
+            if not d:
+                # mntpt is determined during mount
+                d = self.mntpt
+                mnt_msg = d + '\0'
+                encoded_msg = mnt_msg.encode()
+                os.write(mpo, encoded_msg)
+            encoded_msg = 'M'.encode()
+            os.write(mpo, encoded_msg)
+            t = syncdutils.Thread(target=lambda: os.chdir(d))
+            t.start()
+            tlim = rconf.starttime + gconf.get("connection-timeout")
+            while True:
+                if not t.isAlive():
+                    break
+
+                if time.time() >= tlim:
+                    syncdutils.finalize(exval=1)
+                time.sleep(1)
+            os.close(mpo)
+            _, rv = syncdutils.waitpid(mh, 0)
+            if rv:
+                rv = (os.WIFEXITED(rv) and os.WEXITSTATUS(rv) or 0) - \
+                     (os.WIFSIGNALED(rv) and os.WTERMSIG(rv) or 0)
+                logging.warn(lf('stale mount possibly left behind',
+                                path=d))
+                raise GsyncdError("cleaning up temp mountpoint %s "
+                                  "failed with status %d" %
+                                  (d, rv))
+        else:
+            rv = 0
+            try:
+                os.setsid()
+                os.close(mpo)
+                mntdata = ''
+                while True:
+                    c = os.read(mpi, 1)
+                    c = c.decode()
+                    if not c:
+                        break
+                    mntdata += c
+                if mntdata:
+                    mounted = False
+                    if mntdata[-1] == 'M':
+                        mntdata = mntdata[:-1]
+                        assert(mntdata)
+                        mounted = True
+                    assert(mntdata[-1] == '\0')
+                    mntpt = mntdata[:-1]
+                    assert(mntpt)
+
+                    umount_master = False
+                    umount_slave = False
+                    if rconf.args.subcmd == "worker" \
+                       and not unshare_propagation_supported() \
+                       and not gconf.get("access-mount"):
+                        umount_master = True
+                    if rconf.args.subcmd == "slave" \
+                       and not gconf.get("slave-access-mount"):
+                        umount_slave = True
+
+                    if mounted and (umount_master or umount_slave):
+                        po = self.umount_l(mntpt)
+                        po.terminate_geterr(fail_on_err=False)
+                        if po.returncode != 0:
+                            po.errlog()
+                            rv = po.returncode
+                        logging.debug("Lazy umount done: %s" % mntpt)
+                    if umount_master or umount_slave:
+                        self.cleanup_mntpt(mntpt)
+            except:
+                logging.exception('mount cleanup failure:')
+                rv = 200
+            os._exit(rv)
+
+        #Polling the dht.subvol.status value.
+        RETRIES = 10
+        while not gf_mount_ready():
+            if RETRIES < 0:
+                logging.error('Subvols are not up')
+                break
+            RETRIES -= 1
+            time.sleep(0.2)
+
+        logging.debug('auxiliary glusterfs mount prepared')
+
+
+class DirectMounter(Mounter):
+
+    """mounter backend which calls mount(8), umount(8) directly"""
+
+    mountkw = {'stderr': subprocess.PIPE, 'universal_newlines': True}
+    glusterprog = 'glusterfs'
+
+    @staticmethod
+    def make_umount_argv(d):
+        return ['umount', '-l', d]
+
+    def make_mount_argv(self, label=None):
+        self.mntpt = tempfile.mkdtemp(prefix='gsyncd-aux-mount-')
+        rconf.mount_point = self.mntpt
+        return [self.get_glusterprog()] + \
+            ['--' + p for p in self.params] + [self.mntpt]
+
+    def cleanup_mntpt(self, mntpt=None):
+        if not mntpt:
+            mntpt = self.mntpt
+        errno_wrap(os.rmdir, [mntpt], [ENOENT, EBUSY])
+
+
+class MountbrokerMounter(Mounter):
+
+    """mounter backend using the mountbroker gluster service"""
+
+    mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE,
+               'universal_newlines': True}
+    glusterprog = 'gluster'
+
+    @classmethod
+    def make_cli_argv(cls):
+        return [cls.get_glusterprog()] + ['--remote-host=localhost'] + \
+            gconf.get("gluster-cli-options").split() + ['system::']
+
+    @classmethod
+    def make_umount_argv(cls, d):
+        return cls.make_cli_argv() + ['umount', d, 'lazy']
+
+    def make_mount_argv(self, label):
+        return self.make_cli_argv() + \
+            ['mount', label, 'user-map-root=' +
+                syncdutils.getusername()] + self.params
+
+    def handle_mounter(self, po):
+        self.mntpt = po.stdout.readline()[:-1]
+        rconf.mount_point = self.mntpt
+        rconf.mountbroker = True
+        self.umount_cmd = self.make_cli_argv() + ['umount']
+        rconf.mbr_umount_cmd = self.umount_cmd
+        po.stdout.close()
+        sup(self, po)
+        if po.returncode != 0:
+            # if cli terminated with error due to being
+            # refused by glusterd, what it put
+            # out on stdout is a diagnostic message
+            logging.error(lf('glusterd answered', mnt=self.mntpt))
+
+
+class GLUSTERServer(Server):
+
+    "server enhancements for a glusterfs backend"""
+
+    @classmethod
+    def _attr_unpack_dict(cls, xattr, extra_fields=''):
+        """generic volume mark fetching/parsing backed"""
+        fmt_string = cls.NTV_FMTSTR + extra_fields
+        buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
+        buf = str_to_bytearray(buf)
+        vm = struct.unpack(fmt_string, buf)
+        m = re.match(
+            '(.{8})(.{4})(.{4})(.{4})(.{12})',
+            "".join(['%02x' % x for x in vm[2:18]]))
+        uuid = '-'.join(m.groups())
+        volinfo = {'version': vm[0:2],
+                   'uuid': uuid,
+                   'retval': vm[18],
+                   'volume_mark': vm[19:21],
+                   }
+        if extra_fields:
+            return volinfo, vm[-len(extra_fields):]
+        else:
+            return volinfo
+
+    @classmethod
+    def foreign_volume_infos(cls):
+        """return list of valid (not expired) foreign volume marks"""
+        dict_list = []
+        xattr_list = Xattr.llistxattr_buf('.')
+        for ele in xattr_list:
+            if ele.find('.'.join([cls.GX_NSPACE, 'volume-mark', ''])) == 0:
+                d, x = cls._attr_unpack_dict(ele, cls.FRGN_XTRA_FMT)
+                now = int(time.time())
+                if x[0] > now:
+                    logging.debug("volinfo[%s] expires: %d "
+                                  "(%d sec later)" %
+                                  (d['uuid'], x[0], x[0] - now))
+                    d['timeout'] = x[0]
+                    dict_list.append(d)
+                else:
+                    try:
+                        Xattr.lremovexattr('.', ele)
+                    except OSError:
+                        pass
+        return dict_list
+
+    @classmethod
+    def native_volume_info(cls):
+        """get the native volume mark of the underlying gluster volume"""
+        try:
+            return cls._attr_unpack_dict('.'.join([cls.GX_NSPACE,
+                                                   'volume-mark']))
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno != ENODATA:
+                raise
+
+
+class GLUSTER(object):
+
+    """scheme class for gluster:// urls
+
+    can be used to represent a gluster slave server
+    on slave side, or interface to a remote gluster
+    slave on master side, or to represent master
+    (slave-ish features come from the mixins, master
+    functionality is outsourced to GMaster from master)
+    """
+    server = GLUSTERServer
+
+    def __init__(self, host, volume):
+        self.path = "%s:%s" % (host, volume)
+        self.host = host
+        self.volume = volume
+
+        global slv_volume
+        global slv_host
+        slv_volume = self.volume
+        slv_host = self.host
+
+    def connect(self):
+        """inhibit the resource beyond
+
+        Choose mounting backend (direct or mountbroker),
+        set up glusterfs parameters and perform the mount
+        with given backend
+        """
+
+        logging.info("Mounting gluster volume locally...")
+        t0 = time.time()
+        label = gconf.get('mountbroker', None)
+        if not label and not privileged():
+            label = syncdutils.getusername()
+        mounter = label and MountbrokerMounter or DirectMounter
+
+        log_file = gconf.get("gluster-log-file")
+        if rconf.args.subcmd == "slave":
+            log_file = gconf.get("slave-gluster-log-file")
+
+        log_level = gconf.get("gluster-log-level")
+        if rconf.args.subcmd == "slave":
+            log_level = gconf.get("slave-gluster-log-level")
+
+        params = gconf.get("gluster-params").split() + \
+            ['log-level=' + log_level] + \
+            ['log-file=' + log_file, 'volfile-server=' + self.host] + \
+            ['volfile-id=' + self.volume, 'client-pid=-1']
+
+        self.mounter = mounter(params)
+        self.mounter.inhibit(label)
+        logging.info(lf("Mounted gluster volume",
+                        duration="%.4f" % (time.time() - t0)))
+
+    def gmaster_instantiate_tuple(self, slave):
+        """return a tuple of the 'one shot' and the 'main crawl'
+        class instance"""
+        return (gmaster_builder('xsync')(self, slave),
+                gmaster_builder()(self, slave),
+                gmaster_builder('changeloghistory')(self, slave))
+
+    def service_loop(self, slave=None):
+        """enter service loop
+
+        - if slave given, instantiate GMaster and
+          pass control to that instance, which implements
+          master behavior
+        - else do that's what's inherited
+        """
+        if rconf.args.subcmd == "slave":
+            if gconf.get("use-rsync-xattrs") and not privileged():
+                raise GsyncdError(
+                    "using rsync for extended attributes is not supported")
+
+            repce = RepceServer(
+                self.server, sys.stdin, sys.stdout, gconf.get("sync-jobs"))
+            t = syncdutils.Thread(target=lambda: (repce.service_loop(),
+                                                  syncdutils.finalize()))
+            t.start()
+            logging.info("slave listening")
+            if gconf.get("slave-timeout") and gconf.get("slave-timeout") > 0:
+                while True:
+                    lp = self.server.last_keep_alive
+                    time.sleep(gconf.get("slave-timeout"))
+                    if lp == self.server.last_keep_alive:
+                        logging.info(
+                            lf("connection inactive, stopping",
+                               timeout=gconf.get("slave-timeout")))
+                        break
+            else:
+                select((), (), ())
+
+            return
+
+        class brickserver(Server):
+            local_path = rconf.args.local_path
+            aggregated = self.server
+
+            @classmethod
+            def entries(cls, path):
+                e = super(brickserver, cls).entries(path)
+                # on the brick don't mess with /.glusterfs
+                if path == '.':
+                    try:
+                        e.remove('.glusterfs')
+                        e.remove('.trashcan')
+                    except ValueError:
+                        pass
+                return e
+
+            @classmethod
+            def lstat(cls, e):
+                """ path based backend stat """
+                return super(brickserver, cls).lstat(e)
+
+            @classmethod
+            def gfid(cls, e):
+                """ path based backend gfid fetch """
+                return super(brickserver, cls).gfid(e)
+
+            @classmethod
+            def linkto_check(cls, e):
+                return super(brickserver, cls).linkto_check(e)
+
+        # define {,set_}xtime in slave, thus preempting
+        # the call to remote, so that it takes data from
+        # the local brick
+        slave.server.xtime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.xtime(path,
+                                  uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.stime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.stime(path,
+                                  uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.entry_stime = types.MethodType(
+            lambda _self, path, uuid: (
+                brickserver.entry_stime(
+                    path,
+                    uuid + '.' + rconf.args.slave_id)
+            ),
+            slave.server)
+        slave.server.set_stime = types.MethodType(
+            lambda _self, path, uuid, mark: (
+                brickserver.set_stime(path,
+                                      uuid + '.' + rconf.args.slave_id,
+                                      mark)
+            ),
+            slave.server)
+        slave.server.set_entry_stime = types.MethodType(
+            lambda _self, path, uuid, mark: (
+                brickserver.set_entry_stime(
+                    path,
+                    uuid + '.' + rconf.args.slave_id,
+                    mark)
+            ),
+            slave.server)
+
+        (g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
+        g1.master.server = brickserver
+        g2.master.server = brickserver
+        g3.master.server = brickserver
+
+        # bad bad bad: bad way to do things like this
+        # need to make this elegant
+        # register the crawlers and start crawling
+        # g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
+        # g3 ==> changelog History
+        status = GeorepStatus(gconf.get("state-file"),
+                              rconf.args.local_node,
+                              rconf.args.local_path,
+                              rconf.args.local_node_id,
+                              rconf.args.master,
+                              rconf.args.slave)
+        status.reset_on_worker_start()
+
+        try:
+            workdir = g2.setup_working_dir()
+            # Register only when change_detector is not set to
+            # xsync, else agent will generate changelog files
+            # in .processing directory of working dir
+            if gconf.get("change-detector") != 'xsync':
+                # register with the changelog library
+                # 9 == log level (DEBUG)
+                # 5 == connection retries
+                libgfchangelog.register(rconf.args.local_path,
+                                        workdir,
+                                        gconf.get("changelog-log-file"),
+                                        get_changelog_log_level(
+                                            gconf.get("changelog-log-level")),
+                                        g2.CHANGELOG_CONN_RETRIES)
+
+            register_time = int(time.time())
+            g2.register(register_time, status)
+            g3.register(register_time, status)
+        except ChangelogException as e:
+            logging.error(lf("Changelog register failed", error=e))
+            sys.exit(1)
+
+        g1.register(status=status)
+        logging.info(lf("Register time",
+                        time=register_time))
+        # oneshot: Try to use changelog history api, if not
+        # available switch to FS crawl
+        # Note: if config.change_detector is xsync then
+        # it will not use changelog history api
+        try:
+            g3.crawlwrap(oneshot=True)
+        except PartialHistoryAvailable as e:
+            logging.info(lf('Partial history available, using xsync crawl'
+                            ' after consuming history',
+                            till=e))
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except ChangelogHistoryNotAvailable:
+            logging.info('Changelog history not available, using xsync')
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except NoStimeAvailable:
+            logging.info('No stime available, using xsync crawl')
+            g1.crawlwrap(oneshot=True, register_time=register_time)
+        except ChangelogException as e:
+            logging.error(lf("Changelog History Crawl failed",
+                             error=e))
+            sys.exit(1)
+
+        try:
+            g2.crawlwrap()
+        except ChangelogException as e:
+            logging.error(lf("Changelog crawl failed", error=e))
+            sys.exit(1)
+
+
+class SSH(object):
+
+    """scheme class for ssh:// urls
+
+    interface to remote slave on master side
+    implementing an ssh based proxy
+    """
+
+    def __init__(self, host, volume):
+        self.remote_addr = host
+        self.volume = volume
+
+    @staticmethod
+    def parse_ssh_address(self):
+        m = re.match('([^@]+)@(.+)', self.remote_addr)
+        if m:
+            u, h = m.groups()
+        else:
+            u, h = syncdutils.getusername(), self.remote_addr
+        self.remotehost = h
+        return {'user': u, 'host': h}
+
+    def start_fd_client(self, i, o):
+        """set up RePCe client, handshake with server
+
+        It's cut out as a separate method to let
+        subclasses hook into client startup
+        """
+        self.server = RepceClient(i, o)
+        rv = self.server.__version__()
+        exrv = {'proto': repce.repce_version, 'object': Server.version()}
+        da0 = (rv, exrv)
+        da1 = ({}, {})
+        for i in range(2):
+            for k, v in da0[i].items():
+                da1[i][k] = int(v)
+        if da1[0] != da1[1]:
+            raise GsyncdError(
+                "RePCe major version mismatch: local %s, remote %s" %
+                (exrv, rv))
+        slavepath = "/proc/%d/cwd" % self.server.pid()
+        self.slaveurl = ':'.join([self.remote_addr, slavepath])
+
+    def connect_remote(self):
+        """connect to inner slave url through outer ssh url
+
+        Wrap the connecting utility in ssh.
+
+        Much care is put into daemonizing: in that case
+        ssh is started before daemonization, but
+        RePCe client is to be created after that (as ssh
+        interactive password auth would be defeated by
+        a daemonized ssh, while client should be present
+        only in the final process). In that case the action
+        is taken apart to two parts, this method is ivoked
+        once pre-daemon, once post-daemon. Use @go_daemon
+        to deiced what part to perform.
+
+        [NB. ATM gluster product does not makes use of interactive
+        authentication.]
+        """
+        syncdutils.setup_ssh_ctl(tempfile.mkdtemp(prefix='gsyncd-aux-ssh-'),
+                                 self.remote_addr,
+                                 self.volume)
+
+        logging.info("Initializing SSH connection between master and slave...")
+        t0 = time.time()
+
+        extra_opts = []
+        remote_gsyncd = gconf.get("remote-gsyncd")
+        if remote_gsyncd == "":
+            remote_gsyncd = "/nonexistent/gsyncd"
+
+        if gconf.get("use-rsync-xattrs"):
+            extra_opts.append('--use-rsync-xattrs')
+
+        args_to_slave = [gconf.get("ssh-command")] + \
+            gconf.get("ssh-options").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            rconf.ssh_ctl_args + [self.remote_addr] + \
+            [remote_gsyncd, "slave"] + \
+            extra_opts + \
+            [rconf.args.master, rconf.args.slave] + \
+            [
+                '--master-node', rconf.args.local_node,
+                '--master-node-id', rconf.args.local_node_id,
+                '--master-brick', rconf.args.local_path,
+                '--local-node', rconf.args.resource_remote,
+                '--local-node-id', rconf.args.resource_remote_id] + \
+            [
+                # Add all config arguments here, slave gsyncd will not use
+                # config file in slave side, so all overriding options should
+                # be sent as arguments
+                '--slave-timeout', str(gconf.get("slave-timeout")),
+                '--slave-log-level', gconf.get("slave-log-level"),
+                '--slave-gluster-log-level',
+                gconf.get("slave-gluster-log-level"),
+                '--slave-gluster-command-dir',
+                gconf.get("slave-gluster-command-dir"),
+                '--master-dist-count',
+                str(gconf.get("master-distribution-count"))]
+
+        if gconf.get("slave-access-mount"):
+            args_to_slave.append('--slave-access-mount')
+
+        if rconf.args.debug:
+            args_to_slave.append('--debug')
+
+        po = Popen(args_to_slave,
+                   stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE)
+        rconf.transport = po
+        self.start_fd_client(po.stdout, po.stdin)
+        logging.info(lf("SSH connection between master and slave established.",
+                        duration="%.4f" % (time.time() - t0)))
+
+    def rsync(self, files, *args, **kw):
+        """invoke rsync"""
+        if not files:
+            raise GsyncdError("no files to sync")
+        logging.debug("files: " + ", ".join(files))
+
+        extra_rsync_flags = []
+        # Performance flag, --ignore-missing-args, if rsync version is
+        # greater than 3.1.0 then include this flag.
+        if gconf.get("rsync-opt-ignore-missing-args") and \
+           get_rsync_version(gconf.get("rsync-command")) >= "3.1.0":
+            extra_rsync_flags = ["--ignore-missing-args"]
+
+        rsync_ssh_opts = [gconf.get("ssh-command")] + \
+            gconf.get("ssh-options").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            rconf.ssh_ctl_args + \
+            gconf.get("rsync-ssh-options").split()
+
+        argv = [
+            gconf.get("rsync-command"),
+            '-aR0',
+            '--inplace',
+            '--files-from=-',
+            '--super',
+            '--stats',
+            '--numeric-ids',
+            '--no-implied-dirs'
+        ]
+
+        if gconf.get("rsync-opt-existing"):
+            argv += ["--existing"]
+
+        if gconf.get("sync-xattrs"):
+            argv += ['--xattrs']
+
+        if gconf.get("sync-acls"):
+            argv += ['--acls']
+
+        argv = argv + \
+            gconf.get("rsync-options").split() + \
+            extra_rsync_flags + ['.'] + \
+            ["-e", " ".join(rsync_ssh_opts)] + \
+            [self.slaveurl]
+
+        log_rsync_performance = gconf.getr("log-rsync-performance", False)
+
+        if log_rsync_performance:
+            # use stdout=PIPE only when log_rsync_performance enabled
+            # Else rsync will write to stdout and nobody is there
+            # to consume. If PIPE is full rsync hangs.
+            po = Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, universal_newlines=True)
+        else:
+            po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
+                       universal_newlines=True)
+
+        for f in files:
+            po.stdin.write(f)
+            po.stdin.write('\0')
+
+        stdout, stderr = po.communicate()
+
+        if kw.get("log_err", False):
+            for errline in stderr.strip().split("\n")[:-1]:
+                logging.error(lf("SYNC Error",
+                                 sync_engine="Rsync",
+                                 error=errline))
+
+        if log_rsync_performance:
+            rsync_msg = []
+            for line in stdout.split("\n"):
+                if line.startswith("Number of files:") or \
+                   line.startswith("Number of regular files transferred:") or \
+                   line.startswith("Total file size:") or \
+                   line.startswith("Total transferred file size:") or \
+                   line.startswith("Literal data:") or \
+                   line.startswith("Matched data:") or \
+                   line.startswith("Total bytes sent:") or \
+                   line.startswith("Total bytes received:") or \
+                   line.startswith("sent "):
+                    rsync_msg.append(line)
+            logging.info(lf("rsync performance",
+                            data=", ".join(rsync_msg)))
+
+        return po
+
+    def tarssh(self, files, log_err=False):
+        """invoke tar+ssh
+        -z (compress) can be use if needed, but omitting it now
+        as it results in weird error (tar+ssh errors out (errcode: 2)
+        """
+        if not files:
+            raise GsyncdError("no files to sync")
+        logging.debug("files: " + ", ".join(files))
+        (host, rdir) = self.slaveurl.split(':')
+
+        tar_cmd = ["tar"] + \
+            ["--sparse", "-cf", "-", "--files-from", "-"]
+        ssh_cmd = gconf.get("ssh-command").split() + \
+            gconf.get("ssh-options-tar").split() + \
+            ["-p", str(gconf.get("ssh-port"))] + \
+            [host, "tar"] + \
+            ["--overwrite", "-xf", "-", "-C", rdir]
+        p0 = Popen(tar_cmd, stdout=subprocess.PIPE,
+                   stdin=subprocess.PIPE, stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        p1 = Popen(ssh_cmd, stdin=p0.stdout, stderr=subprocess.PIPE,
+                   universal_newlines=True)
+        for f in files:
+            p0.stdin.write(f)
+            p0.stdin.write('\n')
+
+        p0.stdin.close()
+        p0.stdout.close()  # Allow p0 to receive a SIGPIPE if p1 exits.
+
+        # stdin and stdout of p0 is already closed, Reset to None and
+        # wait for child process to complete
+        p0.stdin = None
+        p0.stdout = None
+
+        def wait_for_tar(p0):
+            _, stderr = p0.communicate()
+            if log_err:
+                for errline in stderr.strip().split("\n")[:-1]:
+                    if "No such file or directory" not in errline:
+                        logging.error(lf("SYNC Error",
+                                         sync_engine="Tarssh",
+                                         error=errline))
+
+        t = syncdutils.Thread(target=wait_for_tar, args=(p0, ))
+        # wait for tar to terminate, collecting any errors, further
+        # waiting for transfer to complete
+        t.start()
+
+        # wait for ssh process
+        _, stderr1 = p1.communicate()
+        t.join()
+
+        if log_err:
+            for errline in stderr1.strip().split("\n")[:-1]:
+                logging.error(lf("SYNC Error",
+                                 sync_engine="Tarssh",
+                                 error=errline))
+
+        return p1
diff --git a/geo-replication/syncdaemon/subcmds.py b/geo-replication/syncdaemon/subcmds.py
new file mode 100644
index 00000000000..b8508532e30
--- /dev/null
+++ b/geo-replication/syncdaemon/subcmds.py
@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+from syncdutils import lf
+import logging
+import gsyncdconfig as gconf
+
+
+ERROR_CONFIG_INVALID = 2
+ERROR_CONFIG_INVALID_VALUE = 3
+ERROR_CONFIG_NOT_CONFIGURABLE = 4
+
+
+def subcmd_monitor_status(args):
+    from gsyncdstatus import set_monitor_status
+    from rconf import rconf
+
+    set_monitor_status(gconf.get("state-file"), args.status)
+    rconf.log_exit = False
+    logging.info(lf("Monitor Status Change", status=args.status))
+
+
+def subcmd_status(args):
+    from gsyncdstatus import GeorepStatus
+
+    master_name = args.master.replace(":", "")
+    slave_data = args.slave.replace("ssh://", "")
+
+    brick_status = GeorepStatus(gconf.get("state-file"),
+                                "",
+                                args.local_path,
+                                "",
+                                master_name,
+                                slave_data,
+                                gconf.get("pid-file"))
+    checkpoint_time = gconf.get("checkpoint", 0)
+    brick_status.print_status(checkpoint_time=checkpoint_time,
+                              json_output=args.json)
+
+
+def subcmd_monitor(args):
+    import monitor
+    from resource import GLUSTER, SSH, Popen
+    go_daemon = False if args.debug else True
+
+    monitor.startup(go_daemon)
+    Popen.init_errhandler()
+    local = GLUSTER("localhost", args.master)
+    slavehost, slavevol = args.slave.split("::")
+    remote = SSH(slavehost, slavevol)
+    return monitor.monitor(local, remote)
+
+
+def subcmd_verify_spawning(args):
+    logging.info("Able to spawn gsyncd.py")
+
+
+def subcmd_worker(args):
+    import os
+    import fcntl
+
+    from resource import GLUSTER, SSH, Popen
+
+    Popen.init_errhandler()
+    fcntl.fcntl(args.feedback_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+    local = GLUSTER("localhost", args.master)
+    slave_url, slavevol = args.slave.split("::")
+    if "@" not in slave_url:
+        slavehost = args.resource_remote
+    else:
+        slavehost = "%s@%s" % (slave_url.split("@")[0], args.resource_remote)
+    remote = SSH(slavehost, slavevol)
+    remote.connect_remote()
+    local.connect()
+    logging.info("Worker spawn successful. Acknowledging back to monitor")
+    os.close(args.feedback_fd)
+    local.service_loop(remote)
+
+
+def subcmd_slave(args):
+    from resource import GLUSTER, Popen
+
+    Popen.init_errhandler()
+    slavevol = args.slave.split("::")[-1]
+    local = GLUSTER("localhost", slavevol)
+
+    local.connect()
+    local.service_loop()
+
+
+def subcmd_voluuidget(args):
+    from subprocess import Popen, PIPE
+    import xml.etree.ElementTree as XET
+
+    ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
+
+    cmd = ['gluster', '--xml', '--remote-host=' + args.host,
+           'volume', 'info', args.volname]
+
+    if args.inet6:
+        cmd.append("--inet6")
+
+    po = Popen(cmd, bufsize=0,
+               stdin=None, stdout=PIPE, stderr=PIPE,
+               universal_newlines=True)
+
+    vix, err = po.communicate()
+    if po.returncode != 0:
+        logging.info(lf("Volume info failed, unable to get "
+                        "volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=po.returncode))
+        return ""
+    vi = XET.fromstring(vix)
+    if vi.find('opRet').text != '0':
+        logging.info(lf("Unable to get volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=vi.find('opErrstr').text))
+        return ""
+
+    try:
+        voluuid = vi.find("volInfo/volumes/volume/id").text
+    except (ParseError, AttributeError, ValueError) as e:
+        logging.info(lf("Parsing failed to volume uuid of slavevol, "
+                        "returning empty string",
+                        slavevol=args.volname,
+                        slavehost=args.host,
+                        error=e))
+        voluuid = ""
+
+    print(voluuid)
+
+
+def _unlink(path):
+    import os
+    from errno import ENOENT
+    from syncdutils import GsyncdError
+    import sys
+
+    try:
+        os.unlink(path)
+    except (OSError, IOError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError('Unlink error: %s' % path)
+
+
+def subcmd_delete(args):
+    import logging
+    import shutil
+    import glob
+    import sys
+    from errno import ENOENT, ENODATA
+    import struct
+
+    from syncdutils import GsyncdError, Xattr, errno_wrap
+    import gsyncdconfig as gconf
+
+    logging.info('geo-replication delete')
+    # remove the stime xattr from all the brick paths so that
+    # a re-create of a session will start sync all over again
+    stime_xattr_prefix = gconf.get('stime-xattr-prefix', None)
+
+    # Delete pid file, status file, socket file
+    cleanup_paths = []
+    cleanup_paths.append(gconf.get("pid-file"))
+
+    # Cleanup Session dir
+    try:
+        shutil.rmtree(gconf.get("georep-session-working-dir"))
+    except (IOError, OSError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError(
+                'Error while removing working dir: %s' %
+                gconf.get("georep-session-working-dir"))
+
+    # Cleanup changelog working dirs
+    try:
+        shutil.rmtree(gconf.get("working-dir"))
+    except (IOError, OSError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError(
+                'Error while removing working dir: %s' %
+                gconf.get("working-dir"))
+
+    for path in cleanup_paths:
+        # To delete temp files
+        for f in glob.glob(path + "*"):
+            _unlink(f)
+
+    if args.reset_sync_time and stime_xattr_prefix:
+        for p in args.paths:
+            if p != "":
+                # set stime to (0,0) to trigger full volume content resync
+                # to slave on session recreation
+                # look at master.py::Xcrawl   hint: zero_zero
+                errno_wrap(Xattr.lsetxattr,
+                           (p, stime_xattr_prefix + ".stime",
+                            struct.pack("!II", 0, 0)),
+                           [ENOENT, ENODATA])
+                errno_wrap(Xattr.lremovexattr,
+                           (p, stime_xattr_prefix + ".entry_stime"),
+                           [ENOENT, ENODATA])
+
+    return
+
+
+def print_config(name, value, only_value=False, use_underscore=False):
+    val = value
+    if isinstance(value, bool):
+        val = str(value).lower()
+
+    if only_value:
+        print(val)
+    else:
+        if use_underscore:
+            name = name.replace("-", "_")
+
+        print(("%s:%s" % (name, val)))
+
+
+def config_name_format(val):
+    return val.replace("_", "-")
+
+
+def subcmd_config_get(args):
+    import sys
+    import json
+
+    all_config = gconf.getall(show_defaults=args.show_defaults,
+                              show_non_configurable=True)
+    if args.name is not None:
+        val = all_config.get(config_name_format(args.name), None)
+        if val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        print_config(args.name, val["value"], only_value=args.only_value,
+                     use_underscore=args.use_underscore)
+        return
+
+    if args.json:
+        out = []
+        # Convert all values as string
+        for k in sorted(all_config):
+            v = all_config[k]
+            out.append({
+                "name": k,
+                "value": str(v["value"]),
+                "default": str(v["default"]),
+                "configurable": v["configurable"],
+                "modified": v["modified"]
+            })
+
+        print((json.dumps(out)))
+        return
+
+    for k in sorted(all_config):
+        print_config(k, all_config[k]["value"],
+                     use_underscore=args.use_underscore)
+
+
+def subcmd_config_check(args):
+    import sys
+
+    try:
+        gconf.check(config_name_format(args.name), value=args.value,
+                    with_conffile=False)
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
+    except gconf.GconfInvalidValue:
+        sys.stderr.write("Invalid config value \"%s=%s\"\n" % (args.name,
+                                                               args.value))
+        sys.exit(ERROR_CONFIG_INVALID_VALUE)
+
+
+def subcmd_config_set(args):
+    import sys
+
+    try:
+        gconf.setconfig(config_name_format(args.name), args.value)
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
+    except gconf.GconfInvalidValue:
+        sys.stderr.write("Invalid config value \"%s=%s\"\n" % (args.name,
+                                                               args.value))
+        sys.exit(ERROR_CONFIG_INVALID_VALUE)
+
+
+def subcmd_config_reset(args):
+    import sys
+
+    try:
+        gconf.resetconfig(config_name_format(args.name))
+    except gconf.GconfNotConfigurable:
+        cnf_val = gconf.get(config_name_format(args.name), None)
+        if cnf_val is None:
+            sys.stderr.write("Invalid config name \"%s\"\n" % args.name)
+            sys.exit(ERROR_CONFIG_INVALID)
+
+        # Not configurable
+        sys.stderr.write("Not configurable \"%s\"\n" % args.name)
+        sys.exit(ERROR_CONFIG_NOT_CONFIGURABLE)
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
new file mode 100644
index 00000000000..a3df103e76c
--- /dev/null
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -0,0 +1,1115 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+import pwd
+import time
+import fcntl
+import shutil
+import logging
+import errno
+import threading
+import subprocess
+import socket
+from subprocess import PIPE
+from threading import Lock, Thread as baseThread
+from errno import (EACCES, EAGAIN, EPIPE, ENOTCONN, ENOMEM, ECONNABORTED,
+                   EINTR, ENOENT, ESTALE, EBUSY, ENODATA, errorcode, EIO)
+from signal import signal, SIGTERM
+import select as oselect
+from os import waitpid as owaitpid
+import xml.etree.ElementTree as XET
+from select import error as SelectError
+try:
+    from cPickle import PickleError
+except ImportError:
+    from pickle import PickleError
+
+from conf import GLUSTERFS_LIBEXECDIR, UUID_FILE
+sys.path.insert(1, GLUSTERFS_LIBEXECDIR)
+EVENTS_ENABLED = True
+try:
+    from gfevents.eventtypes import GEOREP_FAULTY as EVENT_GEOREP_FAULTY
+    from gfevents.eventtypes import GEOREP_ACTIVE as EVENT_GEOREP_ACTIVE
+    from gfevents.eventtypes import GEOREP_PASSIVE as EVENT_GEOREP_PASSIVE
+    from gfevents.eventtypes import GEOREP_CHECKPOINT_COMPLETED \
+        as EVENT_GEOREP_CHECKPOINT_COMPLETED
+except ImportError:
+    # Events APIs not installed, dummy eventtypes with None
+    EVENTS_ENABLED = False
+    EVENT_GEOREP_FAULTY = None
+    EVENT_GEOREP_ACTIVE = None
+    EVENT_GEOREP_PASSIVE = None
+    EVENT_GEOREP_CHECKPOINT_COMPLETED = None
+
+import gsyncdconfig as gconf
+from rconf import rconf
+
+from hashlib import sha256 as sha256
+
+ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
+
+# auxiliary gfid based access prefix
+_CL_AUX_GFID_PFX = ".gfid/"
+ROOT_GFID = "00000000-0000-0000-0000-000000000001"
+GF_OP_RETRIES = 10
+
+GX_GFID_CANONICAL_LEN = 37  # canonical gfid len + '\0'
+
+NodeID = None
+rsync_version = None
+unshare_mnt_propagation = None
+slv_bricks = None
+SPACE_ESCAPE_CHAR = "%20"
+NEWLINE_ESCAPE_CHAR = "%0A"
+PERCENTAGE_ESCAPE_CHAR = "%25"
+
+final_lock = Lock()
+
+def sup(x, *a, **kw):
+    """a rubyesque "super" for python ;)
+
+    invoke caller method in parent class with given args.
+    """
+    return getattr(super(type(x), x),
+                   sys._getframe(1).f_code.co_name)(*a, **kw)
+
+
+def escape(s):
+    """the chosen flavor of string escaping, used all over
+       to turn whatever data to creatable representation"""
+    return s.replace("/", "-").strip("-")
+
+
+def escape_space_newline(s):
+    return s.replace("%", PERCENTAGE_ESCAPE_CHAR)\
+            .replace(" ", SPACE_ESCAPE_CHAR)\
+            .replace("\n", NEWLINE_ESCAPE_CHAR)
+
+
+def unescape_space_newline(s):
+    return s.replace(SPACE_ESCAPE_CHAR, " ")\
+            .replace(NEWLINE_ESCAPE_CHAR, "\n")\
+            .replace(PERCENTAGE_ESCAPE_CHAR, "%")
+
+# gf_mount_ready() returns 1 if all subvols are up, else 0
+def gf_mount_ready():
+    ret = errno_wrap(Xattr.lgetxattr,
+                     ['.', 'dht.subvol.status', 16],
+                     [ENOENT, ENOTSUP, ENODATA], [ENOMEM])
+
+    if isinstance(ret, int):
+       logging.error("failed to get the xattr value")
+       return 1
+    ret = ret.rstrip('\x00')
+    if ret == "1":
+       return 1
+    return 0
+
+def norm(s):
+    if s:
+        return s.replace('-', '_')
+
+
+def update_file(path, updater, merger=lambda f: True):
+    """update a file in a transaction-like manner"""
+
+    fr = fw = None
+    try:
+        fd = os.open(path, os.O_CREAT | os.O_RDWR)
+        try:
+            fr = os.fdopen(fd, 'r+b')
+        except:
+            os.close(fd)
+            raise
+        fcntl.lockf(fr, fcntl.LOCK_EX)
+        if not merger(fr):
+            return
+
+        tmpp = path + '.tmp.' + str(os.getpid())
+        fd = os.open(tmpp, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        try:
+            fw = os.fdopen(fd, 'wb', 0)
+        except:
+            os.close(fd)
+            raise
+        updater(fw)
+        os.fsync(fd)
+        os.rename(tmpp, path)
+    finally:
+        for fx in (fr, fw):
+            if fx:
+                fx.close()
+
+
+def create_manifest(fname, content):
+    """
+    Create manifest file for SSH Control Path
+    """
+    fd = None
+    try:
+        fd = os.open(fname, os.O_CREAT | os.O_RDWR)
+        try:
+            os.write(fd, content)
+        except:
+            os.close(fd)
+            raise
+    finally:
+        if fd is not None:
+            os.close(fd)
+
+
+def setup_ssh_ctl(ctld, remote_addr, resource_url):
+    """
+    Setup GConf ssh control path parameters
+    """
+    rconf.ssh_ctl_dir = ctld
+    content = "SLAVE_HOST=%s\nSLAVE_RESOURCE_URL=%s" % (remote_addr,
+                                                        resource_url)
+    encoded_content = content.encode()
+    content_sha256 = sha256hex(encoded_content)
+    """
+    The length of ctl_path for ssh connection should not be > 108.
+    ssh fails with ctl_path too long if it is so. But when rsync
+    is piped to ssh, it is not taking > 90. Hence using first 32
+    bytes of hash. Hash collision doesn't matter as only one sock
+    file is created per directory.
+    """
+    content_sha256 = content_sha256[:32]
+    fname = os.path.join(rconf.ssh_ctl_dir,
+                         "%s.mft" % content_sha256)
+
+    create_manifest(fname, encoded_content)
+    ssh_ctl_path = os.path.join(rconf.ssh_ctl_dir,
+                                "%s.sock" % content_sha256)
+    rconf.ssh_ctl_args = ["-oControlMaster=auto", "-S", ssh_ctl_path]
+
+
+def grabfile(fname, content=None):
+    """open @fname + contest for its fcntl lock
+
+    @content: if given, set the file content to it
+    """
+    # damn those messy open() mode codes
+    fd = os.open(fname, os.O_CREAT | os.O_RDWR)
+    f = os.fdopen(fd, 'r+')
+    try:
+        fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except:
+        ex = sys.exc_info()[1]
+        f.close()
+        if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
+            # cannot grab, it's taken
+            return
+        raise
+    if content:
+        try:
+            f.truncate()
+            f.write(content)
+            f.flush()
+        except:
+            f.close()
+            raise
+    rconf.permanent_handles.append(f)
+    return f
+
+
+def grabpidfile(fname=None, setpid=True):
+    """.grabfile customization for pid files"""
+    if not fname:
+        fname = gconf.get("pid-file")
+    content = None
+    if setpid:
+        content = str(os.getpid()) + '\n'
+    return grabfile(fname, content=content)
+
+
+def finalize(*args, **kwargs):
+    """all those messy final steps we go trough upon termination
+
+    Do away with pidfile, ssh control dir and logging.
+    """
+
+    final_lock.acquire()
+    if gconf.get('pid_file'):
+        rm_pidf = rconf.pid_file_owned
+        if rconf.cpid:
+            # exit path from parent branch of daemonization
+            rm_pidf = False
+            while True:
+                f = grabpidfile(setpid=False)
+                if not f:
+                    # child has already taken over pidfile
+                    break
+                if os.waitpid(rconf.cpid, os.WNOHANG)[0] == rconf.cpid:
+                    # child has terminated
+                    rm_pidf = True
+                    break
+                time.sleep(0.1)
+        if rm_pidf:
+            try:
+                os.unlink(rconf.pid_file)
+            except:
+                ex = sys.exc_info()[1]
+                if ex.errno == ENOENT:
+                    pass
+                else:
+                    raise
+    if rconf.ssh_ctl_dir and not rconf.cpid:
+        def handle_rm_error(func, path, exc_info):
+            if exc_info[1].errno == ENOENT:
+                return
+            raise exc_info[1]
+
+        shutil.rmtree(rconf.ssh_ctl_dir, onerror=handle_rm_error)
+
+    """ Unmount if not done """
+    if rconf.mount_point:
+        if rconf.mountbroker:
+            umount_cmd = rconf.mbr_umount_cmd + [rconf.mount_point, 'lazy']
+        else:
+            umount_cmd = ['umount', '-l', rconf.mount_point]
+        p0 = subprocess.Popen(umount_cmd, stderr=subprocess.PIPE,
+                              universal_newlines=True)
+        _, errdata = p0.communicate()
+        if p0.returncode == 0:
+            try:
+                os.rmdir(rconf.mount_point)
+            except OSError:
+                pass
+        else:
+            pass
+
+    if rconf.log_exit:
+        logging.info("exiting.")
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(kwargs.get('exval', 0))
+
+
+def log_raise_exception(excont):
+    """top-level exception handler
+
+    Try to some fancy things to cover up we face with an error.
+    Translate some weird sounding but well understood exceptions
+    into human-friendly lingo
+    """
+
+    is_filelog = False
+    for h in logging.getLogger().handlers:
+        fno = getattr(getattr(h, 'stream', None), 'fileno', None)
+        if fno and not os.isatty(fno()):
+            is_filelog = True
+
+    exc = sys.exc_info()[1]
+    if isinstance(exc, SystemExit):
+        excont.exval = exc.code or 0
+        raise
+    else:
+        logtag = None
+        if isinstance(exc, GsyncdError):
+            if is_filelog:
+                logging.error(exc.args[0])
+            sys.stderr.write('failure: ' + exc.args[0] + '\n')
+        elif isinstance(exc, PickleError) or isinstance(exc, EOFError) or \
+            ((isinstance(exc, OSError) or isinstance(exc, IOError)) and
+             exc.errno == EPIPE):
+            logging.error('connection to peer is broken')
+            if hasattr(rconf, 'transport'):
+                rconf.transport.wait()
+                if rconf.transport.returncode == 127:
+                    logging.error("getting \"No such file or directory\""
+                                  "errors is most likely due to "
+                                  "MISCONFIGURATION, please remove all "
+                                  "the public keys added by geo-replication "
+                                  "from authorized_keys file in slave nodes "
+                                  "and run Geo-replication create "
+                                  "command again.")
+                    logging.error("If `gsec_create container` was used, then "
+                                  "run `gluster volume geo-replication "
+                                  "<MASTERVOL> [<SLAVEUSER>@]<SLAVEHOST>::"
+                                  "<SLAVEVOL> config remote-gsyncd "
+                                  "<GSYNCD_PATH> (Example GSYNCD_PATH: "
+                                  "`/usr/libexec/glusterfs/gsyncd`)")
+                rconf.transport.terminate_geterr()
+        elif isinstance(exc, OSError) and exc.errno in (ENOTCONN,
+                                                        ECONNABORTED):
+            logging.error(lf('Gluster Mount process exited',
+                             error=errorcode[exc.errno]))
+        elif isinstance(exc, OSError) and exc.errno == EIO:
+            logging.error("Getting \"Input/Output error\" "
+                          "is most likely due to "
+                          "a. Brick is down or "
+                          "b. Split brain issue.")
+            logging.error("This is expected as per design to "
+                          "keep the consistency of the file system. "
+                          "Once the above issue is resolved "
+                          "geo-replication would automatically "
+                          "proceed further.")
+            logtag = "FAIL"
+        else:
+            logtag = "FAIL"
+        if not logtag and logging.getLogger().isEnabledFor(logging.DEBUG):
+            logtag = "FULL EXCEPTION TRACE"
+        if logtag:
+            logging.exception(logtag + ": ")
+            sys.stderr.write("failed with %s: %s.\n" % (type(exc).__name__, exc))
+        excont.exval = 1
+        sys.exit(excont.exval)
+
+
+class FreeObject(object):
+
+    """wildcard class for which any attribute can be set"""
+
+    def __init__(self, **kw):
+        for k, v in kw.items():
+            setattr(self, k, v)
+
+
+class Thread(baseThread):
+
+    """thread class flavor for gsyncd
+
+    - always a daemon thread
+    - force exit for whole program if thread
+      function coughs up an exception
+    """
+
+    def __init__(self, *args, **kwargs):
+        tf = kwargs.get('target')
+        if tf:
+            def twrap(*aargs):
+                excont = FreeObject(exval=0)
+                try:
+                    tf(*aargs)
+                except:
+                    try:
+                        log_raise_exception(excont)
+                    finally:
+                        finalize(exval=excont.exval)
+            kwargs['target'] = twrap
+        baseThread.__init__(self, *args, **kwargs)
+        self.setDaemon(True)
+
+
+class GsyncdError(Exception):
+    pass
+
+
+class _MetaXattr(object):
+
+    """singleton class, a lazy wrapper around the
+    libcxattr module
+
+    libcxattr (a heavy import due to ctypes) is
+    loaded only when when the single
+    instance is tried to be used.
+
+    This reduces runtime for those invocations
+    which do not need filesystem manipulation
+    (eg. for config, url parsing)
+    """
+
+    def __getattr__(self, meth):
+        from libcxattr import Xattr as LXattr
+        xmeth = [m for m in dir(LXattr) if m[0] != '_']
+        if meth not in xmeth:
+            return
+        for m in xmeth:
+            setattr(self, m, getattr(LXattr, m))
+        return getattr(self, meth)
+
+
+Xattr = _MetaXattr()
+
+
+def getusername(uid=None):
+    if uid is None:
+        uid = os.geteuid()
+    return pwd.getpwuid(uid).pw_name
+
+
+def privileged():
+    return os.geteuid() == 0
+
+
+def boolify(s):
+    """
+    Generic string to boolean converter
+
+    return
+    - Quick return if string 's' is of type bool
+    - True if it's in true_list
+    - False if it's in false_list
+    - Warn if it's not present in either and return False
+    """
+    true_list = ['true', 'yes', '1', 'on']
+    false_list = ['false', 'no', '0', 'off']
+
+    if isinstance(s, bool):
+        return s
+
+    rv = False
+    lstr = s.lower()
+    if lstr in true_list:
+        rv = True
+    elif lstr not in false_list:
+        logging.warn(lf("Unknown string in \"string to boolean\" conversion, "
+                        "defaulting to False",
+                        str=s))
+
+    return rv
+
+
+def eintr_wrap(func, exc, *args):
+    """
+    wrapper around syscalls resilient to interrupt caused
+    by signals
+    """
+    while True:
+        try:
+            return func(*args)
+        except exc:
+            ex = sys.exc_info()[1]
+            if not ex.args[0] == EINTR:
+                raise
+
+
+def select(*args):
+    return eintr_wrap(oselect.select, oselect.error, *args)
+
+
+def waitpid(*args):
+    return eintr_wrap(owaitpid, OSError, *args)
+
+
+def term_handler_default_hook(signum, frame):
+    finalize(signum, frame, exval=1)
+
+
+def set_term_handler(hook=term_handler_default_hook):
+    signal(SIGTERM, hook)
+
+
+def get_node_uuid():
+    global NodeID
+    if NodeID is not None:
+        return NodeID
+
+    NodeID = ""
+    with open(UUID_FILE) as f:
+        for line in f:
+            if line.startswith("UUID="):
+                NodeID = line.strip().split("=")[-1]
+                break
+
+    if NodeID == "":
+        raise GsyncdError("Failed to get Host UUID from %s" % UUID_FILE)
+    return NodeID
+
+
+def is_host_local(host_id):
+    return host_id == get_node_uuid()
+
+
+def funcode(f):
+    fc = getattr(f, 'func_code', None)
+    if not fc:
+        # python 3
+        fc = f.__code__
+    return fc
+
+
+def memoize(f):
+    fc = funcode(f)
+    fn = fc.co_name
+
+    def ff(self, *a, **kw):
+        rv = getattr(self, '_' + fn, None)
+        if rv is None:
+            rv = f(self, *a, **kw)
+            setattr(self, '_' + fn, rv)
+        return rv
+    return ff
+
+
+def umask():
+    return os.umask(0)
+
+
+def entry2pb(e):
+    return e.rsplit('/', 1)
+
+
+def gauxpfx():
+    return _CL_AUX_GFID_PFX
+
+
+def sha256hex(s):
+    return sha256(s).hexdigest()
+
+
+def selfkill(sig=SIGTERM):
+    os.kill(os.getpid(), sig)
+
+
+def errno_wrap(call, arg=[], errnos=[], retry_errnos=[]):
+    """ wrapper around calls resilient to errnos.
+    """
+    nr_tries = 0
+    while True:
+        try:
+            return call(*arg)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in errnos:
+                return ex.errno
+            if ex.errno not in retry_errnos:
+                raise
+            nr_tries += 1
+            if nr_tries == GF_OP_RETRIES:
+                # probably a screwed state, cannot do much...
+                logging.warn(lf('reached maximum retries',
+                                args=repr(arg),
+                                error=ex))
+                raise
+            time.sleep(0.250)  # retry the call
+
+
+def lstat(e):
+    return errno_wrap(os.lstat, [e], [ENOENT], [ESTALE, EBUSY])
+
+def get_gfid_from_mnt(gfidpath):
+    return errno_wrap(Xattr.lgetxattr,
+                      [gfidpath, 'glusterfs.gfid.string',
+                       GX_GFID_CANONICAL_LEN], [ENOENT], [ESTALE])
+
+
+def matching_disk_gfid(gfid, entry):
+    disk_gfid = get_gfid_from_mnt(entry)
+    if isinstance(disk_gfid, int):
+        return False
+
+    if not gfid == disk_gfid:
+        return False
+
+    return True
+
+
+class NoStimeAvailable(Exception):
+    pass
+
+
+class PartialHistoryAvailable(Exception):
+    pass
+
+
+class ChangelogHistoryNotAvailable(Exception):
+    pass
+
+
+class ChangelogException(OSError):
+    pass
+
+
+def gf_event(event_type, **kwargs):
+    if EVENTS_ENABLED:
+        from gfevents.gf_event import gf_event as gfevent
+        gfevent(event_type, **kwargs)
+
+
+class GlusterLogLevel(object):
+        NONE = 0
+        EMERG = 1
+        ALERT = 2
+        CRITICAL = 3
+        ERROR = 4
+        WARNING = 5
+        NOTICE = 6
+        INFO = 7
+        DEBUG = 8
+        TRACE = 9
+
+
+def get_changelog_log_level(lvl):
+    return getattr(GlusterLogLevel, lvl, GlusterLogLevel.INFO)
+
+
+def get_master_and_slave_data_from_args(args):
+    master_name = None
+    slave_data = None
+    for arg in args:
+        if arg.startswith(":"):
+            master_name = arg.replace(":", "")
+        if "::" in arg:
+            slave_data = arg.replace("ssh://", "")
+
+    return (master_name, slave_data)
+
+def unshare_propagation_supported():
+    global unshare_mnt_propagation
+    if unshare_mnt_propagation is not None:
+        return unshare_mnt_propagation
+
+    unshare_mnt_propagation = False
+    p = subprocess.Popen(["unshare", "--help"],
+                         stderr=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         universal_newlines=True)
+    out, err = p.communicate()
+    if p.returncode == 0:
+        if "propagation" in out:
+            unshare_mnt_propagation = True
+
+    return unshare_mnt_propagation
+
+
+def get_rsync_version(rsync_cmd):
+    global rsync_version
+    if rsync_version is not None:
+        return rsync_version
+
+    rsync_version = "0"
+    p = subprocess.Popen([rsync_cmd, "--version"],
+                         stderr=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         universal_newlines=True)
+    out, err = p.communicate()
+    if p.returncode == 0:
+        rsync_version = out.split(" ", 4)[3]
+
+    return rsync_version
+
+
+def get_slv_dir_path(slv_host, slv_volume, gfid):
+    global slv_bricks
+
+    dir_path = ENOENT
+    pfx = gauxpfx()
+
+    if not slv_bricks:
+        slv_info = Volinfo(slv_volume, slv_host, master=False)
+        slv_bricks = slv_info.bricks
+    # Result of readlink would be of format as below.
+    # readlink = "../../pgfid[0:2]/pgfid[2:4]/pgfid/basename"
+    for brick in slv_bricks:
+        dir_path = errno_wrap(os.path.join,
+                              [brick['dir'],
+                               ".glusterfs", gfid[0:2],
+                               gfid[2:4],
+                               gfid], [ENOENT], [ESTALE])
+        if dir_path != ENOENT:
+            try:
+                realpath = errno_wrap(os.readlink, [dir_path],
+                                      [ENOENT], [ESTALE])
+                if not isinstance(realpath, int):
+                    realpath_parts = realpath.split('/')
+                    pargfid = realpath_parts[-2]
+                    basename = realpath_parts[-1]
+                    dir_entry = os.path.join(pfx, pargfid, basename)
+                    return dir_entry
+            except OSError:
+                # .gfid/GFID
+                gfidpath = unescape_space_newline(os.path.join(pfx, gfid))
+                realpath = errno_wrap(Xattr.lgetxattr_buf,
+                      [gfidpath, 'glusterfs.gfid2path'], [ENOENT], [ESTALE])
+                if not isinstance(realpath, int):
+                    basename = os.path.basename(realpath).rstrip('\x00')
+                    dirpath = os.path.dirname(realpath)
+                    if dirpath == "/":
+                        pargfid = ROOT_GFID
+                    else:
+                        dirpath = dirpath.strip("/")
+                        pargfid = get_gfid_from_mnt(dirpath)
+                        if isinstance(pargfid, int):
+                            return None
+                    dir_entry = os.path.join(pfx, pargfid, basename)
+                    return dir_entry
+
+    return None
+
+
+def lf(event, **kwargs):
+    """
+    Log Format helper function, log messages can be
+    easily modified to structured log format.
+    lf("Config Change", sync_jobs=4, brick=/bricks/b1) will be
+    converted as "Config Change [{brick=/bricks/b1}, {sync_jobs=4}]"
+    """
+    msgparts = []
+    for k, v in kwargs.items():
+        msgparts.append("{%s=%s}" % (k, v))
+    return "%s [%s]" % (event, ", ".join(msgparts))
+
+
+class Popen(subprocess.Popen):
+
+    """customized subclass of subprocess.Popen with a ring
+    buffer for children error output"""
+
+    @classmethod
+    def init_errhandler(cls):
+        """start the thread which handles children's error output"""
+        cls.errstore = {}
+
+        def tailer():
+            while True:
+                errstore = cls.errstore.copy()
+                try:
+                    poe, _, _ = select(
+                        [po.stderr for po in errstore], [], [], 1)
+                except (ValueError, SelectError):
+                    # stderr is already closed wait for some time before
+                    # checking next error
+                    time.sleep(0.5)
+                    continue
+                for po in errstore:
+                    if po.stderr not in poe:
+                        continue
+                    po.lock.acquire()
+                    try:
+                        if po.on_death_row:
+                            continue
+                        la = errstore[po]
+                        try:
+                            fd = po.stderr.fileno()
+                        except ValueError:  # file is already closed
+                            time.sleep(0.5)
+                            continue
+
+                        try:
+                            l = os.read(fd, 1024)
+                        except OSError:
+                            time.sleep(0.5)
+                            continue
+
+                        if not l:
+                            continue
+                        tots = len(l)
+                        for lx in la:
+                            tots += len(lx)
+                        while tots > 1 << 20 and la:
+                            tots -= len(la.pop(0))
+                        la.append(l)
+                    finally:
+                        po.lock.release()
+        t = Thread(target=tailer)
+        t.start()
+        cls.errhandler = t
+
+    @classmethod
+    def fork(cls):
+        """fork wrapper that restarts errhandler thread in child"""
+        pid = os.fork()
+        if not pid:
+            cls.init_errhandler()
+        return pid
+
+    def __init__(self, args, *a, **kw):
+        """customizations for subprocess.Popen instantiation
+
+        - 'close_fds' is taken to be the default
+        - if child's stderr is chosen to be managed,
+          register it with the error handler thread
+        """
+        self.args = args
+        if 'close_fds' not in kw:
+            kw['close_fds'] = True
+        self.lock = threading.Lock()
+        self.on_death_row = False
+        self.elines = []
+        try:
+            sup(self, args, *a, **kw)
+        except:
+            ex = sys.exc_info()[1]
+            if not isinstance(ex, OSError):
+                raise
+            raise GsyncdError("""execution of "%s" failed with %s (%s)""" %
+                              (args[0], errno.errorcode[ex.errno],
+                               os.strerror(ex.errno)))
+        if kw.get('stderr') == subprocess.PIPE:
+            assert(getattr(self, 'errhandler', None))
+            self.errstore[self] = []
+
+    def errlog(self):
+        """make a log about child's failure event"""
+        logging.error(lf("command returned error",
+                         cmd=" ".join(self.args),
+                         error=self.returncode))
+        lp = ''
+
+        def logerr(l):
+            logging.error(self.args[0] + "> " + l)
+        for l in self.elines:
+            ls = l.split('\n')
+            ls[0] = lp + ls[0]
+            lp = ls.pop()
+            for ll in ls:
+                logerr(ll)
+        if lp:
+            logerr(lp)
+
+    def errfail(self):
+        """fail nicely if child did not terminate with success"""
+        self.errlog()
+        finalize(exval=1)
+
+    def terminate_geterr(self, fail_on_err=True):
+        """kill child, finalize stderr harvesting (unregister
+        from errhandler, set up .elines), fail on error if
+        asked for
+        """
+        self.lock.acquire()
+        try:
+            self.on_death_row = True
+        finally:
+            self.lock.release()
+        elines = self.errstore.pop(self)
+        if self.poll() is None:
+            self.terminate()
+            if self.poll() is None:
+                time.sleep(0.1)
+                self.kill()
+                self.wait()
+        while True:
+            if not select([self.stderr], [], [], 0.1)[0]:
+                break
+            b = os.read(self.stderr.fileno(), 1024)
+            if b:
+                elines.append(b.decode())
+            else:
+                break
+        self.stderr.close()
+        self.elines = elines
+        if fail_on_err and self.returncode != 0:
+            self.errfail()
+
+
+def host_brick_split(value):
+    """
+    IPv6 compatible way to split and get the host
+    and brick information. Example inputs:
+    node1.example.com:/exports/bricks/brick1/brick
+    fe80::af0f:df82:844f:ef66%utun0:/exports/bricks/brick1/brick
+    """
+    parts = value.split(":")
+    brick = parts[-1]
+    hostparts = parts[0:-1]
+    return (":".join(hostparts), brick)
+
+
+class Volinfo(object):
+
+    def __init__(self, vol, host='localhost', prelude=[], master=True):
+        if master:
+            gluster_cmd_dir = gconf.get("gluster-command-dir")
+        else:
+            gluster_cmd_dir = gconf.get("slave-gluster-command-dir")
+
+        gluster_cmd = os.path.join(gluster_cmd_dir, 'gluster')
+        po = Popen(prelude + [gluster_cmd, '--xml', '--remote-host=' + host,
+                              'volume', 'info', vol],
+                   stdout=PIPE, stderr=PIPE, universal_newlines=True)
+        vix = po.stdout.read()
+        po.wait()
+        po.terminate_geterr()
+        vi = XET.fromstring(vix)
+        if vi.find('opRet').text != '0':
+            if prelude:
+                via = '(via %s) ' % prelude.join(' ')
+            else:
+                via = ' '
+            raise GsyncdError('getting volume info of %s%s '
+                              'failed with errorcode %s' %
+                              (vol, via, vi.find('opErrno').text))
+        self.tree = vi
+        self.volume = vol
+        self.host = host
+
+    def get(self, elem):
+        return self.tree.findall('.//' + elem)
+
+    def is_tier(self):
+        return (self.get('typeStr')[0].text == 'Tier')
+
+    def is_hot(self, brickpath):
+        logging.debug('brickpath: ' + repr(brickpath))
+        return brickpath in self.hot_bricks
+
+    @property
+    @memoize
+    def bricks(self):
+        def bparse(b):
+            host, dirp = host_brick_split(b.find("name").text)
+            return {'host': host, 'dir': dirp, 'uuid': b.find("hostUuid").text}
+        return [bparse(b) for b in self.get('brick')]
+
+    @property
+    @memoize
+    def uuid(self):
+        ids = self.get('id')
+        if len(ids) != 1:
+            raise GsyncdError("volume info of %s obtained from %s: "
+                              "ambiguous uuid" % (self.volume, self.host))
+        return ids[0].text
+
+    def replica_count(self, tier, hot):
+        if (tier and hot):
+            return int(self.get('hotBricks/hotreplicaCount')[0].text)
+        elif (tier and not hot):
+            return int(self.get('coldBricks/coldreplicaCount')[0].text)
+        else:
+            return int(self.get('replicaCount')[0].text)
+
+    def disperse_count(self, tier, hot):
+        if (tier and hot):
+            # Tiering doesn't support disperse volume as hot brick,
+            # hence no xml output, so returning 0. In case, if it's
+            # supported later, we should change here.
+            return 0
+        elif (tier and not hot):
+            return int(self.get('coldBricks/colddisperseCount')[0].text)
+        else:
+            return int(self.get('disperseCount')[0].text)
+
+    def distribution_count(self, tier, hot):
+        if (tier and hot):
+            return int(self.get('hotBricks/hotdistCount')[0].text)
+        elif (tier and not hot):
+            return int(self.get('coldBricks/colddistCount')[0].text)
+        else:
+            return int(self.get('distCount')[0].text)
+
+    @property
+    @memoize
+    def hot_bricks(self):
+        return [b.text for b in self.get('hotBricks/brick')]
+
+    def get_hot_bricks_count(self, tier):
+        if (tier):
+            return int(self.get('hotBricks/hotbrickCount')[0].text)
+        else:
+            return 0
+
+
+class VolinfoFromGconf(object):
+    # Glusterd will generate following config items before Geo-rep start
+    # So that Geo-rep need not run gluster commands from inside
+    # Volinfo object API/interface kept as is so that caller need not
+    # change anything except calling this instead of Volinfo()
+    #
+    # master-bricks=
+    # master-bricks=NODEID:HOSTNAME:PATH,..
+    # slave-bricks=NODEID:HOSTNAME,..
+    # master-volume-id=
+    # slave-volume-id=
+    # master-replica-count=
+    # master-disperse_count=
+    def __init__(self, vol, host='localhost', master=True):
+        self.volume = vol
+        self.host = host
+        self.master = master
+
+    def is_tier(self):
+        return False
+
+    def is_hot(self, brickpath):
+        return False
+
+    def is_uuid(self, value):
+        try:
+            uuid.UUID(value)
+            return True
+        except ValueError:
+            return False
+
+    def possible_path(self, value):
+        return "/" in value
+
+    @property
+    @memoize
+    def bricks(self):
+        pfx = "master-" if self.master else "slave-"
+        bricks_data = gconf.get(pfx + "bricks")
+        if bricks_data is None:
+            return []
+
+        bricks_data = bricks_data.split(",")
+        bricks_data = [b.strip() for b in bricks_data]
+        out = []
+        for b in bricks_data:
+            parts = b.split(":")
+            b_uuid = None
+            if self.is_uuid(parts[0]):
+                b_uuid = parts[0]
+                # Set all parts except first
+                parts = parts[1:]
+
+            if self.possible_path(parts[-1]):
+                bpath = parts[-1]
+                # Set all parts except last
+                parts = parts[0:-1]
+
+            out.append({
+                "host": ":".join(parts),   # if remaining parts are IPv6 name
+                "dir": bpath,
+                "uuid": b_uuid
+            })
+
+        return out
+
+    @property
+    @memoize
+    def uuid(self):
+        if self.master:
+            return gconf.get("master-volume-id")
+        else:
+            return gconf.get("slave-volume-id")
+
+    def replica_count(self, tier, hot):
+        return gconf.get("master-replica-count")
+
+    def disperse_count(self, tier, hot):
+        return gconf.get("master-disperse-count")
+
+    def distribution_count(self, tier, hot):
+        return gconf.get("master-distribution-count")
+
+    @property
+    @memoize
+    def hot_bricks(self):
+        return []
+
+    def get_hot_bricks_count(self, tier):
+        return 0
+
+
+def can_ssh(host, port=22):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.connect((host, port))
+        flag = True
+    except socket.error:
+        flag = False
+
+    s.close()
+    return flag
+
+
+def get_up_nodes(hosts, port):
+    # List of hosts with Hostname/IP and UUID
+    up_nodes = []
+    for h in hosts:
+        if can_ssh(h[0], port):
+            up_nodes.append(h)
+
+    return up_nodes
diff --git a/geo-replication/test-requirements.txt b/geo-replication/test-requirements.txt
new file mode 100644
index 00000000000..d6165640d3f
--- /dev/null
+++ b/geo-replication/test-requirements.txt
@@ -0,0 +1,7 @@
+# Hacking already pins down pep8, pyflakes and flake8
+flake8
+coverage
+nose
+nosexcover
+nosehtmloutput
+mock>=0.8.0
diff --git a/geo-replication/tests/__init__.py b/geo-replication/tests/__init__.py
new file mode 100644
index 00000000000..b4648b69645
--- /dev/null
+++ b/geo-replication/tests/__init__.py
@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
diff --git a/geo-replication/tests/unit/__init__.py b/geo-replication/tests/unit/__init__.py
new file mode 100644
index 00000000000..b4648b69645
--- /dev/null
+++ b/geo-replication/tests/unit/__init__.py
@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
diff --git a/geo-replication/tests/unit/test_gsyncdstatus.py b/geo-replication/tests/unit/test_gsyncdstatus.py
new file mode 100755
index 00000000000..9c1aa2ad4ad
--- /dev/null
+++ b/geo-replication/tests/unit/test_gsyncdstatus.py
@@ -0,0 +1,193 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import unittest
+import os
+import urllib
+
+from syncdaemon.gstatus import (GeorepStatus, set_monitor_status,
+                                get_default_values,
+                                MONITOR_STATUS, DEFAULT_STATUS,
+                                STATUS_VALUES, CRAWL_STATUS_VALUES,
+                                human_time, human_time_utc)
+
+
+class GeorepStatusTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.work_dir = os.path.dirname(os.path.abspath(__file__))
+        cls.monitor_status_file = os.path.join(cls.work_dir, "monitor.status")
+        cls.brick = "/exports/bricks/b1"
+        cls.status = GeorepStatus(cls.monitor_status_file, cls.brick)
+        cls.statusfile = os.path.join(cls.work_dir,
+                                      "brick_%s.status"
+                                      % urllib.quote_plus(cls.brick))
+
+    @classmethod
+    def tearDownClass(cls):
+        os.remove(cls.statusfile)
+        os.remove(cls.monitor_status_file)
+
+    def _filter_dict(self, inp, keys):
+        op = {}
+        for k in keys:
+            op[k] = inp.get(k, None)
+        return op
+
+    def test_monitor_status_file_created(self):
+        self.assertTrue(os.path.exists(self.monitor_status_file))
+
+    def test_status_file_created(self):
+        self.assertTrue(os.path.exists(self.statusfile))
+
+    def test_set_monitor_status(self):
+        for st in MONITOR_STATUS:
+            set_monitor_status(self.monitor_status_file, st)
+            self.assertTrue(self.status.get_monitor_status(), st)
+
+    def test_default_values_test(self):
+        self.assertTrue(get_default_values(), {
+            "slave_node": DEFAULT_STATUS,
+            "worker_status": DEFAULT_STATUS,
+            "last_synced": 0,
+            "last_synced_utc": 0,
+            "crawl_status": DEFAULT_STATUS,
+            "entry": 0,
+            "data": 0,
+            "metadata": 0,
+            "failures": 0,
+            "checkpoint_completed": False,
+            "checkpoint_time": 0,
+            "checkpoint_time_utc": 0,
+            "checkpoint_completion_time": 0,
+            "checkpoint_completion_time_utc": 0
+        })
+
+    def test_human_time(self):
+        self.assertTrue(human_time(1429174398), "2015-04-16 14:23:18")
+
+    def test_human_time_utc(self):
+        self.assertTrue(human_time_utc(1429174398), "2015-04-16 08:53:18")
+
+    def test_invalid_human_time(self):
+        self.assertTrue(human_time(142917439), DEFAULT_STATUS)
+        self.assertTrue(human_time("abcdef"), DEFAULT_STATUS)
+
+    def test_invalid_human_time_utc(self):
+        self.assertTrue(human_time_utc(142917439), DEFAULT_STATUS)
+        self.assertTrue(human_time_utc("abcdef"), DEFAULT_STATUS)
+
+    def test_worker_status(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        for st in STATUS_VALUES:
+            self.status.set_worker_status(st)
+            self.assertTrue(self.status.get_status()["worker_status"], st)
+
+    def test_crawl_status(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+        for st in CRAWL_STATUS_VALUES:
+            self.status.set_worker_crawl_status(st)
+            self.assertTrue(self.status.get_status()["crawl_status"], st)
+
+    def test_slave_node(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+        self.status.set_slave_node("fvm2")
+        self.assertTrue(self.status.get_status()["slave_node"], "fvm2")
+
+        self.status.set_worker_status("Passive")
+        self.status.set_slave_node("fvm2")
+        self.assertTrue(self.status.get_status()["slave_node"], "fvm2")
+
+    def test_active_worker_status(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+        self.assertTrue(self.status.get_status()["worker_status"], "Active")
+
+    def test_passive_worker_status(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_passive()
+        self.assertTrue(self.status.get_status()["worker_status"], "Passive")
+
+    def test_set_field(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+        self.status.set_field("entry", 42)
+        self.assertTrue(self.status.get_status()["entry"], 42)
+
+    def test_inc_value(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+        self.status.set_field("entry", 0)
+        self.status.inc_value("entry", 2)
+        self.assertTrue(self.status.get_status()["entry"], 2)
+
+        self.status.set_field("data", 0)
+        self.status.inc_value("data", 2)
+        self.assertTrue(self.status.get_status()["data"], 2)
+
+        self.status.set_field("meta", 0)
+        self.status.inc_value("meta", 2)
+        self.assertTrue(self.status.get_status()["meta"], 2)
+
+        self.status.set_field("failures", 0)
+        self.status.inc_value("failures", 2)
+        self.assertTrue(self.status.get_status()["failures"], 2)
+
+    def test_dec_value(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+
+        self.status.set_field("entry", 4)
+        self.status.inc_value("entry", 2)
+        self.assertTrue(self.status.get_status()["entry"], 2)
+
+        self.status.set_field("data", 4)
+        self.status.inc_value("data", 2)
+        self.assertTrue(self.status.get_status()["data"], 2)
+
+        self.status.set_field("meta", 4)
+        self.status.inc_value("meta", 2)
+        self.assertTrue(self.status.get_status()["meta"], 2)
+
+        self.status.set_field("failures", 4)
+        self.status.inc_value("failures", 2)
+        self.assertTrue(self.status.get_status()["failures"], 2)
+
+    def test_worker_status_when_monitor_status_created(self):
+        set_monitor_status(self.monitor_status_file, "Created")
+        for st in STATUS_VALUES:
+            self.status.set_worker_status(st)
+            self.assertTrue(self.status.get_status()["worker_status"],
+                            "Created")
+
+    def test_worker_status_when_monitor_status_paused(self):
+        set_monitor_status(self.monitor_status_file, "Paused")
+        for st in STATUS_VALUES:
+            self.status.set_worker_status(st)
+            self.assertTrue(self.status.get_status()["worker_status"],
+                            "Paused")
+
+    def test_worker_status_when_monitor_status_stopped(self):
+        set_monitor_status(self.monitor_status_file, "Stopped")
+        for st in STATUS_VALUES:
+            self.status.set_worker_status(st)
+            self.assertTrue(self.status.get_status()["worker_status"],
+                            "Stopped")
+
+    def test_status_when_worker_status_active(self):
+        set_monitor_status(self.monitor_status_file, "Started")
+        self.status.set_active()
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/geo-replication/tests/unit/test_syncdutils.py b/geo-replication/tests/unit/test_syncdutils.py
new file mode 100644
index 00000000000..ff537ab2660
--- /dev/null
+++ b/geo-replication/tests/unit/test_syncdutils.py
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import unittest
+
+from syncdaemon import syncdutils
+
+
+class SyncdutilsTestCase(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_escape(self):
+        self.assertEqual(syncdutils.escape("http://gluster.org"),
+                         "http%3A%2F%2Fgluster.org")
+
+    def test_unescape(self):
+        self.assertEqual(syncdutils.unescape("http%3A%2F%2Fgluster.org"),
+                         "http://gluster.org")
diff --git a/geo-replication/tox.ini b/geo-replication/tox.ini
new file mode 100644
index 00000000000..57ff086b947
--- /dev/null
+++ b/geo-replication/tox.ini
@@ -0,0 +1,32 @@
+[tox]
+envlist = py26,py27,pep8
+
+[testenv]
+whitelist_externals=bash
+setenv = VIRTUAL_ENV={envdir}
+deps =
+     --download-cache={homedir}/.pipcache
+     -r{toxinidir}/test-requirements.txt
+changedir = {toxinidir}/tests/unit
+commands = nosetests -v --exe --with-xunit --with-coverage --cover-package syncdaemon --cover-erase --cover-xml --cover-html --cover-branches --with-html-output {posargs}
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv:pep8]
+changedir = {toxinidir}
+commands =
+         flake8
+         flake8 syncdaemon tests
+
+[testenv:cover]
+setenv = NOSE_WITH_COVERAGE=1
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+ignore = H
+builtins = _
+exclude = .venv,.tox,dist,doc,tests,*egg
+show-source = True
+\ No newline at end of file
diff --git a/geo-replication/unittests.sh b/geo-replication/unittests.sh
new file mode 100644
index 00000000000..d5dbd00bd4c
--- /dev/null
+++ b/geo-replication/unittests.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $(dirname $0)/tests/unit
+nosetests -v --exe --with-coverage --cover-package \
+          syncdaemon --cover-erase --cover-html --cover-branches $@
+
+saved_status=$?
+rm -f .coverage
+exit $saved_status
diff --git a/glusterfs-api.pc.in b/glusterfs-api.pc.in
new file mode 100644
index 00000000000..4a2edb7bf07
--- /dev/null
+++ b/glusterfs-api.pc.in
@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: glusterfs-api
+Description: GlusterFS API
+/* This is the API version, NOT package version */
+Version: @GFAPI_VERSION@
+Requires: @PKGCONFIG_UUID@
+Libs: -L${libdir} @GFAPI_LIBS@ -lgfapi -lglusterfs -lgfrpc -lgfxdr
+Cflags: -I${includedir} -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64 -DUSE_POSIX_ACLS=@USE_POSIX_ACLS@
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index 85ae0d6aae8..b6d63146e14 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -1,197 +1,2268 @@
-# if you make changes, the it is advised to increment this number, and provide 
-# a descriptive suffix to identify who owns or what the change represents
-# e.g. release_version 2.MSW
-%define release 1%{?dist}
+%global _hardened_build 1
 
-# if you wish to compile an rpm without rdma support, compile like this...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without rdma
-%{?_without_rdma:%define _without_rdma --disable-ibverbs}
+%global _for_fedora_koji_builds 0
 
+# uncomment and add '%' to use the prereltag for pre-releases
+# %%global prereltag qa3
+
+##-----------------------------------------------------------------------------
+## All argument definitions should be placed here and keep them sorted
+##
+
+# asan
+# if you wish to compile an rpm with address sanitizer...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with asan
+%{?_with_asan:%global _with_asan --enable-asan}
+
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
+%global _with_asan %{nil}
+%endif
+
+# cmocka
+# if you wish to compile an rpm with cmocka unit testing...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with cmocka
+%{?_with_cmocka:%global _with_cmocka --enable-cmocka}
+
+# debug
+# if you wish to compile an rpm with debugging...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with debug
+%{?_with_debug:%global _with_debug --enable-debug}
+
+# epoll
 # if you wish to compile an rpm without epoll...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without epoll
-%{?_without_epoll:%define _without_epoll --disable-epoll}
-
-# if you wish to compile an rpm with fusermount...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with fusermount
-%{?_with_fusermount:%define _with_fusermount --enable-fusermount}
-
-Summary: Cluster File System
-Name: @PACKAGE_NAME@
-Version: @PACKAGE_VERSION@
-Release: %{release}
-License: AGPLv3+
-Group: System Environment/Base
-Vendor: Gluster Inc
-Packager: @PACKAGE_BUGREPORT@
-URL: http://www.gluster.org/docs/index.php/GlusterFS
-Source0: ftp://ftp.gluster.com/pub/gluster/glusterfs/@PACKAGE_VERSION@/@PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-Requires(post): /sbin/chkconfig
-Requires(preun): /sbin/service, /sbin/chkconfig
-Requires(postun): /sbin/service
+%{?_without_epoll:%global _without_epoll --disable-epoll}
 
-BuildRequires: bison flex 
-BuildRequires: gcc make
+# fusermount
+# if you wish to compile an rpm without fusermount...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without fusermount
+%{?_without_fusermount:%global _without_fusermount --disable-fusermount}
 
-%description
-GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+# geo-rep
+# if you wish to compile an rpm without geo-replication support, compile like this...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without georeplication
+%{?_without_georeplication:%global _without_georeplication --disable-georeplication}
 
-%package core
-Summary: GlusterFS common files for both the client and the server 
-Group: System Environment/Libraries
-Obsoletes: glusterfs-libs <= 2.0.0 glusterfs-common < 3.1.0 glusterfs-server < 3.1.0
-Provides: glusterfs-libs = %{version}-%{release}
+# gnfs
+# if you wish to compile an rpm with the legacy gNFS server xlator
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with gnfs
+%{?_with_gnfs:%global _with_gnfs --enable-gnfs}
 
-%description core
-GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
-
-This package includes the glusterfs binary, the glusterd daemon and the gluster
-command line, libglusterfs and glusterfs translator modules common to both
-GlusterFS server and client framework.
-
-%if 0%{!?_without_rdma:1}
-%package rdma
-Summary: GlusterFS rdma
-Group: Applications/File
-Requires: glusterfs-core = %{version}-%{release}
-BuildRequires: libibverbs-devel
-
-%description rdma
-GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
-or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
-terms of features and extensibility.  It borrows a powerful concept
-called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+# ipv6default
+# if you wish to compile an rpm with IPv6 default...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with ipv6default
+%{?_with_ipv6default:%global _with_ipv6default --with-ipv6-default}
+
+# libtirpc
+# if you wish to compile an rpm without TIRPC (i.e. use legacy glibc rpc)
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without libtirpc
+%{?_without_libtirpc:%global _without_libtirpc --without-libtirpc}
+
+# Do not use libtirpc on EL6, it does not have xdr_uint64_t() and xdr_uint32_t
+# Do not use libtirpc on EL7, it does not have xdr_sizeof()
+%if ( 0%{?rhel} && 0%{?rhel} <= 7 )
+%global _without_libtirpc --without-libtirpc
+%endif
+
+
+# ocf
+# if you wish to compile an rpm without the OCF resource agents...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without ocf
+%{?_without_ocf:%global _without_ocf --without-ocf}
+
+# server
+# if you wish to build rpms without server components, compile like this
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without server
+%{?_without_server:%global _without_server --without-server}
+
+# disable server components forcefully as rhel <= 6
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%global _without_server --without-server
+%endif
+
+# syslog
+# if you wish to build rpms without syslog logging, compile like this
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without syslog
+%{?_without_syslog:%global _without_syslog --disable-syslog}
+
+# disable syslog forcefully as rhel <= 6 doesn't have rsyslog or rsyslog-mmcount
+# Fedora deprecated syslog, see
+#  https://fedoraproject.org/wiki/Changes/NoDefaultSyslog
+# (And what about RHEL7?)
+%if ( 0%{?fedora} && 0%{?fedora} >= 20 ) || ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%global _without_syslog --disable-syslog
+%endif
+
+# tsan
+# if you wish to compile an rpm with thread sanitizer...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with tsan
+%{?_with_tsan:%global _with_tsan --enable-tsan}
+
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
+%global _with_tsan %{nil}
+%endif
+
+# valgrind
+# if you wish to compile an rpm to run all processes under valgrind...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with valgrind
+%{?_with_valgrind:%global _with_valgrind --enable-valgrind}
+
+##-----------------------------------------------------------------------------
+## All %%global definitions should be placed here and keep them sorted
+##
+
+# selinux booleans whose defalut value needs modification
+# these booleans will be consumed by "%%selinux_set_booleans" macro.
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+%global selinuxbooleans rsync_full_access=1 rsync_client=1
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+%global _with_systemd true
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 7 )
+%global _with_firewalld --enable-firewalld
+%endif
+
+%if 0%{?_tmpfilesdir:1}
+%global _with_tmpfilesdir --with-tmpfilesdir=%{_tmpfilesdir}
+%else
+%global _with_tmpfilesdir --without-tmpfilesdir
+%endif
+
+# without server should also disable some server-only components
+%if 0%{?_without_server:1}
+%global _without_events --disable-events
+%global _without_georeplication --disable-georeplication
+%global _with_gnfs %{nil}
+%global _without_ocf --without-ocf
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+%global _usepython3 1
+%global _pythonver 3
+%else
+%global _usepython3 0
+%global _pythonver 2
+%endif
+
+# From https://fedoraproject.org/wiki/Packaging:Python#Macros
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%{!?python2_sitelib: %global python2_sitelib %(python2 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?python2_sitearch: %global python2_sitearch %(python2 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+%global _rundir %{_localstatedir}/run
+%endif
+
+%if ( 0%{?_with_systemd:1} )
+%global service_start()   /bin/systemctl --quiet start %1.service || : \
+%{nil}
+%global service_stop()    /bin/systemctl --quiet stop %1.service || :\
+%{nil}
+%global service_install() install -D -p -m 0644 %1.service %{buildroot}%2 \
+%{nil}
+# can't seem to make a generic macro that works
+%global glusterd_svcfile   %{_unitdir}/glusterd.service
+%global glusterfsd_svcfile %{_unitdir}/glusterfsd.service
+%global glusterta_svcfile %{_unitdir}/gluster-ta-volume.service
+%global glustereventsd_svcfile %{_unitdir}/glustereventsd.service
+%global glusterfssharedstorage_svcfile %{_unitdir}/glusterfssharedstorage.service
+%else
+%global systemd_post()  /sbin/chkconfig --add %1 >/dev/null 2>&1 || : \
+%{nil}
+%global systemd_preun() /sbin/chkconfig --del %1 >/dev/null 2>&1 || : \
+%{nil}
+%global systemd_postun_with_restart() /sbin/service %1 condrestart >/dev/null 2>&1 || : \
+%{nil}
+%global service_start()   /sbin/service %1 start >/dev/null 2>&1 || : \
+%{nil}
+%global service_stop()    /sbin/service %1 stop >/dev/null 2>&1 || : \
+%{nil}
+%global service_install() install -D -p -m 0755 %1.init %{buildroot}%2 \
+%{nil}
+# can't seem to make a generic macro that works
+%global glusterd_svcfile   %{_sysconfdir}/init.d/glusterd
+%global glusterfsd_svcfile %{_sysconfdir}/init.d/glusterfsd
+%global glustereventsd_svcfile %{_sysconfdir}/init.d/glustereventsd
+%endif
+
+%{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}}
+
+# We do not want to generate useless provides and requires for xlator
+# .so files to be set for glusterfs packages.
+# Filter all generated:
+#
+# TODO: RHEL5 does not have a convenient solution
+%if ( 0%{?rhel} == 6 )
+# filter_setup exists in RHEL6 only
+%filter_provides_in %{_libdir}/glusterfs/%{version}/
+%global __filter_from_req %{?__filter_from_req} | grep -v -P '^(?!lib).*\.so.*$'
+%filter_setup
+%else
+# modern rpm and current Fedora do not generate requires when the
+# provides are filtered
+%global __provides_exclude_from ^%{_libdir}/glusterfs/%{version}/.*$
+%endif
+
+
+##-----------------------------------------------------------------------------
+## All package definitions should be placed here in alphabetical order
+##
+Summary:          Distributed File System
+%if ( 0%{_for_fedora_koji_builds} )
+Name:             glusterfs
+Version:          3.8.0
+Release:          0.1%{?prereltag:.%{prereltag}}%{?dist}
+%else
+Name:             @PACKAGE_NAME@
+Version:          @PACKAGE_VERSION@
+Release:          0.@PACKAGE_RELEASE@%{?dist}
+%endif
+License:          GPLv2 or LGPLv3+
+URL:              http://docs.gluster.org/
+%if ( 0%{_for_fedora_koji_builds} )
+Source0:          http://bits.gluster.org/pub/gluster/glusterfs/src/glusterfs-%{version}%{?prereltag}.tar.gz
+Source1:          glusterd.sysconfig
+Source2:          glusterfsd.sysconfig
+Source7:          glusterfsd.service
+Source8:          glusterfsd.init
+%else
+Source0:          @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
+%endif
+
+BuildRoot:        %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
+
+Requires(pre):    shadow-utils
+%if ( 0%{?_with_systemd:1} )
+BuildRequires:    systemd
+%endif
+
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libgfrpc0%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr0%{?_isa} = %{version}-%{release}
+%if ( 0%{?_with_systemd:1} )
+%{?systemd_requires}
+%endif
+%if 0%{?_with_asan:1} && !( 0%{?rhel} && 0%{?rhel} < 7 )
+BuildRequires:    libasan
+%endif
+%if 0%{?_with_tsan:1} && !( 0%{?rhel} && 0%{?rhel} < 7 )
+BuildRequires:    libtsan
+%endif
+BuildRequires:    bison flex
+BuildRequires:    gcc make libtool
+BuildRequires:    ncurses-devel readline-devel
+BuildRequires:    libxml2-devel openssl-devel
+BuildRequires:    libaio-devel libacl-devel
+BuildRequires:    python%{_pythonver}-devel
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
+BuildRequires:    python-ctypes
+%endif
+%if ( 0%{?_with_ipv6default:1} ) || ( 0%{!?_without_libtirpc:1} )
+BuildRequires:    libtirpc-devel
+%endif
+%if ( 0%{?fedora} && 0%{?fedora} > 27 ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+BuildRequires:    rpcgen
+%endif
+BuildRequires:    userspace-rcu-devel >= 0.7
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+BuildRequires:    automake
+%endif
+BuildRequires:    libuuid-devel
+%if ( 0%{?_with_cmocka:1} )
+BuildRequires:    libcmocka-devel >= 1.0.1
+%endif
+%if ( 0%{!?_without_georeplication:1} )
+BuildRequires:    libattr-devel
+%endif
 
-This package provides support to ib-verbs library.
+%if (0%{?_with_firewalld:1})
+BuildRequires:    firewalld
 %endif
 
+Obsoletes:        %{name}-common < %{version}-%{release}
+Obsoletes:        %{name}-core < %{version}-%{release}
+Obsoletes:        %{name}-ganesha
+Obsoletes:        %{name}-rdma < %{version}-%{release}
+%if ( 0%{!?_with_gnfs:1} )
+Obsoletes:        %{name}-gnfs < %{version}-%{release}
+%endif
+Provides:         %{name}-common = %{version}-%{release}
+Provides:         %{name}-core = %{version}-%{release}
+
+%description
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package includes the glusterfs binary, the glusterfsd daemon and the
+libglusterfs and glusterfs translator modules common to both GlusterFS server
+and client framework.
+
+%package cli
+Summary:          GlusterFS CLI
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libglusterd0%{?_isa} = %{version}-%{release}
+
+%description cli
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the GlusterFS CLI application and its man page
+
+%package cloudsync-plugins
+Summary:          Cloudsync Plugins
+BuildRequires:    libcurl-devel
+
+%description cloudsync-plugins
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides cloudsync plugins for archival feature.
+
+%package extra-xlators
+Summary:          Extra Gluster filesystem Translators
+# We need python-gluster rpm for gluster module's __init__.py in Python
+# site-packages area
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+Requires:         python%{_pythonver}
+
+%description extra-xlators
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides extra filesystem Translators, such as Glupy,
+for GlusterFS.
+
 %package fuse
-Summary: GlusterFS Fuse
-Group: Applications/File
-Requires: glusterfs-core = %{version}-%{release}
-Obsoletes: glusterfs-client < 3.1.0
+Summary:          Fuse client
+BuildRequires:    fuse-devel
+Requires:         attr
+Requires:         psmisc
+
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+
+Obsoletes:        %{name}-client < %{version}-%{release}
+Provides:         %{name}-client = %{version}-%{release}
 
 %description fuse
-GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides support to FUSE based clients and inlcudes the
+glusterfs(d) binary.
+
+%if ( 0%{!?_without_server:1} )
+%package ganesha
+Summary:          NFS-Ganesha configuration
+Group:            Applications/File
+
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+Requires:         nfs-ganesha-selinux >= 2.7.6
+Requires:         nfs-ganesha-gluster >= 2.7.6
+Requires:         pcs >= 0.10.0
+Requires:         resource-agents >= 4.2.0
+Requires:         dbus
+
+%if ( 0%{?rhel} && 0%{?rhel} == 6 )
+Requires:         cman, pacemaker, corosync
+%endif
+
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 5 )
+# we need portblock resource-agent in 3.9.5 and later.
+Requires:         net-tools
+%endif
+
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
+Requires: selinux-policy >= 3.13.1-160
+Requires(post):   policycoreutils-python
+Requires(postun): policycoreutils-python
+%else
+Requires(post):   policycoreutils-python-utils
+Requires(postun): policycoreutils-python-utils
+%endif
+%endif
+
+%description ganesha
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
 or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
+system. GlusterFS is one of the most sophisticated file systems in
 terms of features and extensibility.  It borrows a powerful concept
 called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+is in user space and easily manageable.
+
+This package provides the configuration and related files for using
+NFS-Ganesha as the NFS server using GlusterFS
+%endif
+
+%if ( 0%{!?_without_georeplication:1} )
+%package geo-replication
+Summary:          GlusterFS Geo-replication
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+Requires:         python%{_pythonver}
+Requires:         python%{_pythonver}-prettytable
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+
+Requires:         rsync
+Requires:         util-linux
+# required for setting selinux bools
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+Requires(post):      policycoreutils-python-utils
+Requires(postun):    policycoreutils-python-utils
+Requires:            selinux-policy-targeted
+Requires(post):      selinux-policy-targeted
+BuildRequires:       selinux-policy-devel
+%endif
+
+%description geo-replication
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides support to geo-replication.
+%endif
+
+%if ( 0%{?_with_gnfs:1} )
+%package gnfs
+Summary:          GlusterFS gNFS server
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+Requires:         nfs-utils
+
+%description gnfs
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the glusterfs legacy gNFS server xlator
+%endif
+
+%package -n libglusterfs0
+Summary:          GlusterFS libglusterfs library
+Requires:         libgfrpc0%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+Provides:         %{name}-libs = %{version}-%{release}
+
+%description -n libglusterfs0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the base libglusterfs library
+
+%package -n libglusterfs-devel
+Summary:          GlusterFS libglusterfs library
+Requires:         libgfrpc-devel%{?_isa} = %{version}-%{release}
+Requires:         libgfxdr-devel%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+Provides:         %{name}-devel = %{version}-%{release}
+
+%description -n libglusterfs-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libglusterfs.so and the gluster C header files.
+
+%package -n libgfapi0
+Summary:          GlusterFS api library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-api <= %{version}-%{release}
+Provides:         %{name}-api = %{version}-%{release}
+
+%description -n libgfapi0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the glusterfs libgfapi library.
+
+%package -n libgfapi-devel
+Summary:          Development Libraries
+Requires:         libglusterfs-devel%{?_isa} = %{version}-%{release}
+Requires:         libacl-devel
+Obsoletes:        %{name}-api-devel <= %{version}-%{release}
+Provides:         %{name}-api-devel = %{version}-%{release}
+
+%description -n libgfapi-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfapi.so and the api C header files.
+
+%package -n libgfchangelog0
+Summary:          GlusterFS libchangelog library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfchangelog0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfchangelog library
+
+%package -n libgfchangelog-devel
+Summary:          GlusterFS libchangelog library
+Requires:         libglusterfs-devel%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfchangelog-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfchangelog.so and changelog C header files.
+
+%package -n libgfrpc0
+Summary:          GlusterFS libgfrpc0 library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfrpc0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfrpc library
 
-This package provides support to FUSE based clients.
+%package -n libgfrpc-devel
+Summary:          GlusterFS libgfrpc library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfrpc-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfrpc.so and rpc C header files.
+
+%package -n libgfxdr0
+Summary:          GlusterFS libgfxdr0 library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libgfxdr0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libgfxdr library
+
+%package -n libgfxdr-devel
+Summary:          GlusterFS libgfxdr library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-devel <= %{version}-%{release}
+
+%description -n libgfxdr-devel
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides libgfxdr.so.
+
+%package -n libglusterd0
+Summary:          GlusterFS libglusterd library
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Obsoletes:        %{name}-libs <= %{version}-%{release}
+
+%description -n libglusterd0
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the libglusterd library
+
+%package -n python%{_pythonver}-gluster
+Summary:          GlusterFS python library
+Requires:         python%{_pythonver}
+%if ( ! %{_usepython3} )
+%{?python_provide:%python_provide python-gluster}
+Provides:         python-gluster = %{version}-%{release}
+Obsoletes:        python-gluster < 3.10
+%endif
+
+%description -n python%{_pythonver}-gluster
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package contains the python modules of GlusterFS and own gluster
+namespace.
+
+%package regression-tests
+Summary:          Development Tools
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-fuse%{?_isa} = %{version}-%{release}
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+## thin provisioning support
+Requires:         lvm2 >= 2.02.89
+Requires:         perl(App::Prove) perl(Test::Harness) gcc util-linux-ng
+Requires:         python%{_pythonver}
+Requires:         attr dbench file git libacl-devel net-tools
+Requires:         nfs-utils xfsprogs yajl psmisc bc
+
+%description regression-tests
+The Gluster Test Framework, is a suite of scripts used for
+regression testing of Gluster.
+
+%if ( 0%{!?_without_ocf:1} )
+%package resource-agents
+Summary:          OCF Resource Agents for GlusterFS
+License:          GPLv3+
+BuildArch:        noarch
+# this Group handling comes from the Fedora resource-agents package
+# for glusterd
+Requires:         %{name}-server = %{version}-%{release}
+# depending on the distribution, we need pacemaker or resource-agents
+Requires:         %{_prefix}/lib/ocf/resource.d
+
+%description resource-agents
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the resource agents which plug glusterd into
+Open Cluster Framework (OCF) compliant cluster resource managers,
+like Pacemaker.
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%package server
+Summary:          Clustered file-system server
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-cli%{?_isa} = %{version}-%{release}
+Requires:         libglusterfs0%{?_isa} = %{version}-%{release}
+Requires:         libgfchangelog0%{?_isa} = %{version}-%{release}
+%if ( 0%{?fedora} && 0%{?fedora} >= 30  || ( 0%{?rhel} && 0%{?rhel} >= 8 ) )
+Requires:         glusterfs-selinux >= 0.1.0-2
+%endif
+# some daemons (like quota) use a fuse-mount, glusterfsd is part of -fuse
+Requires:         %{name}-fuse%{?_isa} = %{version}-%{release}
+# self-heal daemon, rebalance, nfs-server etc. are actually clients
+Requires:         libgfapi0%{?_isa} = %{version}-%{release}
+Requires:         %{name}-client-xlators%{?_isa} = %{version}-%{release}
+# lvm2 for snapshot, and nfs-utils and rpcbind/portmap for gnfs server
+Requires:         lvm2
+%if ( 0%{?_with_systemd:1} )
+%{?systemd_requires}
+%else
+Requires(post):   /sbin/chkconfig
+Requires(preun):  /sbin/service
+Requires(preun):  /sbin/chkconfig
+Requires(postun): /sbin/service
+%endif
+%if (0%{?_with_firewalld:1})
+# we install firewalld rules, so we need to have the directory owned
+%if ( 0%{!?rhel} )
+# not on RHEL because firewalld-filesystem appeared in 7.3
+# when EL7 rpm gets weak dependencies we can add a Suggests:
+Requires:         firewalld-filesystem
+%endif
+%endif
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+Requires:         rpcbind
+%else
+Requires:         portmap
+%endif
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+Requires:         python-argparse
+%endif
+%if ( 0%{?fedora} && 0%{?fedora} > 27 ) || ( 0%{?rhel} && 0%{?rhel} > 7 )
+Requires:         python%{_pythonver}-pyxattr
+%else
+Requires:         pyxattr
+%endif
+%if (0%{?_with_valgrind:1})
+Requires:         valgrind
+%endif
+
+%description server
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the glusterfs server daemon.
+%endif
+
+%package thin-arbiter
+Summary:          GlusterFS thin-arbiter module
+Requires:         %{name}%{?_isa} = %{version}-%{release}
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+
+%description thin-arbiter
+This package provides a tie-breaker functionality to GlusterFS
+replicate volume. It includes translators required to provide the
+functionality, and also few other scripts required for getting the setup done.
+
+This package provides the glusterfs thin-arbiter translator.
+
+%package client-xlators
+Summary:          GlusterFS client-side translators
+
+%description client-xlators
+GlusterFS is a distributed file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over TCP/IP interconnect
+into one large parallel network filesystem. GlusterFS is one of the
+most sophisticated file systems in terms of features and extensibility.
+It borrows a powerful concept called Translators from GNU Hurd kernel.
+Much of the code in GlusterFS is in user space and easily manageable.
+
+This package provides the translators needed on any GlusterFS client.
+
+%if ( 0%{!?_without_events:1} )
+%package events
+Summary:          GlusterFS Events
+Requires:         %{name}-server%{?_isa} = %{version}-%{release}
+Requires:         python%{_pythonver} python%{_pythonver}-prettytable
+Requires:         python%{_pythonver}-gluster = %{version}-%{release}
+%if ( 0%{?rhel} && 0%{?rhel} < 8 )
+Requires:         python-requests
+%else
+Requires:         python%{_pythonver}-requests
+%endif
+%if ( 0%{?rhel} && 0%{?rhel} < 7 )
+Requires:         python-argparse
+%endif
+%if ( 0%{?_with_systemd:1} )
+%{?systemd_requires}
+%endif
+
+%description events
+GlusterFS Events
+
+%endif
 
 %prep
-%setup -q -n %{name}-%{version}
+%setup -q -n %{name}-%{version}%{?prereltag}
+%if ( ! %{_usepython3} )
+echo "fixing python shebangs..."
+for f in api events extras geo-replication libglusterfs tools xlators; do
+find $f -type f -exec sed -i 's|/usr/bin/python3|/usr/bin/python2|' {} \;
+done
+%endif
 
 %build
 
-%configure %{?_without_rdma} %{?_without_epoll} %{?_with_fusermount}
+# RHEL6 and earlier need to manually replace config.guess and config.sub
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+./autogen.sh
+%endif
+
+%configure \
+        %{?_with_asan} \
+        %{?_with_cmocka} \
+        %{?_with_debug} \
+        %{?_with_firewalld} \
+        %{?_with_gnfs} \
+        %{?_with_tmpfilesdir} \
+        %{?_with_tsan} \
+        %{?_with_valgrind} \
+        %{?_without_epoll} \
+        %{?_without_events} \
+        %{?_without_fusermount} \
+        %{?_without_georeplication} \
+        %{?_without_ocf} \
+        %{?_without_server} \
+        %{?_without_syslog} \
+        %{?_with_ipv6default} \
+        %{?_without_libtirpc}
+
+# fix hardening and remove rpath in shlibs
+%if ( 0%{?fedora} && 0%{?fedora} > 17 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+sed -i 's| \\\$compiler_flags |&\\\$LDFLAGS |' libtool
+%endif
+sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|' libtool
+sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|' libtool
 
-# Remove rpath
-sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool
-sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
-%{__make} %{?_smp_mflags}
+make %{?_smp_mflags}
 
+%check
+make check
 
 %install
-%{__rm} -rf %{buildroot} 
-%{__make} install DESTDIR=%{buildroot}
-%{__mkdir_p} %{buildroot}%{_includedir}/glusterfs
-%{__mkdir_p} %{buildroot}/var/log/glusterfs
-%{__install} -p -m 0644 libglusterfs/src/*.h \
-    %{buildroot}%{_includedir}/glusterfs/
+rm -rf %{buildroot}
+make install DESTDIR=%{buildroot}
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{_for_fedora_koji_builds} )
+install -D -p -m 0644 %{SOURCE1} \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterd
+install -D -p -m 0644 %{SOURCE2} \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterfsd
+%else
+install -D -p -m 0644 extras/glusterd-sysconfig \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterd
+%endif
+%endif
+
+mkdir -p %{buildroot}%{_localstatedir}/log/glusterd
+mkdir -p %{buildroot}%{_localstatedir}/log/glusterfs
+mkdir -p %{buildroot}%{_localstatedir}/log/glusterfsd
+mkdir -p %{buildroot}%{_rundir}/gluster
 
 # Remove unwanted files from all the shared libraries
-find %{buildroot}%{_libdir} -name '*.la' | xargs rm -f
-find %{buildroot}%{_libdir} -name '*.a' | xargs rm -f
+find %{buildroot}%{_libdir} -name '*.a' -delete
+find %{buildroot}%{_libdir} -name '*.la' -delete
+
+# Remove installed docs, the ones we want are included by %%doc, in
+# /usr/share/doc/glusterfs or /usr/share/doc/glusterfs-x.y.z depending
+# on the distribution
+%if ( 0%{?fedora} && 0%{?fedora} > 19 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+rm -rf %{buildroot}%{_pkgdocdir}/*
+%else
+rm -rf %{buildroot}%{_defaultdocdir}/%{name}
+mkdir -p %{buildroot}%{_pkgdocdir}
+%endif
+head -50 ChangeLog > ChangeLog.head && mv ChangeLog.head ChangeLog
+cat << EOM >> ChangeLog
+
+More commit messages for this ChangeLog can be found at
+https://forge.gluster.org/glusterfs-core/glusterfs/commits/v%{version}%{?prereltag}
+EOM
+
+# Remove benchmarking and other unpackaged files
+# make install always puts these in %%{_defaultdocdir}/%%{name} so don't
+# use %%{_pkgdocdir}; that will be wrong on later Fedora distributions
+rm -rf %{buildroot}%{_defaultdocdir}/%{name}/benchmarking
+rm -f %{buildroot}%{_defaultdocdir}/%{name}/glusterfs-mode.el
+rm -f %{buildroot}%{_defaultdocdir}/%{name}/glusterfs.vim
+
+%if ( 0%{!?_without_server:1} )
+# Create working directory
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd
+
+# Update configuration file to /var/lib working directory
+sed -i 's|option working-directory /etc/glusterd|option working-directory %{_sharedstatedir}/glusterd|g' \
+    %{buildroot}%{_sysconfdir}/glusterfs/glusterd.vol
+%endif
+
+# Install glusterfsd .service or init.d file
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{_for_fedora_koji_builds} )
+%service_install glusterfsd %{glusterfsd_svcfile}
+%endif
+%endif
+
+install -D -p -m 0644 extras/glusterfs-logrotate \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs
+
+# ganesha ghosts
+%if ( 0%{!?_without_server:1} )
+mkdir -p %{buildroot}%{_sysconfdir}/ganesha
+touch %{buildroot}%{_sysconfdir}/ganesha/ganesha-ha.conf
+mkdir -p %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/
+touch %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha.conf
+touch %{buildroot}%{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha-ha.conf
+%endif
+
+%if ( 0%{!?_without_georeplication:1} )
+# geo-rep ghosts
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/geo-replication
+touch %{buildroot}%{_sharedstatedir}/glusterd/geo-replication/gsyncd_template.conf
+install -D -p -m 0644 extras/glusterfs-georep-logrotate \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs-georep
+%endif
+
+%if ( 0%{!?_without_server:1} )
+# the rest of the ghosts
+touch %{buildroot}%{_sharedstatedir}/glusterd/glusterd.info
+touch %{buildroot}%{_sharedstatedir}/glusterd/options
+subdirs=(add-brick create copy-file delete gsync-create remove-brick reset set start stop)
+for dir in ${subdirs[@]}; do
+    mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/"$dir"/{pre,post}
+done
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/glustershd
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/peers
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/vols
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/nfs/run
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/bitd
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/quotad
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/scrub
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/snaps
+mkdir -p %{buildroot}%{_sharedstatedir}/glusterd/ss_brick
+touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+%endif
+
+find ./tests ./run-tests.sh -type f | cpio -pd %{buildroot}%{_prefix}/share/glusterfs
+
+## Install bash completion for cli
+install -p -m 0744 -D extras/command-completion/gluster.bash \
+    %{buildroot}%{_sysconfdir}/bash_completion.d/gluster
 
 %clean
-%{__rm} -rf %{buildroot}
-
-%post core
-/sbin/ldconfig -n %{_libdir}
-/sbin/chkconfig --add glusterd
-#killall glusterfsd,glusterfs,glusterd
-/usr/bin/killall glusterfsd 2>/dev/null || true;
-/usr/bin/killall glusterfs 2>/dev/null || true;
-/usr/bin/killall glusterd 2>/dev/null || true;
-#start glusterd
-/usr/sbin/glusterd
-
-%postun core
-/sbin/ldconfig -n %{_libdir}
-
-%preun core
-if [ $1 -eq 0 ]; then 
-    /sbin/chkconfig --del glusterd
+rm -rf %{buildroot}
+
+##-----------------------------------------------------------------------------
+## All %%post should be placed here and keep them sorted
+##
+%post
+/sbin/ldconfig
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%systemd_postun_with_restart rsyslog
+%endif
+%endif
+exit 0
+
+%if ( 0%{!?_without_events:1} )
+%post events
+%systemd_post glustereventsd
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25 || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%post ganesha
+semanage boolean -m ganesha_use_fusefs --on
+exit 0
+%endif
+%endif
+
+%if ( 0%{!?_without_georeplication:1} )
+%post geo-replication
+%if ( 0%{?rhel} && 0%{?rhel} >= 8 )
+%selinux_set_booleans %{selinuxbooleans}
+%endif
+if [ $1 -ge 1 ]; then
+    %systemd_postun_with_restart glusterd
 fi
+exit 0
+%endif
 
-%files core
-%defattr(-,root,root)
-%doc AUTHORS ChangeLog COPYING INSTALL NEWS README THANKS
-%doc /usr/share/doc/glusterfs
-%if 0%{?_with_fusermount:1}
-%{_bindir}/fusermount-glusterfs 
-%endif
-%{_datadir}/glusterfs
-%{_bindir}/glusterfs-volgen
-%{_libdir}/glusterfs
-%{_libdir}/*.so.*
-%{_sbindir}/glusterfs*
-%{_bindir}/glusterfs-defrag
-%{_sbindir}/gluster
-%{_sbindir}/glusterd
-%{_mandir}/man8/*glusterfs.8*
-%{_mandir}/man8/*gluster.8*
-%{_mandir}/man8/*glusterd.8*
-%{_mandir}/man8/*glusterfs-volgen.8*
-%dir /var/log/glusterfs
-%if 0%{!?_without_rdma:1}
-%exclude %{_libdir}/glusterfs/%{version}/rpc-transport/rdma*
-%endif
-%exclude %{_libdir}/glusterfs/%{version}/xlator/mount/fuse*
-%config %{_sysconfdir}/glusterfs
-%{_sysconfdir}/init.d/glusterd
-%{_includedir}/glusterfs
-%exclude %{_includedir}/glusterfs/y.tab.h
-%{_libdir}/*.so
-
-%if 0%{!?_without_rdma:1}
-%files rdma
-%defattr(-,root,root)
-%{_libdir}/glusterfs/%{version}/rpc-transport/rdma*
+%post -n libglusterfs0
+/sbin/ldconfig
+
+%post -n libgfapi0
+/sbin/ldconfig
+
+%post -n libgfchangelog0
+/sbin/ldconfig
+
+%post -n libgfrpc0
+/sbin/ldconfig
+
+%post -n libgfxdr0
+/sbin/ldconfig
+
+%post -n libglusterd0
+/sbin/ldconfig
+
+%if ( 0%{!?_without_server:1} )
+%post server
+# Legacy server
+%systemd_post glusterd
+%if ( 0%{_for_fedora_koji_builds} )
+%systemd_post glusterfsd
+%endif
+# ".cmd_log_history" is renamed to "cmd_history.log" in GlusterFS-3.7 .
+# While upgrading glusterfs-server package form GlusterFS version <= 3.6 to
+# GlusterFS version 3.7, ".cmd_log_history" should be renamed to
+# "cmd_history.log" to retain cli command history contents.
+if [ -f %{_localstatedir}/log/glusterfs/.cmd_log_history ]; then
+    mv %{_localstatedir}/log/glusterfs/.cmd_log_history \
+       %{_localstatedir}/log/glusterfs/cmd_history.log
+fi
+
+# Genuine Fedora (and EPEL) builds never put gluster files in /etc; if
+# there are any files in /etc from a prior gluster.org install, move them
+# to /var/lib. (N.B. Starting with 3.3.0 all gluster files are in /var/lib
+# in gluster.org RPMs.) Be careful to copy them on the off chance that
+# /etc and /var/lib are on separate file systems
+if [ -d /etc/glusterd -a ! -h %{_sharedstatedir}/glusterd ]; then
+    mkdir -p %{_sharedstatedir}/glusterd
+    cp -a /etc/glusterd %{_sharedstatedir}/glusterd
+    rm -rf /etc/glusterd
+    ln -sf %{_sharedstatedir}/glusterd /etc/glusterd
+fi
+
+# Rename old volfiles in an RPM-standard way.  These aren't actually
+# considered package config files, so %%config doesn't work for them.
+if [ -d %{_sharedstatedir}/glusterd/vols ]; then
+    for file in $(find %{_sharedstatedir}/glusterd/vols -name '*.vol'); do
+        newfile=${file}.rpmsave
+        echo "warning: ${file} saved as ${newfile}"
+        cp ${file} ${newfile}
+    done
+fi
+
+# add marker translator
+# but first make certain that there are no old libs around to bite us
+# BZ 834847
+if [ -e /etc/ld.so.conf.d/glusterfs.conf ]; then
+    rm -f /etc/ld.so.conf.d/glusterfs.conf
+    /sbin/ldconfig
+fi
+
+%if (0%{?_with_firewalld:1})
+    %firewalld_reload
+%endif
+
+pidof -c -o %PPID -x glusterd &> /dev/null
+if [ $? -eq 0 ]; then
+    kill -9 `pgrep -f gsyncd.py` &> /dev/null
+
+    killall --wait glusterd &> /dev/null
+    glusterd --xlator-option *.upgrade=on -N
+
+    #Cleaning leftover glusterd socket file which is created by glusterd in
+    #rpm_script_t context.
+    rm -f %{_rundir}/glusterd.socket
+
+    # glusterd _was_ running, we killed it, it exited after *.upgrade=on,
+    # so start it again
+    %service_start glusterd
+else
+    glusterd --xlator-option *.upgrade=on -N
+
+    #Cleaning leftover glusterd socket file which is created by glusterd in
+    #rpm_script_t context.
+    rm -f %{_rundir}/glusterd.socket
+fi
+exit 0
+%endif
+
+##-----------------------------------------------------------------------------
+## All %%pre should be placed here and keep them sorted
+##
+%pre
+getent group gluster > /dev/null || groupadd -r gluster
+getent passwd gluster > /dev/null || useradd -r -g gluster -d %{_rundir}/gluster -s /sbin/nologin -c "GlusterFS daemons" gluster
+exit 0
+
+##-----------------------------------------------------------------------------
+## All %%preun should be placed here and keep them sorted
+##
+%if ( 0%{!?_without_events:1} )
+%preun events
+if [ $1 -eq 0 ]; then
+    if [ -f %glustereventsd_svcfile ]; then
+        %service_stop glustereventsd
+        %systemd_preun glustereventsd
+    fi
+fi
+exit 0
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%preun server
+if [ $1 -eq 0 ]; then
+    if [ -f %glusterfsd_svcfile ]; then
+        %service_stop glusterfsd
+    fi
+    %service_stop glusterd
+    if [ -f %glusterfsd_svcfile ]; then
+        %systemd_preun glusterfsd
+    fi
+    %systemd_preun glusterd
+fi
+if [ $1 -ge 1 ]; then
+    if [ -f %glusterfsd_svcfile ]; then
+        %systemd_postun_with_restart glusterfsd
+    fi
+    %systemd_postun_with_restart glusterd
+fi
+exit 0
+%endif
+
+%preun thin-arbiter
+if [ $1 -eq 0 ]; then
+    if [ -f %glusterta_svcfile ]; then
+        %service_stop gluster-ta-volume
+        %systemd_preun gluster-ta-volume
+    fi
+fi
+
+##-----------------------------------------------------------------------------
+## All %%postun should be placed here and keep them sorted
+##
+%postun
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%systemd_postun_with_restart rsyslog
+%endif
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%postun server
+%if (0%{?_with_firewalld:1})
+    %firewalld_reload
+%endif
+exit 0
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%postun ganesha
+semanage boolean -m ganesha_use_fusefs --off
+exit 0
+%endif
+%endif
+
+##-----------------------------------------------------------------------------
+## All %%trigger should be placed here and keep them sorted
+##
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%trigger ganesha -- selinux-policy-targeted
+semanage boolean -m ganesha_use_fusefs --on
+exit 0
+%endif
+%endif
+
+##-----------------------------------------------------------------------------
+## All %%triggerun should be placed here and keep them sorted
+##
+%if ( 0%{!?_without_server:1} )
+%if ( 0%{?fedora} && 0%{?fedora} > 25  || ( 0%{?rhel} && 0%{?rhel} > 6 ) )
+%triggerun ganesha -- selinux-policy-targeted
+semanage boolean -m ganesha_use_fusefs --off
+exit 0
+%endif
+%endif
+
+##-----------------------------------------------------------------------------
+## All %%files should be placed here and keep them grouped
+##
+%files
+%doc ChangeLog COPYING-GPLV2 COPYING-LGPLV3 INSTALL README.md THANKS COMMITMENT
+%{_mandir}/man8/*gluster*.8*
+%if ( 0%{!?_without_server:1} )
+%exclude %{_mandir}/man8/gluster.8*
+%endif
+%dir %{_localstatedir}/log/glusterfs
+%if 0%{?!_without_server:1}
+%dir %{_datadir}/glusterfs
+%dir %{_datadir}/glusterfs/scripts
+     %{_datadir}/glusterfs/scripts/post-upgrade-script-for-quota.sh
+     %{_datadir}/glusterfs/scripts/pre-upgrade-script-for-quota.sh
+%endif
+# xlators that are needed on the client- and on the server-side
+%dir %{_libdir}/glusterfs
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/auth
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/auth/addr.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/auth/login.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/socket.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/error-gen.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/delay-gen.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/io-stats.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/sink.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/debug/trace.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/access-control.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/barrier.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/cdc.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/changelog.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/utime.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/gfid-access.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/namespace.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/read-only.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/shard.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-client.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/worm.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/cloudsync.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/meta.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-cache.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/io-threads.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/md-cache.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/open-behind.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/quick-read.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/read-ahead.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/readdir-ahead.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/stat-prefetch.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/write-behind.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/performance/nl-cache.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/system
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/system/posix-acl.so
+%dir %attr(0775,gluster,gluster) %{_rundir}/gluster
+%if 0%{?_tmpfilesdir:1} && 0%{!?_without_server:1}
+%{_tmpfilesdir}/gluster.conf
 %endif
 
+%if ( 0%{?_without_server:1} )
+#exclude ganesha related files
+%exclude %{_sysconfdir}/ganesha/ganesha-ha.conf.sample
+%exclude %{_libexecdir}/ganesha/*
+%exclude %{_prefix}/lib/ocf/resource.d/heartbeat/*
+%endif
+
+%files cli
+%{_sbindir}/gluster
+%{_mandir}/man8/gluster.8*
+%{_sysconfdir}/bash_completion.d/gluster
+
+%files cloudsync-plugins
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsyncs3.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/cloudsync-plugins/cloudsynccvlt.so
+
+%files -n libglusterfs-devel
+%dir %{_includedir}/glusterfs
+     %{_includedir}/glusterfs/*.h
+     %{_includedir}/glusterfs/server/*.h
+%{_libdir}/libglusterfs.so
+
+%files -n libgfapi-devel
+%dir %{_includedir}/glusterfs/api
+     %{_includedir}/glusterfs/api/*.h
+%{_libdir}/libgfapi.so
+%{_libdir}/pkgconfig/glusterfs-api.pc
+
+
+%files -n libgfchangelog-devel
+%dir %{_includedir}/glusterfs/gfchangelog
+     %{_includedir}/glusterfs/gfchangelog/*.h
+%{_libdir}/libgfchangelog.so
+%{_libdir}/pkgconfig/libgfchangelog.pc
+
+%files -n libgfrpc-devel
+%dir %{_includedir}/glusterfs/rpc
+     %{_includedir}/glusterfs/rpc/*.h
+%{_libdir}/libgfrpc.so
+
+%files -n libgfxdr-devel
+%{_libdir}/libgfxdr.so
+
+%files client-xlators
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/cluster/*.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/client.so
+
+%files extra-xlators
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quiesce.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/playground
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/playground/template.so
+
 %files fuse
-%defattr(-,root,root)
-%{_libdir}/glusterfs/%{version}/xlator/mount/fuse*
+# glusterfs is a symlink to glusterfsd, -server depends on -fuse.
+%{_sbindir}/glusterfs
+%{_sbindir}/glusterfsd
+%config(noreplace) %{_sysconfdir}/logrotate.d/glusterfs
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/fuse.so
 /sbin/mount.glusterfs
+%if ( 0%{!?_without_fusermount:1} )
+%{_bindir}/fusermount-glusterfs
+%endif
+
+%if ( 0%{?_with_gnfs:1} && 0%{!?_without_server:1} )
+%files gnfs
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs/server.so
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs/run
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+%endif
+
+%files thin-arbiter
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/thin-arbiter.so
+%dir %{_datadir}/glusterfs/scripts
+     %{_datadir}/glusterfs/scripts/setup-thin-arbiter.sh
+%config %{_sysconfdir}/glusterfs/thin-arbiter.vol
+
+%if ( 0%{?_with_systemd:1} )
+%{_unitdir}/gluster-ta-volume.service
+%endif
+
+%if ( 0%{!?_without_georeplication:1} )
+%files geo-replication
+%config(noreplace) %{_sysconfdir}/logrotate.d/glusterfs-georep
+
+%{_sbindir}/gfind_missing_files
+%{_sbindir}/gluster-mountbroker
+%dir %{_libexecdir}/glusterfs
+%dir %{_libexecdir}/glusterfs/python
+%dir %{_libexecdir}/glusterfs/python/syncdaemon
+     %{_libexecdir}/glusterfs/gsyncd
+     %{_libexecdir}/glusterfs/python/syncdaemon/*
+%dir %{_libexecdir}/glusterfs/scripts
+     %{_libexecdir}/glusterfs/scripts/get-gfid.sh
+     %{_libexecdir}/glusterfs/scripts/slave-upgrade.sh
+     %{_libexecdir}/glusterfs/scripts/gsync-upgrade.sh
+     %{_libexecdir}/glusterfs/scripts/generate-gfid-file.sh
+     %{_libexecdir}/glusterfs/scripts/gsync-sync-gfid
+     %{_libexecdir}/glusterfs/scripts/schedule_georep.py*
+     %{_libexecdir}/glusterfs/gverify.sh
+     %{_libexecdir}/glusterfs/set_geo_rep_pem_keys.sh
+     %{_libexecdir}/glusterfs/peer_gsec_create
+     %{_libexecdir}/glusterfs/peer_mountbroker
+     %{_libexecdir}/glusterfs/peer_mountbroker.py*
+     %{_libexecdir}/glusterfs/gfind_missing_files
+     %{_libexecdir}/glusterfs/peer_georep-sshkey.py*
+%{_sbindir}/gluster-georep-sshkey
+
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/geo-replication
+%ghost      %attr(0644,-,-) %{_sharedstatedir}/glusterd/geo-replication/gsyncd_template.conf
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create/post/S56glusterd-geo-rep-create-post.sh
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/gsync-create/pre
+
+%endif
+
+%files -n libglusterfs0
+%{_libdir}/libglusterfs.so.*
+
+%files -n libgfapi0
+%{_libdir}/libgfapi.so.*
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api.so
+
+%files -n libgfchangelog0
+%{_libdir}/libgfchangelog.so.*
+
+%files -n libgfrpc0
+%{_libdir}/libgfrpc.so.*
+
+%files -n libgfxdr0
+%{_libdir}/libgfxdr.so.*
+
+%files -n libglusterd0
+%{_libdir}/libglusterd.so.*
+%exclude %{_libdir}/libglusterd.so
+
+%files -n python%{_pythonver}-gluster
+# introducing glusterfs module in site packages.
+# so that all other gluster submodules can reside in the same namespace.
+%if ( %{_usepython3} )
+%dir %{python3_sitelib}/gluster
+     %{python3_sitelib}/gluster/__init__.*
+     %{python3_sitelib}/gluster/__pycache__
+     %{python3_sitelib}/gluster/cliutils
+%else
+%dir %{python2_sitelib}/gluster
+     %{python2_sitelib}/gluster/__init__.*
+     %{python2_sitelib}/gluster/cliutils
+%endif
+
+%files regression-tests
+%dir %{_datadir}/glusterfs
+     %{_datadir}/glusterfs/run-tests.sh
+     %{_datadir}/glusterfs/tests
+%exclude %{_datadir}/glusterfs/tests/vagrant
+
+%if ( 0%{!?_without_server:1} )
+%files ganesha
+%dir %{_libexecdir}/ganesha
+%{_sysconfdir}/ganesha/ganesha-ha.conf.sample
+%{_libexecdir}/ganesha/*
+%{_prefix}/lib/ocf/resource.d/heartbeat/*
+%{_sharedstatedir}/glusterd/hooks/1/start/post/S31ganesha-start.sh
+%ghost      %attr(0644,-,-) %config(noreplace) %{_sysconfdir}/ganesha/ganesha-ha.conf
+%ghost %dir %attr(0755,-,-) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha
+%ghost      %attr(0644,-,-) %config(noreplace) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha.conf
+%ghost      %attr(0644,-,-) %config(noreplace) %{_localstatedir}/run/gluster/shared_storage/nfs-ganesha/ganesha-ha.conf
+%endif
+
+%if ( 0%{!?_without_ocf:1} )
+%files resource-agents
+# /usr/lib is the standard for OCF, also on x86_64
+%{_prefix}/lib/ocf/resource.d/glusterfs
+%endif
+
+%if ( 0%{!?_without_server:1} )
+%files server
+%doc extras/clear_xattrs.sh
+# sysconf
+%config(noreplace) %{_sysconfdir}/glusterfs
+%exclude %{_sysconfdir}/glusterfs/thin-arbiter.vol
+%exclude %{_sysconfdir}/glusterfs/eventsconfig.json
+%exclude %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+%exclude %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+%if ( 0%{?_with_gnfs:1} )
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs/*
+%endif
+%config(noreplace) %{_sysconfdir}/sysconfig/glusterd
+%if ( 0%{_for_fedora_koji_builds} )
+%config(noreplace) %{_sysconfdir}/sysconfig/glusterfsd
+%endif
+
+# init files
+%glusterd_svcfile
+%if ( 0%{_for_fedora_koji_builds} )
+%glusterfsd_svcfile
+%endif
+%if ( 0%{?_with_systemd:1} )
+%glusterfssharedstorage_svcfile
+%endif
+
+# binaries
+%{_sbindir}/glusterd
+%{_libexecdir}/glusterfs/glfsheal
+%{_sbindir}/gf_attach
+%{_sbindir}/gluster-setgfid2path
+# {_sbindir}/glusterfsd is the actual binary, but glusterfs (client) is a
+# symlink. The binary itself (and symlink) are part of the glusterfs-fuse
+# package, because glusterfs-server depends on that anyway.
+
+# Manpages
+%{_mandir}/man8/gluster-setgfid2path.8*
+
+# xlators
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/arbiter.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bit-rot.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/bitrot-stub.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/sdfs.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/index.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/locks.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix*
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/snapview-server.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/marker.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/quota*
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/selinux.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/trash.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/upcall.so
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/leases.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt/glusterd.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server.so
+%dir %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage
+     %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage/posix.so
+
+# snap_scheduler
+%{_sbindir}/snap_scheduler.py
+%{_sbindir}/gcron.py
+%{_sbindir}/conf.py
+
+# /var/lib/glusterd, e.g. hookscripts, etc.
+%ghost      %attr(0644,-,-) %config(noreplace) %{_sharedstatedir}/glusterd/glusterd.info
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/bitd
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/groups
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/virt
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/metadata-cache
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/gluster-block
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/nl-cache
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/db-workload
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/distributed-virt
+            %attr(0644,-,-) %{_sharedstatedir}/glusterd/groups/samba
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glusterfind
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glusterfind/.keys
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glustershd
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/disabled-quota-root-xattr-heal.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/S10selinux-label-brick.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post/S13create-subdir-mounts.sh
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post/S10selinux-label-brick.sh
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/copy-file
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/copy-file/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/copy-file/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/post
+                            %{_sharedstatedir}/glusterd/hooks/1/delete/post/S57glusterfind-delete-post
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre/S10selinux-del-fcontext.sh
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/reset
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/reset/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/reset/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/post/S30samba-set.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/post/S32gluster_enable_shared_storage.sh
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/post
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/post/S29CTDBsetup.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/post/S30samba-start.sh
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/pre
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/post
+       %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre/S30samba-stop.sh
+            %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre/S29CTDB-teardown.sh
+%config(noreplace) %ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/options
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/peers
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/quotad
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/scrub
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/snaps
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/ss_brick
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/vols
+
+# Extra utility script
+%dir %{_libexecdir}/glusterfs
+%dir %{_datadir}/glusterfs/scripts
+     %{_datadir}/glusterfs/scripts/stop-all-gluster-processes.sh
+%if ( 0%{?_with_systemd:1} )
+     %{_libexecdir}/glusterfs/mount-shared-storage.sh
+     %{_datadir}/glusterfs/scripts/control-cpu-load.sh
+     %{_datadir}/glusterfs/scripts/control-mem.sh
+%endif
+
+# Incrementalapi
+     %{_libexecdir}/glusterfs/glusterfind
+%{_bindir}/glusterfind
+     %{_libexecdir}/glusterfs/peer_add_secret_pub
+
+%if ( 0%{?_with_firewalld:1} )
+%{_prefix}/lib/firewalld/services/glusterfs.xml
+%endif
+# end of server files
+%endif
+
+# Events
+%if ( 0%{!?_without_events:1} )
+%files events
+%config(noreplace) %{_sysconfdir}/glusterfs/eventsconfig.json
+%dir %{_sharedstatedir}/glusterd
+%dir %{_sharedstatedir}/glusterd/events
+%dir %{_libexecdir}/glusterfs
+     %{_libexecdir}/glusterfs/gfevents
+     %{_libexecdir}/glusterfs/peer_eventsapi.py*
+%{_sbindir}/glustereventsd
+%{_sbindir}/gluster-eventsapi
+%{_datadir}/glusterfs/scripts/eventsdash.py*
+%if ( 0%{?_with_systemd:1} )
+%{_unitdir}/glustereventsd.service
+%else
+%{_sysconfdir}/init.d/glustereventsd
+%endif
+%endif
 
 %changelog
-* Wed Jul 01 2009 Harshavardhana <harsha@gluster.com> - 2.1
-- Removed mod_glusterfs.so and added new --without epoll build
-  option. 
-
-* Thu Apr 16 2009 Harshavardhana <harsha@gluster.com> - 2.0
-- Galore of updates including new packages added common,
-  client,server splitting the original package. rpmbuild 
-  fully restructured to adhere to Fedora rpm standards. 
-  Older changelog removed as there were warnings when 
-  tried with 'rpmlint'. 
+* Thu May 14 2020 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- refactor, common practice, Issue #1126
+
+* Mon May 11 2020 Sunny Kumar <sunkumar@redhat.com>
+- added requires policycoreutils-python-utils on rhel8 for geo-replication
+
+* Wed Oct 9 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- remove leftover bd xlator cruft
+
+* Fri Aug 23 2019 Shwetha K Acharya <sacharya@redhat.com>
+- removed {name}-ufs from Obsoletes
+- added  "< version" for obsoletes {name}-gnfs and {name}-rdma
+
+* Mon Jul 15 2019 Jiffin Tony Thottan <jthottan@redhat.com>
+- Adding ganesha ha bits back in gluster repository
+
+* Fri Jul 12 2019 Amar Tumballi <amarts@redhat.com>
+- Remove rdma package, and mark older rdma package as 'Obsoletes'
+
+* Fri Jun 14 2019 Niels de Vos <ndevos@redhat.com>
+- always build glusterfs-cli to allow monitoring/managing from clients
+
+* Wed Mar 6 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- remove unneeded ldconfig in scriptlets
+-  reported by Igor Gnatenko in Fedora
+-   https://src.fedoraproject.org/rpms/glusterfs/pull-request/5
+
+* Mon Mar 4 2019 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- s390x has RDMA, since around Fedora 27 and in RHEL7 since June 2016.
+
+* Tue Feb 26 2019 Ashish Pandey <aspandey@redhat.com>
+- Add thin-arbiter package
+
+* Sun Feb 24 2019 Aravinda VK <avishwan@redhat.com>
+- Renamed events package to gfevents
+
+* Thu Feb 21 2019 Jiffin Tony Thottan <jthottan@redhat.com>
+- Obsoleting gluster-gnfs package
+
+* Wed Nov 28 2018 Krutika Dhananjay <kdhananj@redhat.com>
+- Install /var/lib/glusterd/groups/distributed-virt by default
+
+* Tue Nov 13 2018 Niels de Vos <ndevos@redhat.com>
+- Add an option to build with ThreadSanitizer (TSAN)
+
+* Fri Sep 7 2018 Niels de Vos <ndevos@redhat.com>
+- Add an option to build with address sanitizer (ASAN)
+
+* Sun Jul 29 2018 Niels de Vos <ndevos@redhat.com>
+- Disable building glusterfs-resource-agents on el6 (#1609551)
+
+* Thu Feb 22 2018 Kotresh HR <khiremat@redhat.com>
+- Added util-linux as dependency to georeplication rpm (#1544382)
+
+* Thu Feb 1 2018 Niels de Vos <ndevos@redhat.com>
+- Add '--without server' option to facilitate el6 builds (#1074947)
+
+* Wed Jan 24 2018 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python-ctypes no long exists, now in python stdlib (#1538258)
+
+* Thu Jan 18 2018 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Fedora 28 glibc has removed rpc headers and rpcgen, use libtirpc
+
+* Mon Dec 25 2017 Niels de Vos <ndevos@redhat.com>
+- Fedora 28 has renamed pyxattr
+
+* Wed Sep 27 2017 Mohit Agrawal <moagrawa@redhat.com>
+- Added control-cpu-load.sh and control-mem.sh scripts to glusterfs-server section(#1496335)
+
+* Tue Aug 22 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- libibverbs-devel, librdmacm-devel -> rdma-core-devel #1483995
+
+* Thu Jul 20 2017 Aravinda VK <avishwan@redhat.com>
+- Added new tool/binary to set the gfid2path xattr on files
+
+* Thu Jul 13 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- various directories not owned by any package
+
+* Fri Jun 16 2017 Jiffin Tony Thottan <jthottan@redhat.com>
+- Add glusterfssharedstorage.service systemd file
+
+* Fri Jun 9 2017 Poornima G <pgurusid@redhat.com>
+- Install /var/lib/glusterd/groups/nl-cache by default
+
+* Wed May 10 2017 Pranith Kumar K <pkarampu@redhat.com>
+- Install /var/lib/glusterd/groups/gluster-block by default
+
+* Thu Apr 27 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- gnfs in an optional subpackage
+
+* Wed Apr 26 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- /var/run/gluster owner gluster:gluster(0775) for qemu(gfapi)
+  statedumps (#1445569)
+
+* Mon Apr 24 2017 Jiffin Tony Thottan <jhottan@redhat.com>
+- Install SELinux hook scripts that manage contexts for bricks (#1047975)
+
+* Thu Apr 20 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- firewalld-filesystem -> firewalld (#1443959)
+
+* Thu Apr 13 2017 Niels de Vos <ndevos@redhat.com>
+- the -regression-tests sub-package needs "bc" for some tests (#1442145)
+
+* Mon Mar 20 2017 Niels de Vos <ndevos@redhat.com>
+- Drop dependency on psmisc, pkill is used instead of killall (#1197308)
+
+* Thu Feb 16 2017 Niels de Vos <ndevos@redhat.com>
+- Obsolete and Provide python-gluster for upgrading from glusterfs < 3.10
+
+* Wed Feb 1 2017 Poornima G <pgurusid@redhat.com>
+- Install /var/lib/glusterd/groups/metadata-cache by default
+
+* Wed Jan 18 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python2 (versus python3) cleanup (#1414902)
+
+* Fri Jan 13 2017 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- switch to storhaug HA
+
+* Fri Jan 6 2017 Niels de Vos <ndevos@redhat.com>
+- use macro provided by firewalld-filesystem to reload firewalld
+
+* Thu Nov 24 2016 Jiffin Tony Thottan <jhottan@redhat.com>
+- remove S31ganesha-reset.sh from hooks (#1397795)
+
+* Thu Sep 22 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python-ctypes no long exists, now in python stdlib (#1378436)
+
+* Wed Sep 14 2016 Aravinda VK <avishwan@redhat.com>
+- Changed attribute of eventsconfig.json file as same as other configs (#1375532)
+
+* Thu Sep 08 2016 Aravinda VK <avishwan@redhat.com>
+- Added init script for glustereventsd (#1365395)
+
+* Wed Aug 31 2016 Avra Sengupta <asengupt@redhat.com>
+- Added conf.py for snap scheduler
+
+* Wed Aug 31 2016 Aravinda VK <avishwan@redhat.com>
+- Added new Geo-replication utility "gluster-georep-sshkey" (#1356508)
+
+* Thu Aug 25 2016 Aravinda VK <avishwan@redhat.com>
+- Added gluster-mountbroker utility for geo-rep mountbroker setup (#1343333)
+
+* Mon Aug 22 2016 Milind Changire <mchangir@redhat.com>
+- Add psmisc as dependency for glusterfs-fuse for killall command (#1367665)
+
+* Thu Aug 4 2016 Jiffin Tony Thottan <jthottan@redhat.com>
+- Remove ganesha.so from client xlators
+
+* Sun Jul 31 2016 Soumya Koduri <skoduri@redhat.com>
+- Add dependency on portblock resource agent for ganesha package (#1354439)
+
+* Mon Jul 18 2016 Aravinda VK <avishwan@redhat.com>
+- Added new subpackage events(glusterfs-events) (#1334044)
+
+* Fri Jul 15 2016 Aravinda VK <avishwan@redhat.com>
+- Removed ".py" extension from symlink(S57glusterfind-delete-post)(#1356868)
+
+* Thu Jul 14 2016 Aravinda VK <avishwan@redhat.com>
+- Removed extra packaging line of cliutils(python-gluster)(#1342356)
+
+* Mon Jul 11 2016 Aravinda VK <avishwan@redhat.com>
+- Added Python subpackage "cliutils" under gluster
+
+* Tue May 31 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- broken brp-python-bytecompile in RHEL7 results in installed
+  but unpackaged files.
+
+* Fri May 6 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- additional dirs and files in /var/lib/glusterd/... (#1326410)
+
+* Tue Apr 26 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- %%postun libs w/o firewalld on RHEL6 (#1330583)
+
+* Tue Apr 12 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- additional dirs and files in /var/lib/glusterd/... (#1326410)
+
+* Mon Mar 7 2016 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- %%pre, %%post etc. scriptlet cleanup, ... -p /sbin/ldconfig (#1315024)
+
+* Fri Jan 22 2016 Aravinda VK <avishwan@redhat.com>
+- Added schedule_georep.py script to the glusterfs-geo-replication (#1300956)
+
+* Sat Jan 16 2016 Niels de Vos <ndevos@redhat.com>
+- glusterfs-server depends on -api (#1296931)
+
+* Sun Jan 10 2016 Niels de Vos <ndevos@redhat.com>
+- build system got fixed so that special glupy build is not needed anymore
+
+* Mon Dec 28 2015 Niels de Vos <ndevos@redhat.com>
+- hook scripts in glusterfs-ganesha use dbus-send, add dependency (#1294446)
+
+* Tue Dec 22 2015 Niels de Vos <ndevos@redhat.com>
+- move hook scripts for nfs-ganesha to the -ganesha sub-package
+- use standard 'make' installation for the hook scripts (#1174765)
+
+* Tue Sep 1 2015 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- erroneous ghost of ../hooks/1/delete causes install failure (#1258975)
+
+* Tue Aug 25 2015 Anand Nekkunti <anekkunt@redhat.com>
+- adding glusterfs-firewalld service (#1253967)
+
+* Tue Aug 18 2015 Niels de Vos <ndevos@redhat.com>
+- Include missing directories for glusterfind hooks scripts (#1225465)
+
+* Mon Jun 15 2015 Niels de Vos <ndevos@redhat.com>
+- Replace hook script S31ganesha-set.sh by S31ganesha-start.sh (#1231738)
+
+* Fri Jun 12 2015 Aravinda VK <avishwan@redhat.com>
+- Added rsync as dependency to georeplication rpm (#1231205)
+
+* Tue Jun 02 2015 Aravinda VK <avishwan@redhat.com>
+- Added post hook for volume delete as part of glusterfind (#1225465)
+
+* Wed May 27 2015 Aravinda VK <avishwan@redhat.com>
+- Added stop-all-gluster-processes.sh in glusterfs-server section (#1204641)
+
+* Wed May 20 2015 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- python-gluster should be 'noarch' (#1219954)
+
+* Wed May 20 2015 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- move libgf{db,changelog}.pc from -api-devel to -devel (#1223385)
+
+* Wed May 20 2015 Anand Nekkunti <anekkunt@redhat.com>
+- glusterd.socket file cleanup during post run upgrade (#1210404)
+
+* Tue May 19 2015 Avra Sengupta <asengupt@redhat.com>
+- Added S32gluster_enable_shared_storage.sh as volume set hook script (#1222013)
+
+* Mon May 18 2015 Milind Changire <mchangir@redhat.com>
+- Move file peer_add_secret_pub to the server RPM to support glusterfind (#1221544)
+* Sun May 17 2015 Niels de Vos <ndevos@redhat.com>
+- Fix building on RHEL-5 based distributions (#1222317)
+
+* Tue May 05 2015 Niels de Vos <ndevos@redhat.com>
+- Introduce glusterfs-client-xlators to reduce dependencies (#1195947)
+
+* Wed Apr 15 2015 Humble Chirammal <hchiramm@redhat.com>
+- Introducing python-gluster package to own gluster namespace in sitelib (#1211848)
+
+* Sat Mar 28 2015 Mohammed Rafi KC <rkavunga@redhat.com>
+- Add dependency for librdmacm version >= 1.0.15 (#1206744)
+
+* Tue Mar 24 2015 Niels de Vos <ndevos@redhat.com>
+- move libgfdb (with sqlite dependency) to -server subpackage (#1194753)
+
+* Tue Mar 17 2015 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- glusterfs-ganesha sub-package
+
+* Thu Mar 12 2015 Kotresh H R <khiremat@redhat.com>
+- gfind_missing_files tool is included (#1187140)
+
+* Tue Mar 03 2015 Aravinda VK <avishwan@redhat.com>
+- Included glusterfind files as part of server package.
+
+* Sun Mar 1 2015 Avra Sengupta <asengupt@redhat.com>
+- Added installation of snap-scheduler
+
+* Thu Feb 26 2015 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- enable cmocka unittest support only when asked for (#1067059)
+
+* Tue Feb 24 2015 Niels de Vos <ndevos@redhat.com>
+- POSIX ACL conversion needs BuildRequires libacl-devel (#1185654)
+
+* Wed Feb 18 2015 Andreas Schneider <asn@redhat.com>
+- Change cmockery2 to cmocka.
+
+* Wed Feb 18 2015 Kaushal M <kaushal@redhat.com>
+- add userspace-rcu as a requirement
+
+* Fri Feb 13 2015 Gaurav Kumar Garg <ggarg@redhat.com>
+- .cmd_log_history file should be renamed to cmd_history.log post
+  upgrade (#1165996)
+
+* Fri Jan 30 2015 Nandaja Varma <nvarma@redhat.com>
+- remove checks for rpmbuild/mock from run-tests.sh (#178008)
+
+* Fri Jan 16 2015 Niels de Vos <ndevos@redhat.com>
+- add support for /run/gluster through a tmpfiles.d config file (#1182934)
+
+* Tue Jan 6 2015 Aravinda VK<avishwan@redhat.com>
+- Added new libexec script for mountbroker user management (peer_mountbroker)
+
+* Fri Dec 12 2014 Niels de Vos <ndevos@redhat.com>
+- do not package all /usr/share/glusterfs/* files in regression-tests (#1169005)
+
+* Fri Sep 26 2014 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- smarter logic in %%post server (#1146426)
+
+* Wed Sep 24 2014 Balamurugan Arumugam <barumuga@redhat.com>
+- remove /sbin/ldconfig as interpreter (#1145989)
+
+* Fri Sep 5 2014 Lalatendu Mohanty <lmohanty@redhat.com>
+- Changed the description as "GlusterFS a distributed filesystem"
+
+* Tue Aug 5 2014 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- logrotate files (#1126832)
+
+* Wed Jul 16 2014 Luis Pabon <lpabon@redhat.com>
+- Added cmockery2 dependency
+
+* Fri Jun 27 2014 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- killall --wait in %%post server, (#1113543)
+
+* Thu Jun 19 2014 Humble Chirammal <hchiramm@redhat.com>
+- Added dynamic loading of fuse module with glusterfs-fuse package installation in el5.
+
+* Thu Jun 12 2014 Varun Shastry <vshastry@redhat.com>
+- Add bash completion config to the cli package
+
+* Tue Jun 03 2014 Vikhyat Umrao <vumrao@redhat.com>
+- add nfs-utils package dependency for server package (#1065654)
+
+* Thu May 22 2014 Poornima G <pgurusid@redhat.com>
+- Rename old hookscripts in an RPM-standard way.
+
+* Tue May 20 2014 Niels de Vos <ndevos@redhat.com>
+- Almost drop calling ./autogen.sh
+
+* Fri Apr 25 2014 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora spec (#1091408, #1091392)
+
+* Fri Apr 25 2014 Arumugam Balamurugan <barumuga@redhat.com>
+- fix RHEL 7 build failure "Installed (but unpackaged) file(s) found" (#1058188)
+
+* Wed Apr 02 2014 Arumugam Balamurugan <barumuga@redhat.com>
+- cleanup to rearrange spec file elements
+
+* Wed Apr 02 2014 Arumugam Balamurugan <barumuga@redhat.com>
+- add version/release dynamically (#1074919)
+
+* Thu Mar 27 2014 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- attr dependency (#1184626)
+
+* Wed Mar 26 2014 Poornima G <pgurusid@redhat.com>
+- Include the hook scripts of add-brick, volume start, stop and set
+
+* Wed Feb 26 2014 Niels de Vos <ndevos@redhat.com>
+- Drop glusterfs-devel dependency from glusterfs-api (#1065750)
+
+* Wed Feb 19 2014 Justin Clift <justin@gluster.org>
+- Rename gluster.py to glupy.py to avoid namespace conflict (#1018619)
+- Move the main Glupy files into glusterfs-extra-xlators rpm
+- Move the Glupy Translator examples into glusterfs-devel rpm
+
+* Thu Feb 06 2014 Aravinda VK <avishwan@redhat.com>
+- Include geo-replication upgrade scripts and hook scripts.
+
+* Wed Jan 15 2014 Niels de Vos <ndevos@redhat.com>
+- Install /var/lib/glusterd/groups/virt by default
+
+* Sat Jan 4 2014 Niels de Vos <ndevos@redhat.com>
+- The main glusterfs package should not provide glusterfs-libs (#1048489)
+
+* Tue Dec 10 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec 3.5.0-0.1.qa3
+
+* Fri Oct 11 2013 Harshavardhana <fharshav@redhat.com>
+- Add '_sharedstatedir' macro to `/var/lib` on <= RHEL5 (#1003184)
+
+* Wed Oct 9 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec 3.4.1-2+
+
+* Wed Oct 9 2013 Niels de Vos <ndevos@redhat.com>
+- glusterfs-api-devel requires glusterfs-devel (#1016938, #1017094)
+
+* Mon Sep 30 2013 Niels de Vos <ndevos@redhat.com>
+- Package gfapi.py into the Python site-packages path (#1005146)
+
+* Tue Sep 17 2013 Harshavardhana <fharshav@redhat.com>
+- Provide a new package called "glusterfs-regression-tests" for standalone
+  regression testing.
+
+* Thu Aug 22 2013 Niels de Vos <ndevos@redhat.com>
+- Correct the day/date for some entries in this changelog (#1000019)
+
+* Wed Aug 7 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec
+-  add Requires
+-  add -cli subpackage,
+-  fix other minor differences with Fedora glusterfs.spec
+
+* Tue Jul 30 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec, add glusterfs-libs RPM for oVirt/qemu-kvm
+
+* Thu Jul 25 2013 Csaba Henk <csaba@redhat.com>
+- Added peer_add_secret_pub and peer_gsec_create to %%{_libexecdir}/glusterfs
+
+* Thu Jul 25 2013 Aravinda VK <avishwan@redhat.com>
+- Added gverify.sh to %%{_libexecdir}/glusterfs directory.
+
+* Thu Jul 25 2013 Harshavardhana <fharshav@redhat.com>
+- Allow to build with '--without bd' to disable 'bd' xlator
+
+* Thu Jun 27 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- fix the hardening fix for shlibs, use %%sed macro, shorter ChangeLog
+
+* Wed Jun 26 2013 Niels de Vos <ndevos@redhat.com>
+- move the mount/api xlator to glusterfs-api
+
+* Fri Jun 7 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec, remove G4S/UFO and Swift
+
+* Mon Mar 4 2013 Niels de Vos <ndevos@redhat.com>
+- Package /var/run/gluster so that statedumps can be created
+
+* Wed Feb 6 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec
+
+* Tue Dec 11 2012 Filip Pytloun <filip.pytloun@gooddata.com>
+- add sysconfig file
+
+* Thu Oct 25 2012 Niels de Vos <ndevos@redhat.com>
+- Add a sub-package for the OCF resource agents
+
+* Wed Sep 05 2012 Niels de Vos <ndevos@redhat.com>
+- Don't use python-ctypes on SLES (from Jörg Petersen)
+
+* Tue Jul 10 2012 Niels de Vos <ndevos@redhat.com>
+- Include extras/clear_xattrs.sh in the glusterfs-server sub-package
+
+* Thu Jun 07 2012 Niels de Vos <ndevos@redhat.com>
+- Mark /var/lib/glusterd as owned by glusterfs, subdirs belong to -server
+
+* Wed May 9 2012 Kaleb S. KEITHLEY <kkeithle[at]redhat.com>
+- Add BuildRequires: libxml2-devel so that configure will DTRT on for
+- Fedora's Koji build system
+
+* Wed Nov 9 2011 Joe Julian <me@joejulian.name> - git master
+- Merge fedora specfile into gluster's spec.in.
+- Add conditionals to allow the same spec file to be used for both 3.1 and 3.2
+- http://bugs.gluster.com/show_bug.cgi?id=2970
+
+* Wed Oct  5 2011 Joe Julian <me@joejulian.name> - 3.2.4-1
+- Update to 3.2.4
+- Removed the $local_fs requirement from the init scripts as in RHEL/CentOS that's provided
+- by netfs, which needs to be started after glusterd.
+
+* Sun Sep 25 2011 Joe Julian <me@joejulian.name> - 3.2.3-2
+- Merged in upstream changes
+- Fixed version reporting 3.2git
+- Added nfs init script (disabled by default)
+
+* Thu Sep  1 2011 Joe Julian <me@joejulian.name> - 3.2.3-1
+- Update to 3.2.3
+
+* Tue Jul 19 2011 Joe Julian <me@joejulian.name> - 3.2.2-3
+- Add readline and libtermcap dependencies
+
+* Tue Jul 19 2011 Joe Julian <me@joejulian.name> - 3.2.2-2
+- Critical patch to prevent glusterd from walking outside of its own volume during rebalance
+
+* Thu Jul 14 2011 Joe Julian <me@joejulian.name> - 3.2.2-1
+- Update to 3.2.2
+
+* Wed Jul 13 2011 Joe Julian <me@joejulian.name> - 3.2.1-2
+- fix hardcoded path to gsyncd in source to match the actual file location
+
+* Tue Jun 21 2011 Joe Julian <me@joejulian.name> - 3.2.1
+- Update to 3.2.1
+
+* Mon Jun 20 2011 Joe Julian <me@joejulian.name> - 3.1.5
+- Update to 3.1.5
+
+* Tue May 31 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.4
+- Current git
+
+* Sun May 29 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.2
+- set _sharedstatedir to /var/lib for FHS compliance in RHEL5/CentOS5
+- mv /etc/glusterd, if it exists, to the new state dir for upgrading from gluster packaging
+
+* Sat May 28 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.1
+- Update to 3.1.5-qa1
+- Add patch to remove optimization disabling
+- Add patch to remove forced 64 bit compile
+- Obsolete glusterfs-core to allow for upgrading from gluster packaging
+
+* Sat Mar 19 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.3-1
+- Update to 3.1.3
+- Merge in more upstream SPEC changes
+- Remove patches from GlusterFS bugzilla #2309 and #2311
+- Remove inode-gen.patch
+
+* Sun Feb 06 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-3
+- Add back in legacy SPEC elements to support older branches
+
+* Thu Feb 03 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-2
+- Add patches from CloudFS project
+
+* Tue Jan 25 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-1
+- Update to 3.1.2
+
+* Wed Jan 5 2011 Dan Horák <dan[at]danny.cz> - 3.1.1-3
+- no InfiniBand on s390(x)
+
+* Sat Jan 1 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.1-2
+- Update to support readline
+- Update to not parallel build
+
+* Mon Dec 27 2010 Silas Sewell <silas@sewell.ch> - 3.1.1-1
+- Update to 3.1.1
+- Change package names to mirror upstream
+
+* Mon Dec 20 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.0.7-1
+- Update to 3.0.7
+
+* Wed Jul 28 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.0.5-1
+- Update to 3.0.x
+
+* Sat Apr 10 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.9-2
+- Move python version requires into a proper BuildRequires otherwise
+  the spec always turned off python bindings as python is not part
+  of buildsys-build and the chroot will never have python unless we
+  require it
+- Temporarily set -D_FORTIFY_SOURCE=1 until upstream fixes code
+  GlusterFS Bugzilla #197 (#555728)
+- Move glusterfs-volgen to devel subpackage (#555724)
+- Update description (#554947)
+
+* Sat Jan 2 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.9-1
+- Update to 2.0.9
+
+* Sun Nov 8 2009 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.8-1
+- Update to 2.0.8
+- Remove install of glusterfs-volgen, it's properly added to
+  automake upstream now
+
+* Sat Oct 31 2009 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.7-1
+- Update to 2.0.7
+- Install glusterfs-volgen, until it's properly added to automake
+  by upstream
+- Add macro to be able to ship more docs
+
+* Thu Sep 17 2009 Peter Lemenkov <lemenkov@gmail.com> 2.0.6-2
+- Rebuilt with new fuse
+
+* Sat Sep 12 2009 Matthias Saou <http://freshrpms.net/> 2.0.6-1
+- Update to 2.0.6.
+- No longer default to disable the client on RHEL5 (#522192).
+- Update spec file URLs.
+
+* Mon Jul 27 2009 Matthias Saou <http://freshrpms.net/> 2.0.4-1
+- Update to 2.0.4.
+
+* Thu Jun 11 2009 Matthias Saou <http://freshrpms.net/> 2.0.1-2
+- Remove libglusterfs/src/y.tab.c to fix koji F11/devel builds.
+
+* Sat May 16 2009 Matthias Saou <http://freshrpms.net/> 2.0.1-1
+- Update to 2.0.1.
+
+* Thu May  7 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-1
+- Update to 2.0.0 final.
+
+* Wed Apr 29 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.3.rc8
+- Move glusterfsd to common, since the client has a symlink to it.
+
+* Fri Apr 24 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.2.rc8
+- Update to 2.0.0rc8.
+
+* Sun Apr 12 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.2.rc7
+- Update glusterfsd init script to the new style init.
+- Update files to match the new default vol file names.
+- Include logrotate for glusterfsd, use a pid file by default.
+- Include logrotate for glusterfs, using killall for lack of anything better.
+
+* Sat Apr 11 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.1.rc7
+- Update to 2.0.0rc7.
+- Rename "libs" to "common" and move the binary, man page and log dir there.
+
+* Tue Feb 24 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org>
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild
+
+* Mon Feb 16 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.1.rc1
+- Update to 2.0.0rc1.
+- Include new libglusterfsclient.h.
+
+* Mon Feb 16 2009 Matthias Saou <http://freshrpms.net/> 1.3.12-1
+- Update to 1.3.12.
+- Remove no longer needed ocreat patch.
+
+* Thu Jul 17 2008 Matthias Saou <http://freshrpms.net/> 1.3.10-1
+- Update to 1.3.10.
+- Remove mount patch, it's been included upstream now.
+
+* Fri May 16 2008 Matthias Saou <http://freshrpms.net/> 1.3.9-1
+- Update to 1.3.9.
+
+* Fri May  9 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-1
+- Update to 1.3.8 final.
+
+* Wed Apr 23 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.10
+- Include short patch to include fixes from latest TLA 751.
+
+* Tue Apr 22 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.9
+- Update to 1.3.8pre6.
+- Include glusterfs binary in both the client and server packages, now that
+  glusterfsd is a symlink to it instead of a separate binary.
+* Sun Feb  3 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.8
+- Add python version check and disable bindings for version < 2.4.
+
+* Sun Feb  3 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.7
+- Add --without client rpmbuild option, make it the default for RHEL (no fuse).
+  (I hope "rhel" is the proper default macro name, couldn't find it...)
+
+* Wed Jan 30 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.6
+- Add --without ibverbs rpmbuild option to the package.
+
+* Mon Jan 14 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.5
+- Update to current TLA again, patch-636 which fixes the known segfaults.
+
+* Thu Jan 10 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.4
+- Downgrade to glusterfs--mainline--2.5--patch-628 which is more stable.
+
+* Tue Jan  8 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.3
+- Update to current TLA snapshot.
+- Include umount.glusterfs wrapper script (really needed? dunno).
+- Include patch to mount wrapper to avoid multiple identical mounts.
+
+* Sun Dec 30 2007 Matthias Saou <http://freshrpms.net/> 1.3.8-0.1
+- Update to current TLA snapshot, which includes "volume-name=" fstab option.
+
+* Mon Dec  3 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-6
+- Re-add the /var/log/glusterfs directory in the client sub-package (required).
+- Include custom patch to support vol= in fstab for -n glusterfs client option.
+
+* Mon Nov 26 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-4
+- Re-enable libibverbs.
+- Check and update License field to GPLv3+.
+- Add glusterfs-common obsoletes, to provide upgrade path from old packages.
+- Include patch to add mode to O_CREATE opens.
+
+* Thu Nov 22 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-3
+- Remove Makefile* files from examples.
+- Include RHEL/Fedora type init script, since the included ones don't do.
+
+* Wed Nov 21 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-1
+- Major spec file cleanup.
+- Add missing %%clean section.
+- Fix ldconfig calls (weren't set for the proper sub-package).
+
+* Sat Aug 4 2007 Matt Paine <matt@mattsoftware.com> - 1.3.pre7
+- Added support to build rpm without ibverbs support (use --without ibverbs
+  switch)
+
+* Sun Jul 15 2007 Matt Paine <matt@mattsoftware.com> - 1.3.pre6
+- Initial spec file
diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am
index 8cfe4b55461..a0a778158d8 100644
--- a/glusterfsd/src/Makefile.am
+++ b/glusterfsd/src/Makefile.am
@@ -1,25 +1,38 @@
 sbin_PROGRAMS = glusterfsd
+if WITH_SERVER
+sbin_PROGRAMS += glusterfsd gf_attach
+endif
 
 glusterfsd_SOURCES = glusterfsd.c glusterfsd-mgmt.c
-if GF_DARWIN_HOST_OS
-glusterfsd_SOURCES += $(CONTRIBDIR)/apple/daemon.c
-endif
 glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
-glusterfsd_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
-noinst_HEADERS = glusterfsd.h glusterfsd-common.h glusterfsd-mem-types.h
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la $(GF_LDADD) $(LIB_DL)
+glusterfsd_LDFLAGS = $(GF_LDFLAGS)
+
+gf_attach_SOURCES = gf_attach.c
+gf_attach_LDADD   = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+		    $(top_builddir)/api/src/libgfapi.la \
+		    $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		    $(top_builddir)/rpc/xdr/src/libgfxdr.la
+gf_attach_LDFLAGS = $(GF_LDFLAGS)
 
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
+noinst_HEADERS = glusterfsd.h glusterfsd-mem-types.h glusterfsd-messages.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) \
-	-I$(top_srcdir)/rpc/rpc-lib/src -I$(top_srcdir)/rpc/xdr/src
-if GF_DARWIN_HOST_OS
-AM_CFLAGS += -I$(CONTRIBDIR)/apple
-endif
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-DLIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/xlators/nfs/server/src \
+	-I$(top_srcdir)/xlators/protocol/server/src \
+	-I$(top_srcdir)/api/src
 
-CLEANFILES = 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
@@ -30,8 +43,12 @@ uninstall-local:
 
 install-data-local:
 	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/run
+	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/run/gluster
 	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/log/glusterfs
+	$(INSTALL) -d -m 755 $(DESTDIR)$(sbindir)
 	rm -f $(DESTDIR)$(sbindir)/glusterfs
-	rm -f $(DESTDIR)$(sbindir)/glusterd
 	ln -s glusterfsd $(DESTDIR)$(sbindir)/glusterfs
+if WITH_SERVER
+	rm -f $(DESTDIR)$(sbindir)/glusterd
 	ln -s glusterfsd $(DESTDIR)$(sbindir)/glusterd
+endif
diff --git a/glusterfsd/src/gf_attach.c b/glusterfsd/src/gf_attach.c
new file mode 100644
index 00000000000..c553b0b1f61
--- /dev/null
+++ b/glusterfsd/src/gf_attach.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <glusterfs/glusterfs.h>
+#include "glfs-internal.h"
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+
+/* In seconds */
+#define CONNECT_TIMEOUT 60
+#define REPLY_TIMEOUT 120
+
+int done = 0;
+int rpc_status;
+
+pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+
+struct rpc_clnt_procedure gf_attach_actors[GLUSTERD_BRICK_MAXVALUE] = {
+    [GLUSTERD_BRICK_NULL] = {"NULL", NULL},
+    [GLUSTERD_BRICK_OP] = {"BRICK_OP", NULL},
+};
+
+struct rpc_clnt_program gf_attach_prog = {
+    .progname = "brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .proctable = gf_attach_actors,
+    .numproc = GLUSTERD_BRICK_MAXVALUE,
+};
+
+int32_t
+my_callback(struct rpc_req *req, struct iovec *iov, int count, void *frame)
+{
+    pthread_mutex_lock(&mutex);
+    rpc_status = req->rpc_status;
+    done = 1;
+    /* Signal main thread which is the only waiter */
+    pthread_cond_signal(&cond);
+    pthread_mutex_unlock(&mutex);
+    return 0;
+}
+
+/* copied from gd_syncop_submit_request */
+int
+send_brick_req(xlator_t *this, struct rpc_clnt *rpc, char *path, int op)
+{
+    int ret = -1;
+    struct timespec ts;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
+    gd1_mgmt_brick_op_req brick_req;
+    void *req = &brick_req;
+
+    brick_req.op = op;
+    brick_req.name = path;
+    brick_req.input.input_val = NULL;
+    brick_req.input.input_len = 0;
+    brick_req.dict.dict_val = NULL;
+    brick_req.dict.dict_len = 0;
+
+    req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf)
+        goto out;
+
+    iobref = iobref_new();
+    if (!iobref)
+        goto out;
+
+    iobref_add(iobref, iobuf);
+
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret == -1)
+        goto out;
+
+    iov.iov_len = ret;
+
+    /* Wait for connection */
+    timespec_now_realtime(&ts);
+    ts.tv_sec += CONNECT_TIMEOUT;
+    pthread_mutex_lock(&rpc->conn.lock);
+    {
+        while (!rpc->conn.connected)
+            if (pthread_cond_timedwait(&rpc->conn.cond, &rpc->conn.lock, &ts) ==
+                ETIMEDOUT) {
+                fprintf(stderr, "timeout waiting for RPC connection\n");
+                pthread_mutex_unlock(&rpc->conn.lock);
+                return EXIT_FAILURE;
+            }
+    }
+    pthread_mutex_unlock(&rpc->conn.lock);
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, &gf_attach_prog, op, my_callback, &iov, 1, NULL,
+                          0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    if (!ret) {
+        /* OK, wait for callback */
+        timespec_now_realtime(&ts);
+        ts.tv_sec += REPLY_TIMEOUT;
+        pthread_mutex_lock(&mutex);
+        {
+            while (!done)
+                if (pthread_cond_timedwait(&cond, &mutex, &ts) == ETIMEDOUT) {
+                    fprintf(stderr, "timeout waiting for RPC reply\n");
+                    pthread_mutex_unlock(&mutex);
+                    return EXIT_FAILURE;
+                }
+        }
+        pthread_mutex_unlock(&mutex);
+    }
+
+out:
+
+    iobref_unref(iobref);
+    iobuf_unref(iobuf);
+    if (frame)
+        STACK_DESTROY(frame->root);
+
+    if (rpc_status != 0) {
+        fprintf(stderr, "got error %d on RPC\n", rpc_status);
+        return EXIT_FAILURE;
+    }
+
+    printf("OK\n");
+    return EXIT_SUCCESS;
+}
+
+int
+usage(char *prog)
+{
+    fprintf(stderr, "Usage: %s uds_path volfile_path (to attach)\n", prog);
+    fprintf(stderr, "       %s -d uds_path brick_path (to detach)\n", prog);
+
+    return EXIT_FAILURE;
+}
+
+int
+main(int argc, char *argv[])
+{
+    glfs_t *fs;
+    struct rpc_clnt *rpc;
+    dict_t *options;
+    int ret;
+    int op = GLUSTERD_BRICK_ATTACH;
+
+    for (;;) {
+        switch (getopt(argc, argv, "d")) {
+            case 'd':
+                op = GLUSTERD_BRICK_TERMINATE;
+                break;
+            case -1:
+                goto done_parsing;
+            default:
+                return usage(argv[0]);
+        }
+    }
+done_parsing:
+    if (optind != (argc - 2)) {
+        return usage(argv[0]);
+    }
+
+    fs = glfs_new("gf-attach");
+    if (!fs) {
+        fprintf(stderr, "glfs_new failed\n");
+        return EXIT_FAILURE;
+    }
+
+    (void)glfs_set_logging(fs, "/dev/stderr", 7);
+    /*
+     * This will actually fail because we haven't defined a volume, but
+     * it will do enough initialization to get us going.
+     */
+    (void)glfs_init(fs);
+
+    options = dict_new();
+    if (!options) {
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret != 0) {
+        fprintf(stderr, "failed to set transport type\n");
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport.address-family", "unix");
+    if (ret != 0) {
+        fprintf(stderr, "failed to set address family\n");
+        return EXIT_FAILURE;
+    }
+    ret = dict_set_str(options, "transport.socket.connect-path", argv[optind]);
+    if (ret != 0) {
+        fprintf(stderr, "failed to set connect path\n");
+        return EXIT_FAILURE;
+    }
+
+    rpc = rpc_clnt_new(options, fs->ctx->master, "gf-attach-rpc", 0);
+    if (!rpc) {
+        fprintf(stderr, "rpc_clnt_new failed\n");
+        return EXIT_FAILURE;
+    }
+
+    if (rpc_clnt_register_notify(rpc, NULL, NULL) != 0) {
+        fprintf(stderr, "rpc_clnt_register_notify failed\n");
+        return EXIT_FAILURE;
+    }
+
+    if (rpc_clnt_start(rpc) != 0) {
+        fprintf(stderr, "rpc_clnt_start failed\n");
+        return EXIT_FAILURE;
+    }
+
+    return send_brick_req(fs->ctx->master, rpc, argv[optind + 1], op);
+}
diff --git a/glusterfsd/src/glusterfsd-common.h b/glusterfsd/src/glusterfsd-common.h
deleted file mode 100644
index e6a6bc5cdf3..00000000000
--- a/glusterfsd/src/glusterfsd-common.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __GLUSTERFSD_COMMON_H__
-#define __GLUSTERFSD_COMMON_H__
-
-#define ZR_MOUNTPOINT_OPT       "mountpoint"
-#define ZR_ATTR_TIMEOUT_OPT     "attribute-timeout"
-#define ZR_ENTRY_TIMEOUT_OPT    "entry-timeout"
-#define ZR_DIRECT_IO_OPT        "direct-io-mode"
-#define ZR_STRICT_VOLFILE_CHECK "strict-volfile-check"
-#define ZR_DUMP_FUSE            "dump-fuse"
-
-#endif
diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 0625f725ef7..e59b558deb0 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -1,36 +1,27 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
-
 #ifndef __GLUSTERFSD_MEM_TYPES_H__
 #define __GLUSTERFSD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum gfd_mem_types_ {
-        gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
-        gfd_mt_xlator_t,
-        gfd_mt_xlator_cmdline_option_t,
-        gfd_mt_char,
-        gfd_mt_call_pool_t,
-        gfd_mt_end
+    gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
+    gfd_mt_xlator_t,
+    gfd_mt_server_cmdline_t,
+    gfd_mt_xlator_cmdline_option_t,
+    gfd_mt_char,
+    gfd_mt_call_pool_t,
+    gfd_mt_end
 
 };
 #endif
diff --git a/glusterfsd/src/glusterfsd-messages.h b/glusterfsd/src/glusterfsd-messages.h
new file mode 100644
index 00000000000..0cdbffa71ea
--- /dev/null
+++ b/glusterfsd/src/glusterfsd-messages.h
@@ -0,0 +1,93 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERFSD_MESSAGES_H_
+#define _GLUSTERFSD_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    GLUSTERFSD, glusterfsd_msg_1, glusterfsd_msg_2, glusterfsd_msg_3,
+    glusterfsd_msg_4, glusterfsd_msg_5, glusterfsd_msg_6, glusterfsd_msg_7,
+    glusterfsd_msg_8, glusterfsd_msg_9, glusterfsd_msg_10, glusterfsd_msg_11,
+    glusterfsd_msg_12, glusterfsd_msg_13, glusterfsd_msg_14, glusterfsd_msg_15,
+    glusterfsd_msg_16, glusterfsd_msg_17, glusterfsd_msg_18, glusterfsd_msg_19,
+    glusterfsd_msg_20, glusterfsd_msg_21, glusterfsd_msg_22, glusterfsd_msg_23,
+    glusterfsd_msg_24, glusterfsd_msg_25, glusterfsd_msg_26, glusterfsd_msg_27,
+    glusterfsd_msg_28, glusterfsd_msg_29, glusterfsd_msg_30, glusterfsd_msg_31,
+    glusterfsd_msg_32, glusterfsd_msg_33, glusterfsd_msg_34, glusterfsd_msg_35,
+    glusterfsd_msg_36, glusterfsd_msg_37, glusterfsd_msg_38, glusterfsd_msg_39,
+    glusterfsd_msg_40, glusterfsd_msg_41, glusterfsd_msg_42, glusterfsd_msg_43,
+    glusterfsd_msg_029, glusterfsd_msg_041, glusterfsd_msg_042);
+
+#define glusterfsd_msg_1_STR "Could not create absolute mountpoint path"
+#define glusterfsd_msg_2_STR "Could not get current working directory"
+#define glusterfsd_msg_4_STR "failed to set mount-point to options dictionary"
+#define glusterfsd_msg_3_STR "failed to set dict value for key"
+#define glusterfsd_msg_5_STR "failed to set disable for key"
+#define glusterfsd_msg_6_STR "failed to set enable for key"
+#define glusterfsd_msg_7_STR                                                   \
+    "Not a client process, not performing mount operation"
+#define glusterfsd_msg_8_STR "MOUNT_POINT initialization failed"
+#define glusterfsd_msg_9_STR "loading volume file failed"
+#define glusterfsd_msg_10_STR "xlator option is invalid"
+#define glusterfsd_msg_11_STR "Fetching the volume file from server..."
+#define glusterfsd_msg_12_STR "volume initialization failed"
+#define glusterfsd_msg_34_STR "memory init failed"
+#define glusterfsd_msg_13_STR "ERROR: glusterfs uuid generation failed"
+#define glusterfsd_msg_14_STR "ERROR: glusterfs pool creation failed"
+#define glusterfsd_msg_15_STR                                                  \
+    "ERROR: '--volfile-id' is mandatory if '-s' OR '--volfile-server' option " \
+    "is given"
+#define glusterfsd_msg_16_STR "ERROR: parsing the volfile failed"
+#define glusterfsd_msg_33_STR                                                  \
+    "obsolete option '--volfile-max-fecth-attempts or fetch-attempts' was "    \
+    "provided"
+#define glusterfsd_msg_17_STR "pidfile open failed"
+#define glusterfsd_msg_18_STR "pidfile lock failed"
+#define glusterfsd_msg_20_STR "pidfile truncation failed"
+#define glusterfsd_msg_21_STR "pidfile write failed"
+#define glusterfsd_msg_22_STR "failed to exeute pthread_sigmask"
+#define glusterfsd_msg_23_STR "failed to create pthread"
+#define glusterfsd_msg_24_STR "daemonization failed"
+#define glusterfsd_msg_25_STR "mount failed"
+#define glusterfsd_msg_26_STR "failed to construct the graph"
+#define glusterfsd_msg_27_STR "fuse xlator cannot be specified in volume file"
+#define glusterfsd_msg_28_STR "Cannot reach volume specification file"
+#define glusterfsd_msg_29_STR "ERROR: glusterfsd context not initialized"
+#define glusterfsd_msg_43_STR                                                  \
+    "command line argument --brick-mux is valid only for brick process"
+#define glusterfsd_msg_029_STR "failed to create command line string"
+#define glusterfsd_msg_30_STR "Started running version"
+#define glusterfsd_msg_31_STR "Could not create new sync-environment"
+#define glusterfsd_msg_40_STR "No change in volfile, countinuing"
+#define glusterfsd_msg_39_STR "Unable to create/delete temporary file"
+#define glusterfsd_msg_38_STR                                                  \
+    "Not processing brick-op since volume graph is not yet active"
+#define glusterfsd_msg_35_STR "rpc req buffer unserialization failed"
+#define glusterfsd_msg_36_STR "problem in xlator loading"
+#define glusterfsd_msg_37_STR "failed to get dict value"
+#define glusterfsd_msg_41_STR "received attach request for volfile"
+#define glusterfsd_msg_42_STR "failed to unserialize xdata to dictionary"
+#define glusterfsd_msg_041_STR "can't detach. flie not found"
+#define glusterfsd_msg_042_STR                                                 \
+    "couldnot detach old graph. Aborting the reconfiguration operation"
+
+#endif /* !_GLUSTERFSD_MESSAGES_H_ */
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index be8b77f4f09..eaf6796e4c3 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1,664 +1,3055 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <stdlib.h>
+#include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "glusterfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/defaults.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
+#include "glusterfsd-messages.h"
 #include "glusterfs3.h"
-#include "portmap.h"
+#include "portmap-xdr.h"
+#include "xdr-generic.h"
 
 #include "glusterfsd.h"
+#include "rpcsvc.h"
+#include "cli1-xdr.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/monitoring.h>
+#include "server.h"
+
+static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false;
+int need_emancipate = 0;
 
-static char is_mgmt_rpc_reconnect;
-
-typedef ssize_t (*mgmt_serialize_t) (struct iovec outmsg, void *args);
-
+int
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx);
+int
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx);
+int
+glusterfs_process_volfp(glusterfs_ctx_t *ctx, FILE *fp);
+int
+emancipate(glusterfs_ctx_t *ctx, int ret);
+int
+glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp,
+                                   char *volfile_id, char *checksum,
+                                   dict_t *dict);
+int
+glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx,
+                                  gf_volfile_t *volfile_obj, char *checksum,
+                                  dict_t *dict);
+int
+glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp,
+                                   char *volfile_id, char *checksum,
+                                   dict_t *dict);
+int
+glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj);
 
+gf_boolean_t
+mgmt_is_multiplexed_daemon(char *name);
 
-int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
-int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
-int glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp);
+static int
+glusterfs_volume_top_perf(const char *brick_path, dict_t *dict,
+                          gf_boolean_t write_test);
 
 int
-mgmt_cbk_spec (void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_get ();
-        gf_log ("mgmt", GF_LOG_INFO, "Volume file changed");
+    ctx = glusterfsd_ctx;
+    gf_log("mgmt", GF_LOG_INFO, "Volume file changed");
 
-        glusterfs_volfile_fetch (ctx);
-        return 0;
+    glusterfs_volfile_fetch(ctx);
+    return 0;
 }
 
-rpcclnt_cb_actor_t gluster_cbk_actors[] = {
-        [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-};
+int
+mgmt_process_volfile(const char *volfile, ssize_t size, char *volfile_id,
+                     dict_t *dict)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    FILE *tmpfp = NULL;
+    gf_volfile_t *volfile_obj = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+    char sha256_hash[SHA256_DIGEST_LENGTH] = {
+        0,
+    };
+    int tmp_fd = -1;
+    char template[] = "/tmp/glfs.volfile.XXXXXX";
+
+    glusterfs_compute_sha256((const unsigned char *)volfile, size, sha256_hash);
+    ctx = THIS->ctx;
+    LOCK(&ctx->volfile_lock);
+    {
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(volfile_id, volfile_obj->vol_id)) {
+                if (!memcmp(sha256_hash, volfile_obj->volfile_checksum,
+                            sizeof(volfile_obj->volfile_checksum))) {
+                    UNLOCK(&ctx->volfile_lock);
+                    gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_40,
+                            NULL);
+                    goto out;
+                }
+                volfile_tmp = volfile_obj;
+                break;
+            }
+        }
 
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode */
+        tmp_fd = mkstemp(template);
+        if (-1 == tmp_fd) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_39,
+                    "create template=%s", template, NULL);
+            ret = -1;
+            goto out;
+        }
 
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-        .progname  = "GlusterFS Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
-        .actors    = gluster_cbk_actors,
-        .numactors = GF_CBK_MAXVALUE,
-};
+        /* Calling unlink so that when the file is closed or program
+         * terminates the temporary file is deleted.
+         */
+        ret = sys_unlink(template);
+        if (ret < 0) {
+            gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_39,
+                    "delete template=%s", template, NULL);
+            ret = 0;
+        }
 
-char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = {
-        [GF_PMAP_NULL]        = "NULL",
-        [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
-        [GF_PMAP_BRICKBYPORT] = "BRICKBYPORT",
-        [GF_PMAP_SIGNIN]      = "SIGNIN",
-        [GF_PMAP_SIGNOUT]     = "SIGNOUT",
-        [GF_PMAP_SIGNUP]      = "SIGNUP",
-};
+        tmpfp = fdopen(tmp_fd, "w+b");
+        if (!tmpfp) {
+            ret = -1;
+            goto unlock;
+        }
 
+        fwrite(volfile, size, 1, tmpfp);
+        fflush(tmpfp);
+        if (ferror(tmpfp)) {
+            ret = -1;
+            goto unlock;
+        }
 
-rpc_clnt_prog_t clnt_pmap_prog = {
-        .progname  = "Gluster Portmap",
-        .prognum   = GLUSTER_PMAP_PROGRAM,
-        .progver   = GLUSTER_PMAP_VERSION,
-        .procnames = clnt_pmap_procs,
-};
+        if (!volfile_tmp) {
+            /* There is no checksum in the list, which means simple attach
+             * the volfile
+             */
+            ret = glusterfs_process_svc_attach_volfp(ctx, tmpfp, volfile_id,
+                                                     sha256_hash, dict);
+            goto unlock;
+        }
+        ret = glusterfs_mux_volfile_reconfigure(tmpfp, ctx, volfile_obj,
+                                                sha256_hash, dict);
+        if (ret < 0) {
+            gf_msg_debug("glusterfsd-mgmt", EINVAL, "Reconfigure failed !!");
+        }
+    }
+unlock:
+    UNLOCK(&ctx->volfile_lock);
+out:
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
+    return ret;
+}
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = "NULL",
-        [GF_HNDSK_SETVOLUME]    = "SETVOLUME",
-        [GF_HNDSK_GETSPEC]      = "GETSPEC",
-        [GF_HNDSK_PING]         = "PING",
-};
+int
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    return 0;
+}
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .procnames = clnt_handshake_procs,
-};
+struct iobuf *
+glusterfs_serialize_reply(rpcsvc_request_t *req, void *arg,
+                          struct iovec *outmsg, xdrproc_t xdrproc)
+{
+    struct iobuf *iob = NULL;
+    ssize_t retlen = -1;
+    ssize_t xdr_size = 0;
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    xdr_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, xdr_size);
+    if (!iob) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    /* retlen is used to received the error since size_t is unsigned and we
+     * need -1 for error notification during encoding.
+     */
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to encode message");
+        GF_FREE(iob);
+        goto ret;
+    }
+
+    outmsg->iov_len = retlen;
+ret:
+    if (retlen == -1) {
+        iob = NULL;
+    }
+
+    return iob;
+}
 
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-                     glusterfs_ctx_t *ctx,
-                     rpc_clnt_prog_t *prog, int procnum,
-                     mgmt_serialize_t sfunc, fop_cbk_fn_t cbkfn)
+glusterfs_submit_reply(rpcsvc_request_t *req, void *arg, struct iovec *payload,
+                       int payloadcount, struct iobref *iobref,
+                       xdrproc_t xdrproc)
 {
-        int                     ret         = -1;
-        int                     count      = 0;
-        struct iovec            iov         = {0, };
-        struct iobuf            *iobuf = NULL;
-        struct iobref           *iobref = NULL;
-
-        iobref = iobref_new ();
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iovec rsp = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req) {
+        GF_ASSERT(req);
+        goto out;
+    }
+
+    if (!iobref) {
+        iobref = iobref_new();
         if (!iobref) {
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR, "out of memory");
+            goto out;
         }
 
-        iobuf = iobuf_get (ctx->iobuf_pool);
-        if (!iobuf) {
-                goto out;
-        };
+        new_iobref = 1;
+    }
 
-        iobref_add (iobref, iobuf);
+    iob = glusterfs_serialize_reply(req, arg, &rsp, xdrproc);
+    if (!iob) {
+        gf_log_callingfn(THIS->name, GF_LOG_ERROR, "Failed to serialize reply");
+    } else {
+        iobref_add(iobref, iob);
+    }
 
-        iov.iov_base = iobuf->ptr;
-        iov.iov_len  = 128 * GF_UNIT_KB;
+    ret = rpcsvc_submit_generic(req, &rsp, 1, payload, payloadcount, iobref);
 
+    /* Now that we've done our job of handing the message to the RPC layer
+     * we can safely unref the iob in the hope that RPC layer must have
+     * ref'ed the iob on receiving into the txlist.
+     */
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Reply submission failed");
+        goto out;
+    }
 
-        /* Create the xdr payload */
-        if (req && sfunc) {
-                ret = sfunc (iov, req);
-                if (ret == -1) {
-                        goto out;
+    ret = 0;
+out:
+    if (iob)
+        iobuf_unref(iob);
+
+    if (new_iobref && iobref)
+        iobref_unref(iobref);
+
+    return ret;
+}
+
+int
+glusterfs_terminate_response_send(rpcsvc_request_t *req, int op_ret)
+{
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = 0;
+    rsp.op_errstr = "";
+    dict = dict_new();
+
+    if (dict)
+        ret = dict_allocate_and_serialize(dict, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+
+    if (ret == 0)
+        ret = glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+
+    GF_FREE(rsp.output.output_val);
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+int
+glusterfs_handle_terminate(rpcsvc_request_t *req)
+{
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    ssize_t ret;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *top = NULL;
+    xlator_t *victim = NULL;
+    xlator_t *tvictim = NULL;
+    xlator_list_t **trav_p = NULL;
+    gf_boolean_t lockflag = _gf_false;
+    gf_boolean_t still_bricks_attached = _gf_false;
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ctx = glusterfsd_ctx;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        /* Find the xlator_list_t that points to our victim. */
+        if (glusterfsd_ctx->active) {
+            top = glusterfsd_ctx->active->first;
+            for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+                victim = (*trav_p)->xlator;
+                if (!victim->cleanup_starting &&
+                    strcmp(victim->name, xlator_req.name) == 0) {
+                    break;
                 }
-                iov.iov_len = ret;
-                count = 1;
+            }
         }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+        if (!top)
+            goto err;
+    }
+    if (!*trav_p) {
+        gf_log(THIS->name, GF_LOG_ERROR, "can't terminate %s - not found",
+               xlator_req.name);
+        /*
+         * Used to be -ENOENT.  However, the caller asked us to
+         * make sure it's down and if it's already down that's
+         * good enough.
+         */
+        glusterfs_terminate_response_send(req, 0);
+        goto err;
+    }
+
+    glusterfs_terminate_response_send(req, 0);
+    for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+        tvictim = (*trav_p)->xlator;
+        if (!tvictim->cleanup_starting &&
+            !strcmp(tvictim->name, xlator_req.name)) {
+            continue;
+        }
+        if (!tvictim->cleanup_starting) {
+            still_bricks_attached = _gf_true;
+            break;
+        }
+    }
+    if (!still_bricks_attached) {
+        gf_log(THIS->name, GF_LOG_INFO,
+               "terminating after loss of last child %s", xlator_req.name);
+        rpc_clnt_mgmt_pmap_signout(glusterfsd_ctx, xlator_req.name);
+        kill(getpid(), SIGTERM);
+    } else {
+        /* TODO cleanup sequence needs to be done properly for
+           Quota and Changelog
+        */
+        if (victim->cleanup_starting)
+            goto err;
+
+        rpc_clnt_mgmt_pmap_signout(glusterfsd_ctx, xlator_req.name);
+        victim->cleanup_starting = 1;
+
+        UNLOCK(&ctx->volfile_lock);
+        lockflag = _gf_true;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "detaching not-only"
+               " child %s",
+               xlator_req.name);
+        top->notify(top, GF_EVENT_CLEANUP, victim);
+    }
+err:
+    if (!lockflag)
+        UNLOCK(&ctx->volfile_lock);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    free(xlator_req.name);
+    xlator_req.name = NULL;
+    return 0;
+}
 
-out:
-        if (iobref)
-                iobref_unref (iobref);
+int
+glusterfs_translator_info_response_send(rpcsvc_request_t *req, int ret,
+                                        char *msg, dict_t *output)
+{
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    gf_boolean_t free_ptr = _gf_false;
+    GF_ASSERT(req);
+
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg && msg[0])
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
+
+    ret = -1;
+    if (output) {
+        ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+    }
+    if (!ret)
+        free_ptr = _gf_true;
+
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
+    if (free_ptr)
+        GF_FREE(rsp.output.output_val);
+    return ret;
+}
 
-        return ret;
+int
+glusterfs_xlator_op_response_send(rpcsvc_request_t *req, int op_ret, char *msg,
+                                  dict_t *output)
+{
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    gf_boolean_t free_ptr = _gf_false;
+    GF_ASSERT(req);
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = 0;
+    if (op_ret && msg && msg[0])
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
+
+    if (output) {
+        ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                          &rsp.output.output_len);
+    }
+    if (!ret)
+        free_ptr = _gf_true;
+
+    ret = glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+
+    if (free_ptr)
+        GF_FREE(rsp.output.output_val);
+
+    return ret;
 }
 
+int
+glusterfs_handle_translator_info_get(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    gf1_cli_top_op top_op = 0;
+    xlator_t *any = NULL;
+    xlator_t *xlator = NULL;
+    glusterfs_graph_t *active = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char msg[2048] = {
+        0,
+    };
+    dict_t *output = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "top-op", (int32_t *)&top_op);
+    if (ret)
+        goto cont;
+    if (GF_CLI_TOP_READ_PERF == top_op) {
+        ret = glusterfs_volume_top_perf(xlator_req.name, dict, _gf_false);
+    } else if (GF_CLI_TOP_WRITE_PERF == top_op) {
+        ret = glusterfs_volume_top_perf(xlator_req.name, dict, _gf_true);
+    }
+
+cont:
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    xlator = get_xlator_by_name(any, xlator_req.name);
+    if (!xlator) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
+
+    if (strcmp(xlator->type, "debug/io-stats")) {
+        xlator = get_xlator_by_type(xlator, "debug/io-stats");
+        if (!xlator) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "xlator-type debug/io-stats is not loaded");
+            goto out;
+        }
+    }
 
-/* XXX: move these into @ctx */
-static char oldvolfile[131072];
-static int oldvollen = 0;
+    output = dict_new();
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
 
-static int
-xlator_equal_rec (xlator_t *xl1, xlator_t *xl2)
-{
-	xlator_list_t *trav1 = NULL;
-        xlator_list_t *trav2 = NULL;
-        int            ret   = 0;
-
-	if (xl1 == NULL || xl2 == NULL)	{
-		gf_log ("xlator", GF_LOG_DEBUG, "invalid argument");
-		return -1;
-	}
-
-	trav1 = xl1->children;
-        trav2 = xl2->children;
-
-	while (trav1 && trav2) {
-		ret = xlator_equal_rec (trav1->xlator, trav2->xlator);
-                if (ret) {
-                        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                                "xlators children not equal");
-                        goto out;
-                }
+out:
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
+
+    free(xlator_req.name);
+    free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
 
-		trav1 = trav1->next;
-                trav2 = trav2->next;
-	}
+static int
+glusterfs_volume_top_perf(const char *brick_path, dict_t *dict,
+                          gf_boolean_t write_test)
+{
+    int32_t fd = -1;
+    int32_t output_fd = -1;
+    char export_path[PATH_MAX] = {
+        0,
+    };
+    char *buf = NULL;
+    int32_t iter = 0;
+    int32_t ret = -1;
+    uint64_t total_blks = 0;
+    uint32_t blk_size;
+    uint32_t blk_count;
+    double throughput = 0;
+    double time = 0;
+    struct timeval begin, end = {
+                              0,
+                          };
+
+    GF_ASSERT(brick_path);
+
+    ret = dict_get_uint32(dict, "blk-size", &blk_size);
+    if (ret)
+        goto out;
+    ret = dict_get_uint32(dict, "blk-cnt", &blk_count);
+    if (ret)
+        goto out;
+
+    if (!(blk_size > 0) || !(blk_count > 0))
+        goto out;
+
+    buf = GF_CALLOC(1, blk_size * sizeof(*buf), gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not allocate memory");
+        goto out;
+    }
+
+    snprintf(export_path, sizeof(export_path), "%s/%s", brick_path,
+             ".gf-tmp-stats-perf");
+    fd = open(export_path, O_CREAT | O_RDWR, S_IRWXU);
+    if (-1 == fd) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+        goto out;
+    }
+
+    gettimeofday(&begin, NULL);
+    for (iter = 0; iter < blk_count; iter++) {
+        ret = sys_write(fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        total_blks += ret;
+    }
+    gettimeofday(&end, NULL);
+    if (total_blks != ((uint64_t)blk_size * blk_count)) {
+        gf_log("glusterd", GF_LOG_WARNING, "Error in write");
+        ret = -1;
+        goto out;
+    }
+
+    time = gf_tvdiff(&begin, &end);
+    throughput = total_blks / time;
+    gf_log("glusterd", GF_LOG_INFO,
+           "Throughput %.2f Mbps time %.2f secs "
+           "bytes written %" PRId64,
+           throughput, time, total_blks);
+
+    /* if it's a write test, we are done. Otherwise, we continue to the read
+     * part */
+    if (write_test == _gf_true) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = sys_fsync(fd);
+    if (ret) {
+        gf_log("glusterd", GF_LOG_ERROR, "could not flush cache");
+        goto out;
+    }
+    ret = sys_lseek(fd, 0L, 0);
+    if (ret != 0) {
+        gf_log("glusterd", GF_LOG_ERROR, "could not seek back to start");
+        ret = -1;
+        goto out;
+    }
+
+    output_fd = open("/dev/null", O_RDWR);
+    if (-1 == output_fd) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_ERROR, "Could not open output file");
+        goto out;
+    }
+
+    total_blks = 0;
+
+    gettimeofday(&begin, NULL);
+    for (iter = 0; iter < blk_count; iter++) {
+        ret = sys_read(fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        ret = sys_write(output_fd, buf, blk_size);
+        if (ret != blk_size) {
+            ret = -1;
+            goto out;
+        }
+        total_blks += ret;
+    }
+    gettimeofday(&end, NULL);
+    if (total_blks != ((uint64_t)blk_size * blk_count)) {
+        ret = -1;
+        gf_log("glusterd", GF_LOG_WARNING, "Error in read");
+        goto out;
+    }
+
+    time = gf_tvdiff(&begin, &end);
+    throughput = total_blks / time;
+    gf_log("glusterd", GF_LOG_INFO,
+           "Throughput %.2f Mbps time %.2f secs "
+           "bytes read %" PRId64,
+           throughput, time, total_blks);
+    ret = 0;
+out:
+    if (fd >= 0)
+        sys_close(fd);
+    if (output_fd >= 0)
+        sys_close(output_fd);
+    GF_FREE(buf);
+    sys_unlink(export_path);
+    if (ret == 0) {
+        ret = dict_set_double(dict, "time", time);
+        if (ret)
+            goto end;
+        ret = dict_set_double(dict, "throughput", throughput);
+        if (ret)
+            goto end;
+    }
+end:
+    return ret;
+}
 
-	if (trav1 || trav2) {
-		ret = -1;
-    		goto out;
-	}
+int
+glusterfs_handle_translator_op(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    int32_t op_ret = 0;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *input = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char key[32] = {0};
+    int len;
+    char *xname = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+    int i = 0;
+    int count = 0;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    active = ctx->active;
+    if (!active) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, glusterfsd_msg_38,
+                "brick-op_no.=%d", xlator_req.op, NULL);
+        goto out;
+    }
+    any = active->first;
+    input = dict_new();
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &input);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    } else {
+        input->extra_stdfree = xlator_req.input.input_val;
+    }
+
+    ret = dict_get_int32(input, "count", &count);
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < count; i++) {
+        len = snprintf(key, sizeof(key), "xl-%d", i);
+        ret = dict_get_strn(input, key, len, &xname);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Couldn't get "
+                   "xlator %s ",
+                   key);
+            goto out;
+        }
+        xlator = xlator_search_by_name(any, xname);
+        if (!xlator) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "xlator %s is not "
+                   "loaded",
+                   xname);
+            goto out;
+        }
+    }
+    for (i = 0; i < count; i++) {
+        len = snprintf(key, sizeof(key), "xl-%d", i);
+        ret = dict_get_strn(input, key, len, &xname);
+        xlator = xlator_search_by_name(any, xname);
+        XLATOR_NOTIFY(ret, xlator, GF_EVENT_TRANSLATOR_OP, input, output);
+        /* If notify fails for an xlator we need to capture it but
+         * continue with the loop. */
+        if (ret)
+            op_ret = -1;
+    }
+    ret = op_ret;
+out:
+    glusterfs_xlator_op_response_send(req, ret, "", output);
+    if (input)
+        dict_unref(input);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return 0;
+}
 
-	if (strcmp (xl1->name, xl2->name)) {
-                ret = -1;
-		goto out;
-	}
-out :
-        return ret;
+int
+glusterfs_handle_bitrot(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    xlator_t *any = NULL;
+    xlator_t *this = NULL;
+    xlator_t *xlator = NULL;
+    char msg[2048] = {
+        0,
+    };
+    char xname[1024] = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    char *scrub_opt = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+
+    input = dict_new();
+    if (!input)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &input);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_35, NULL);
+        goto out;
+    }
+
+    /* Send scrubber request to bitrot xlator */
+    snprintf(xname, sizeof(xname), "%s-bit-rot-0", xlator_req.name);
+    xlator = xlator_search_by_name(any, xname);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_36, NULL);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(input, "scrub-value", &scrub_opt);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get scrub value");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, glusterfsd_msg_37, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (!strncmp(scrub_opt, "status", SLEN("status"))) {
+        ret = xlator->notify(xlator, GF_EVENT_SCRUB_STATUS, input, output);
+    } else if (!strncmp(scrub_opt, "ondemand", SLEN("ondemand"))) {
+        ret = xlator->notify(xlator, GF_EVENT_SCRUB_ONDEMAND, input, output);
+        if (ret == -2) {
+            snprintf(msg, sizeof(msg),
+                     "Scrubber is in "
+                     "Pause/Inactive/Running state");
+            ret = -1;
+            goto out;
+        }
+    }
+out:
+    glusterfs_translator_info_response_send(req, ret, msg, output);
+
+    if (input)
+        dict_unref(input);
+    free(xlator_req.input.input_val); /*malloced by xdr*/
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);
+
+    return 0;
 }
 
-static gf_boolean_t
-is_graph_topology_equal (glusterfs_graph_t *graph1,
-                                glusterfs_graph_t *graph2)
+int
+glusterfs_handle_attach(rpcsvc_request_t *req)
 {
-        xlator_t    *trav1    = NULL;
-        xlator_t    *trav2    = NULL;
-        gf_boolean_t ret      = _gf_true;
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    xlator_t *nextchild = NULL;
+    glusterfs_graph_t *newgraph = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *srv_xl = NULL;
+    server_conf_t *srv_conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = this->ctx;
+    if (!ctx->cmd_args.volfile_id) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "No volfile-id provided, erroring out");
+        return -1;
+    }
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ret = 0;
+
+    if (!this->ctx->active) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "got attach for %s but no active graph", xlator_req.name);
+        goto post_unlock;
+    }
+
+    gf_log(this->name, GF_LOG_INFO, "got attach for %s", xlator_req.name);
+
+    LOCK(&ctx->volfile_lock);
+    {
+        ret = glusterfs_graph_attach(this->ctx->active, xlator_req.name,
+                                     &newgraph);
+        if (!ret && (newgraph && newgraph->first)) {
+            nextchild = newgraph->first;
+            ret = xlator_notify(nextchild, GF_EVENT_PARENT_UP, nextchild);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED,
+                        "event=ParentUp", "name=%s", nextchild->name, NULL);
+                goto unlock;
+            }
+            /* we need a protocol/server xlator as
+             * nextchild
+             */
+            srv_xl = this->ctx->active->first;
+            srv_conf = (server_conf_t *)srv_xl->private;
+            rpcsvc_autoscale_threads(this->ctx, srv_conf->rpc, 1);
+        }
+        if (ret) {
+            ret = -1;
+        }
+        ret = glusterfs_translator_info_response_send(req, ret, NULL, NULL);
+        if (ret) {
+            /* Response sent back to glusterd, req is already destroyed. So
+             * resetting the ret to 0. Otherwise another response will be
+             * send from rpcsvc_check_and_reply_error. Which will lead to
+             * double resource leak.
+             */
+            ret = 0;
+        }
+    unlock:
+        UNLOCK(&ctx->volfile_lock);
+    }
+post_unlock:
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    free(xlator_req.input.input_val);
+    free(xlator_req.name);
+
+    return ret;
+}
 
-        trav1 = graph1->first;
-        trav2 = graph2->first;
+int
+glusterfs_handle_svc_attach(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_smsg(THIS->name, GF_LOG_INFO, 0, glusterfsd_msg_41, "volfile-id=%s",
+            xlator_req.name, NULL);
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(xlator_req.dict.dict_val, xlator_req.dict.dict_len,
+                           &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, glusterfsd_msg_42, NULL);
+        goto out;
+    }
+    dict->extra_stdfree = xlator_req.dict.dict_val;
+
+    ret = 0;
+
+    ret = mgmt_process_volfile(xlator_req.input.input_val,
+                               xlator_req.input.input_len, xlator_req.name,
+                               dict);
+out:
+    if (dict)
+        dict_unref(dict);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.name)
+        free(xlator_req.name);
+    glusterfs_translator_info_response_send(req, ret, NULL, NULL);
+    return 0;
+}
 
-        ret = xlator_equal_rec (trav1, trav2);
+int
+glusterfs_handle_svc_detach(rpcsvc_request_t *req)
+{
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    ssize_t ret;
+    gf_volfile_t *volfile_obj = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ctx = glusterfsd_ctx;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(xlator_req.name, volfile_obj->vol_id)) {
+                volfile_tmp = volfile_obj;
+                break;
+            }
+        }
 
+        if (!volfile_tmp) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, glusterfsd_msg_041, "name=%s",
+                    xlator_req.name, NULL);
+            /*
+             * Used to be -ENOENT.  However, the caller asked us to
+             * make sure it's down and if it's already down that's
+             * good enough.
+             */
+            ret = 0;
+            goto out;
+        }
+        /* coverity[ORDER_REVERSAL] */
+        ret = glusterfs_process_svc_detach(ctx, volfile_tmp);
         if (ret) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "graphs are not equal");
-                ret = _gf_false;
-                goto out;
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg("glusterfsd-mgmt", GF_LOG_ERROR, EINVAL, glusterfsd_msg_042,
+                    NULL);
+            goto out;
         }
+    }
+    UNLOCK(&ctx->volfile_lock);
+out:
+    glusterfs_terminate_response_send(req, ret);
+    free(xlator_req.name);
+    xlator_req.name = NULL;
 
-	ret = _gf_true;
-        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                "graphs are equal");
+    return 0;
+}
 
+int
+glusterfs_handle_dump_metrics(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char *filepath = NULL;
+    int fd = -1;
+    struct stat statbuf = {
+        0,
+    };
+    char *msg = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+    if (ret < 0) {
+        /*failed to decode msg;*/
+        req->rpc_err = GARBAGE_ARGS;
+        return -1;
+    }
+    ret = -1;
+    ctx = this->ctx;
+
+    /* Infra for monitoring */
+    filepath = gf_monitor_metrics(ctx);
+    if (!filepath)
+        goto out;
+
+    fd = sys_open(filepath, O_RDONLY, 0);
+    if (fd < 0)
+        goto out;
+
+    if (sys_fstat(fd, &statbuf) < 0)
+        goto out;
+
+    if (statbuf.st_size > GF_UNIT_MB) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                "reconsider logic (%" PRId64 ")", statbuf.st_size, NULL);
+    }
+    msg = GF_CALLOC(1, (statbuf.st_size + 1), gf_common_mt_char);
+    if (!msg)
+        goto out;
+
+    ret = sys_read(fd, msg, statbuf.st_size);
+    if (ret < 0)
+        goto out;
+
+    /* Send all the data in errstr, instead of dictionary for now */
+    glusterfs_translator_info_response_send(req, 0, msg, NULL);
+
+    ret = 0;
 out:
-        return ret;
+    if (fd >= 0)
+        sys_close(fd);
+
+    GF_FREE(msg);
+    GF_FREE(filepath);
+    if (xlator_req.input.input_val)
+        free(xlator_req.input.input_val);
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+
+    return ret;
 }
 
-/* Function has 3types of return value 0, -ve , 1
- *   return 0          =======> reconfiguration of options has succeded
- *   return 1          =======> the graph has to be reconstructed and all the xlators should be inited
- *   return -1(or -ve) =======> Some Internal Error occured during the operation
- */
-static int
-glusterfs_volfile_reconfigure (FILE *newvolfile_fp)
+int
+glusterfs_handle_defrag(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char msg[2048] = {0};
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+    xlator = xlator_search_by_name(any, xlator_req.name);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = xlator->notify(xlator, GF_EVENT_VOLUME_DEFRAG, dict, output);
+
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
+out:
+    if (dict)
+        dict_unref(dict);
+    free(xlator_req.input.input_val);  // malloced by xdr
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return ret;
+}
+int
+glusterfs_handle_brick_status(rpcsvc_request_t *req)
 {
-        glusterfs_graph_t *oldvolfile_graph = NULL;
-        glusterfs_graph_t *newvolfile_graph = NULL;
-        FILE              *oldvolfile_fp    = NULL;
-	glusterfs_ctx_t   *ctx              = NULL;
+    int ret = -1;
+    gd1_mgmt_brick_op_req brick_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+    xlator_t *server_xl = NULL;
+    xlator_t *brick_xl = NULL;
+    dict_t *dict = NULL;
+    dict_t *output = NULL;
+    uint32_t cmd = 0;
+    char *msg = NULL;
+    char *brickname = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &brick_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(brick_req.input.input_val, brick_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "req-buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Couldn't get status op");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick-name", &brickname);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Couldn't get brickname from"
+               " dict");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    if (ctx == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "ctx returned NULL");
+        ret = -1;
+        goto out;
+    }
+    if (ctx->active == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    active = ctx->active;
+    if (ctx->active->first == NULL) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "ctx->active->first "
+               "returned NULL");
+        ret = -1;
+        goto out;
+    }
+    server_xl = active->first;
+
+    brick_xl = get_xlator_by_name(server_xl, brickname);
+    if (!brick_xl) {
+        gf_log(this->name, GF_LOG_ERROR, "xlator is not loaded");
+        ret = -1;
+        goto out;
+    }
+
+    output = dict_new();
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            ret = 0;
+            gf_proc_dump_mem_info_to_dict(output);
+            gf_proc_dump_mempool_info_to_dict(ctx, output);
+            break;
+
+        case GF_CLI_STATUS_CLIENTS:
+        case GF_CLI_STATUS_CLIENT_LIST:
+            ret = server_xl->dumpops->priv_to_dict(server_xl, output,
+                                                   brickname);
+            break;
+
+        case GF_CLI_STATUS_INODE:
+            ret = server_xl->dumpops->inode_to_dict(brick_xl, output);
+            break;
+
+        case GF_CLI_STATUS_FD:
+            ret = server_xl->dumpops->fd_to_dict(brick_xl, output);
+            break;
+
+        case GF_CLI_STATUS_CALLPOOL:
+            ret = 0;
+            gf_proc_dump_pending_frames_to_dict(ctx->pool, output);
+            break;
 
-        int ret = -1;
+        default:
+            ret = -1;
+            msg = gf_strdup("Unknown status op");
+            break;
+    }
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg)
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
+
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
+
+out:
+    if (dict)
+        dict_unref(dict);
+    if (output)
+        dict_unref(output);
+    free(brick_req.input.input_val);
+    if (brick_req.dict.dict_val)
+        free(brick_req.dict.dict_val);
+    free(brick_req.name);
+    GF_FREE(msg);
+    GF_FREE(rsp.output.output_val);
+
+    return ret;
+}
+
+int
+glusterfs_handle_node_status(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    gd1_mgmt_brick_op_req node_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *any = NULL;
+    xlator_t *node = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *dict = NULL;
+    dict_t *output = NULL;
+    char *volname = NULL;
+    char *node_name = NULL;
+    char *subvol_name = NULL;
+    uint32_t cmd = 0;
+    char *msg = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &node_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(node_req.input.input_val, node_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "req buffer to dictionary");
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get status op");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    if ((cmd & GF_CLI_STATUS_SHD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "glustershd");
+#ifdef BUILD_GNFS
+    else if ((cmd & GF_CLI_STATUS_NFS) != 0)
+        ret = gf_asprintf(&node_name, "%s", "nfs-server");
+#endif
+    else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "quotad");
+    else if ((cmd & GF_CLI_STATUS_BITD) != 0)
+        ret = gf_asprintf(&node_name, "%s", "bitd");
+    else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
+        ret = gf_asprintf(&node_name, "%s", "scrubber");
+
+    else {
+        ret = -1;
+        goto out;
+    }
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to set node xlator name");
+        goto out;
+    }
+
+    node = xlator_search_by_name(any, node_name);
+    if (!node) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded", node_name);
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_SHD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s-replicate-0", volname);
+    else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_BITD) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else if ((cmd & GF_CLI_STATUS_SCRUB) != 0)
+        ret = gf_asprintf(&subvol_name, "%s", volname);
+    else {
+        ret = -1;
+        goto out;
+    }
+    if (ret == -1) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to set node xlator name");
+        goto out;
+    }
+
+    subvol = xlator_search_by_name(node, subvol_name);
+    if (!subvol) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               subvol_name);
+        goto out;
+    }
+
+    output = dict_new();
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            ret = 0;
+            gf_proc_dump_mem_info_to_dict(output);
+            gf_proc_dump_mempool_info_to_dict(ctx, output);
+            break;
+
+        case GF_CLI_STATUS_CLIENTS:
+            // clients not available for SHD
+            if ((cmd & GF_CLI_STATUS_SHD) != 0)
+                break;
 
-        oldvolfile_fp = tmpfile ();
-        if (!oldvolfile_fp)
+            ret = dict_set_str(output, "volname", volname);
+            if (ret) {
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Error setting volname to dict");
                 goto out;
+            }
+            ret = node->dumpops->priv_to_dict(node, output, NULL);
+            break;
+
+        case GF_CLI_STATUS_INODE:
+            ret = 0;
+            inode_table_dump_to_dict(subvol->itable, "conn0", output);
+            ret = dict_set_int32(output, "conncount", 1);
+            break;
+
+        case GF_CLI_STATUS_FD:
+            // cannot find fd-tables in nfs-server graph
+            // TODO: finish once found
+            break;
+
+        case GF_CLI_STATUS_CALLPOOL:
+            ret = 0;
+            gf_proc_dump_pending_frames_to_dict(ctx->pool, output);
+            break;
 
-	if (!oldvollen) {
-		ret = 1; // Has to call INIT for the whole graph
-		goto out;
-	}
-        fwrite (oldvolfile, oldvollen, 1, oldvolfile_fp);
-        fflush (oldvolfile_fp);
+        default:
+            ret = -1;
+            msg = gf_strdup("Unknown status op");
+            gf_log(THIS->name, GF_LOG_ERROR, "%s", msg);
+            break;
+    }
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    if (ret && msg)
+        rsp.op_errstr = msg;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
+
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
 
+out:
+    if (dict)
+        dict_unref(dict);
+    free(node_req.input.input_val);
+    if (node_req.dict.dict_val)
+        free(node_req.dict.dict_val);
+    GF_FREE(msg);
+    GF_FREE(rsp.output.output_val);
+    GF_FREE(node_name);
+    GF_FREE(subvol_name);
+
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        oldvolfile_graph = glusterfs_graph_construct (oldvolfile_fp);
-        if (!oldvolfile_graph) {
-                goto out;
-        }
+int
+glusterfs_handle_nfs_profile(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    gd1_mgmt_brick_op_req nfs_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *any = NULL;
+    xlator_t *nfs = NULL;
+    xlator_t *subvol = NULL;
+    char *volname = NULL;
+    dict_t *output = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &nfs_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    dict = dict_new();
+    ret = dict_unserialize(nfs_req.input.input_val, nfs_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to "
+               "unserialize req-buffer to dict");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    any = active->first;
+
+    // is this needed?
+    // are problems possible by searching for subvol directly from "any"?
+    nfs = xlator_search_by_name(any, "nfs-server");
+    if (!nfs) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "xlator nfs-server is "
+               "not loaded");
+        goto out;
+    }
+
+    subvol = xlator_search_by_name(nfs, volname);
+    if (!subvol) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "xlator %s is no loaded", volname);
+        goto out;
+    }
+
+    output = dict_new();
+    ret = subvol->notify(subvol, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
+    rsp.op_ret = ret;
+    rsp.op_errno = 0;
+    rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(output, &rsp.output.output_val,
+                                      &rsp.output.output_len);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to serialize output dict to rsp");
+        goto out;
+    }
+
+    glusterfs_submit_reply(req, &rsp, NULL, 0, NULL,
+                           (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    ret = 0;
 
-        newvolfile_graph = glusterfs_graph_construct (newvolfile_fp);
-        if (!newvolfile_graph) {
-                goto out;
-        }
+out:
+    free(nfs_req.input.input_val);
+    if (nfs_req.dict.dict_val)
+        free(nfs_req.dict.dict_val);
+    if (dict)
+        dict_unref(dict);
+    if (output)
+        dict_unref(output);
+    GF_FREE(rsp.output.output_val);
+
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-        if (!is_graph_topology_equal (oldvolfile_graph,
-                                      newvolfile_graph)) {
+int
+glusterfs_handle_volume_barrier_op(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_brick_op_req xlator_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *any = NULL;
+    dict_t *output = NULL;
+    char msg[2048] = {0};
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+
+    active = ctx->active;
+    if (!active) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    any = active->first;
+    ret = xdr_to_generic(req->msg[0], &xlator_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_unserialize(xlator_req.input.input_val,
+                           xlator_req.input.input_len, &dict);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to "
+               "unserialize req-buffer to dictionary");
+        goto out;
+    }
+    xlator = xlator_search_by_name(any, xlator_req.name);
+    if (!xlator) {
+        snprintf(msg, sizeof(msg), "xlator %s is not loaded", xlator_req.name);
+        goto out;
+    }
+
+    output = dict_new();
+    if (!output) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = xlator->notify(xlator, GF_EVENT_VOLUME_BARRIER_OP, dict, output);
+
+    ret = glusterfs_translator_info_response_send(req, ret, msg, output);
+out:
+    if (dict)
+        dict_unref(dict);
+    free(xlator_req.input.input_val);  // malloced by xdr
+    if (xlator_req.dict.dict_val)
+        free(xlator_req.dict.dict_val);
+    if (output)
+        dict_unref(output);
+    free(xlator_req.name);  // malloced by xdr
+
+    return ret;
+}
 
-                ret = 1;
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "Graph topology not equal(should call INIT)");
-                goto out;
+int
+glusterfs_handle_barrier(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    gd1_mgmt_brick_op_req brick_req = {
+        0,
+    };
+    gd1_mgmt_brick_op_rsp brick_rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *top = NULL;
+    xlator_t *xlator = NULL;
+    xlator_t *old_THIS = NULL;
+    dict_t *dict = NULL;
+    gf_boolean_t barrier = _gf_true;
+    xlator_list_t *trav;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &brick_req,
+                         (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ctx = glusterfsd_ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    if (active == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "ctx->active returned NULL");
+        ret = -1;
+        goto out;
+    }
+    top = active->first;
+
+    for (trav = top->children; trav; trav = trav->next) {
+        if (strcmp(trav->xlator->name, brick_req.name) == 0) {
+            break;
         }
+    }
+    if (!trav) {
+        ret = -1;
+        goto out;
+    }
+    top = trav->xlator;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(brick_req.input.input_val, brick_req.input.input_len,
+                           &dict);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to unserialize "
+               "request dictionary");
+        goto out;
+    }
+
+    brick_rsp.op_ret = 0;
+    brick_rsp.op_errstr = "";  // initing to prevent serilaztion failures
+    old_THIS = THIS;
+
+    /* Send barrier request to the barrier xlator */
+    xlator = get_xlator_by_type(top, "features/barrier");
+    if (!xlator) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               "features/barrier");
+        goto out;
+    }
+
+    THIS = xlator;
+    // TODO: Extend this to accept return of errnos
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_OP, dict);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "barrier notify failed");
+        brick_rsp.op_ret = ret;
+        brick_rsp.op_errstr = gf_strdup(
+            "Failed to reconfigure "
+            "barrier.");
+        /* This is to invoke changelog-barrier disable if barrier
+         * disable fails and don't invoke if barrier enable fails.
+         */
+        barrier = dict_get_str_boolean(dict, "barrier", _gf_true);
+        if (barrier)
+            goto submit_reply;
+    }
+
+    /* Reset THIS so that we have it correct in case of an error below
+     */
+    THIS = old_THIS;
+
+    /* Send barrier request to changelog as well */
+    xlator = get_xlator_by_type(top, "features/changelog");
+    if (!xlator) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+               "features/changelog");
+        goto out;
+    }
+
+    THIS = xlator;
+    ret = xlator->notify(xlator, GF_EVENT_TRANSLATOR_OP, dict);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "changelog notify failed");
+        brick_rsp.op_ret = ret;
+        brick_rsp.op_errstr = gf_strdup("changelog notify failed");
+        goto submit_reply;
+    }
+
+submit_reply:
+    THIS = old_THIS;
+
+    ret = glusterfs_submit_reply(req, &brick_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
-        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                "Only options have changed in the new "
-                "graph");
+out:
+    if (dict)
+        dict_unref(dict);
+    free(brick_req.input.input_val);
+    if (brick_req.dict.dict_val)
+        free(brick_req.dict.dict_val);
+    gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
 
-	ctx = glusterfs_ctx_get ();
+int
+glusterfs_handle_rpc_msg(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    /* for now, nothing */
+    return ret;
+}
 
-	if (!ctx) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-			"glusterfs_ctx_get() returned NULL");
-		goto out;
-	}
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_event, GF_CBK_STATEDUMP},
+};
 
-	oldvolfile_graph = ctx->active;
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
+};
 
-	if (!oldvolfile_graph) {
-		gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-			"glsuterfs_ctx->active is NULL");
-		goto out;
-	}
+static char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = {
+    [GF_PMAP_NULL] = "NULL",
+    [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
+    [GF_PMAP_BRICKBYPORT] = "BRICKBYPORT",
+    [GF_PMAP_SIGNIN] = "SIGNIN",
+    [GF_PMAP_SIGNOUT] = "SIGNOUT",
+    [GF_PMAP_SIGNUP] = "SIGNUP", /* DEPRECATED - DON'T USE! */
+};
 
-        /* */
-        ret = glusterfs_graph_reconfigure (oldvolfile_graph,
-                                           newvolfile_graph);
-        if (ret) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "Could not reconfigure new options in old graph");
-                goto out;
-        }
+static rpc_clnt_prog_t clnt_pmap_prog = {
+    .progname = "Gluster Portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+    .procnames = clnt_pmap_procs,
+};
 
-        ret = 0;
-out:
-        return ret;
-}
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+};
+
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
+};
+
+static rpcsvc_actor_t glusterfs_actors[GLUSTERD_BRICK_MAXVALUE] = {
+    [GLUSTERD_BRICK_NULL] = {"NULL", glusterfs_handle_rpc_msg, NULL,
+                             GLUSTERD_BRICK_NULL, DRC_NA, 0},
+    [GLUSTERD_BRICK_TERMINATE] = {"TERMINATE", glusterfs_handle_terminate, NULL,
+                                  GLUSTERD_BRICK_TERMINATE, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_INFO] = {"TRANSLATOR INFO",
+                                    glusterfs_handle_translator_info_get, NULL,
+                                    GLUSTERD_BRICK_XLATOR_INFO, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_OP] = {"TRANSLATOR OP",
+                                  glusterfs_handle_translator_op, NULL,
+                                  GLUSTERD_BRICK_XLATOR_OP, DRC_NA, 0},
+    [GLUSTERD_BRICK_STATUS] = {"STATUS", glusterfs_handle_brick_status, NULL,
+                               GLUSTERD_BRICK_STATUS, DRC_NA, 0},
+    [GLUSTERD_BRICK_XLATOR_DEFRAG] = {"TRANSLATOR DEFRAG",
+                                      glusterfs_handle_defrag, NULL,
+                                      GLUSTERD_BRICK_XLATOR_DEFRAG, DRC_NA, 0},
+    [GLUSTERD_NODE_PROFILE] = {"NFS PROFILE", glusterfs_handle_nfs_profile,
+                               NULL, GLUSTERD_NODE_PROFILE, DRC_NA, 0},
+    [GLUSTERD_NODE_STATUS] = {"NFS STATUS", glusterfs_handle_node_status, NULL,
+                              GLUSTERD_NODE_STATUS, DRC_NA, 0},
+    [GLUSTERD_VOLUME_BARRIER_OP] = {"VOLUME BARRIER OP",
+                                    glusterfs_handle_volume_barrier_op, NULL,
+                                    GLUSTERD_VOLUME_BARRIER_OP, DRC_NA, 0},
+    [GLUSTERD_BRICK_BARRIER] = {"BARRIER", glusterfs_handle_barrier, NULL,
+                                GLUSTERD_BRICK_BARRIER, DRC_NA, 0},
+    [GLUSTERD_NODE_BITROT] = {"BITROT", glusterfs_handle_bitrot, NULL,
+                              GLUSTERD_NODE_BITROT, DRC_NA, 0},
+    [GLUSTERD_BRICK_ATTACH] = {"ATTACH", glusterfs_handle_attach, NULL,
+                               GLUSTERD_BRICK_ATTACH, DRC_NA, 0},
+
+    [GLUSTERD_DUMP_METRICS] = {"DUMP METRICS", glusterfs_handle_dump_metrics,
+                               NULL, GLUSTERD_DUMP_METRICS, DRC_NA, 0},
+
+    [GLUSTERD_SVC_ATTACH] = {"ATTACH CLIENT", glusterfs_handle_svc_attach, NULL,
+                             GLUSTERD_SVC_ATTACH, DRC_NA, 0},
+
+    [GLUSTERD_SVC_DETACH] = {"DETACH CLIENT", glusterfs_handle_svc_detach, NULL,
+                             GLUSTERD_SVC_DETACH, DRC_NA, 0},
+
+};
+
+static struct rpcsvc_program glusterfs_mop_prog = {
+    .progname = "Gluster Brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .actors = glusterfs_actors,
+    .numactors = GLUSTERD_BRICK_MAXVALUE,
+    .synctask = _gf_true,
+};
 
 int
-mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                  void *myframe)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-        gf_getspec_rsp           rsp   = {0,};
-        call_frame_t            *frame = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        int                      ret   = 0;
-        ssize_t                  size = 0;
-        FILE                    *tmpfp = NULL;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-        frame = myframe;
-        ctx = frame->this->ctx;
+        iobref_add(iobref, iobuf);
 
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                goto out;
-        }
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
 
-        ret = xdr_to_getspec_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "error");
-                ret   = -1;
-                goto out;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to create XDR payload");
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the 'volume file' from server");
-                ret = -1;
-                goto out;
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+
+out:
+    if (iobref)
+        iobref_unref(iobref);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
+}
+
+int
+mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0, locked = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    char *volfile_id = NULL;
+    gf_volfile_t *volfile_obj = NULL;
+    gf_volfile_t *volfile_tmp = NULL;
+    char sha256_hash[SHA256_DIGEST_LENGTH] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/glfs.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the 'volume file' from server");
+        ret = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        /* Server list is set by glusterd at the time of getspec */
+        ret = dict_get_str(dict, GLUSTERD_BRICK_SERVERS, &servers_list);
+        if (ret)
+            goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    size = rsp.op_ret;
+    volfile_id = frame->local;
+    if (mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) {
+        ret = mgmt_process_volfile((const char *)rsp.spec, size, volfile_id,
+                                   dict);
+        goto post_graph_mgmt;
+    }
+
+    ret = 0;
+    glusterfs_compute_sha256((const unsigned char *)rsp.spec, size,
+                             sha256_hash);
+
+    LOCK(&ctx->volfile_lock);
+    {
+        locked = 1;
+
+        list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+        {
+            if (!strcmp(volfile_id, volfile_obj->vol_id)) {
+                if (!memcmp(sha256_hash, volfile_obj->volfile_checksum,
+                            sizeof(volfile_obj->volfile_checksum))) {
+                    UNLOCK(&ctx->volfile_lock);
+                    gf_log(frame->this->name, GF_LOG_INFO,
+                           "No change in volfile,"
+                           "continuing");
+                    goto post_unlock;
+                }
+                volfile_tmp = volfile_obj;
+                break;
+            }
         }
 
-        ret = 0;
-        size = rsp.op_ret;
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode */
+        tmp_fd = mkstemp(template);
+        if (-1 == tmp_fd) {
+            UNLOCK(&ctx->volfile_lock);
+            gf_smsg(frame->this->name, GF_LOG_ERROR, 0, glusterfsd_msg_39,
+                    "create template=%s", template, NULL);
+            ret = -1;
+            goto post_unlock;
+        }
 
-        if (size == oldvollen && (memcmp (oldvolfile, rsp.spec, size) == 0)) {
-                gf_log ("", GF_LOG_NORMAL, "No change in volfile, continuing");
-                goto out;
+        /* Calling unlink so that when the file is closed or program
+         * terminates the temporary file is deleted.
+         */
+        ret = sys_unlink(template);
+        if (ret < 0) {
+            gf_smsg(frame->this->name, GF_LOG_INFO, 0, glusterfsd_msg_39,
+                    "delete template=%s", template, NULL);
+            ret = 0;
         }
 
-        tmpfp = tmpfile ();
+        tmpfp = fdopen(tmp_fd, "w+b");
         if (!tmpfp) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        fwrite (rsp.spec, size, 1, tmpfp);
-        fflush (tmpfp);
+        fwrite(rsp.spec, size, 1, tmpfp);
+        fflush(tmpfp);
+        if (ferror(tmpfp)) {
+            ret = -1;
+            goto out;
+        }
 
         /*  Check if only options have changed. No need to reload the
-        *  volfile if topology hasn't changed.
-        *  glusterfs_volfile_reconfigure returns 3 possible return states
-        *  return 0          =======> reconfiguration of options has succeded
-        *  return 1          =======> the graph has to be reconstructed and all the xlators should be inited
-        *  return -1(or -ve) =======> Some Internal Error occured during the operation
-        */
-
-        ret = glusterfs_volfile_reconfigure (tmpfp);
+         *  volfile if topology hasn't changed.
+         *  glusterfs_volfile_reconfigure returns 3 possible return states
+         *  return 0          =======> reconfiguration of options has succeeded
+         *  return 1          =======> the graph has to be reconstructed and all
+         * the xlators should be inited return -1(or -ve) =======> Some Internal
+         * Error occurred during the operation
+         */
+
+        ret = glusterfs_volfile_reconfigure(tmpfp, ctx);
         if (ret == 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "No need to re-load volfile, reconfigure done");
-                goto out;
+            gf_log("glusterfsd-mgmt", GF_LOG_DEBUG,
+                   "No need to re-load volfile, reconfigure done");
+            if (!volfile_tmp) {
+                ret = -1;
+                UNLOCK(&ctx->volfile_lock);
+                gf_log("mgmt", GF_LOG_ERROR,
+                       "Graph reconfigure succeeded with out having "
+                       "checksum.");
+                goto post_unlock;
+            }
+            memcpy(volfile_tmp->volfile_checksum, sha256_hash,
+                   sizeof(volfile_tmp->volfile_checksum));
+            goto out;
         }
 
         if (ret < 0) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG, "Reconfigure failed !!");
-                goto out;
+            UNLOCK(&ctx->volfile_lock);
+            gf_log("glusterfsd-mgmt", GF_LOG_DEBUG, "Reconfigure failed !!");
+            goto post_unlock;
         }
 
-        ret = glusterfs_process_volfp (ctx, tmpfp);
+        ret = glusterfs_process_volfp(ctx, tmpfp);
+        /* tmpfp closed */
+        tmpfp = NULL;
+        tmp_fd = -1;
         if (ret)
+            goto out;
+
+        if (!volfile_tmp) {
+            volfile_tmp = GF_CALLOC(1, sizeof(gf_volfile_t),
+                                    gf_common_volfile_t);
+            if (!volfile_tmp) {
+                ret = -1;
                 goto out;
+            }
 
-        oldvollen = size;
-        memcpy (oldvolfile, rsp.spec, size);
-        if (!is_mgmt_rpc_reconnect) {
-                glusterfs_mgmt_pmap_signin (ctx);
-                is_mgmt_rpc_reconnect = 1;
+            INIT_LIST_HEAD(&volfile_tmp->volfile_list);
+            volfile_tmp->graph = ctx->active;
+            list_add(&volfile_tmp->volfile_list, &ctx->volfile_list);
+            snprintf(volfile_tmp->vol_id, sizeof(volfile_tmp->vol_id), "%s",
+                     volfile_id);
         }
+        memcpy(volfile_tmp->volfile_checksum, sha256_hash,
+               sizeof(volfile_tmp->volfile_checksum));
+    }
+    UNLOCK(&ctx->volfile_lock);
+
+    locked = 0;
+
+post_graph_mgmt:
+    if (!is_mgmt_rpc_reconnect) {
+        need_emancipate = 1;
+        glusterfs_mgmt_pmap_signin(ctx);
+        is_mgmt_rpc_reconnect = _gf_true;
+    }
 
 out:
-        STACK_DESTROY (frame->root);
 
-        if (rsp.spec)
-                free (rsp.spec);
+    if (locked)
+        UNLOCK(&ctx->volfile_lock);
+post_unlock:
+    GF_FREE(frame->local);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_log("mgmt", GF_LOG_ERROR,
+               "Server is operating at an "
+               "op-version which is not supported");
+        cleanup_and_exit(0);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_log("mgmt", GF_LOG_ERROR, "failed to fetch volume file (key:%s)",
+               ctx->cmd_args.volfile_id);
+        cleanup_and_exit(0);
+    }
+
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
+
+    return 0;
+}
 
-        if (ret && ctx && !ctx->active) {
-                /* Do it only for the first time */
-                /* Failed to get the volume file, something wrong,
-                   restart the process */
-                gf_log ("mgmt", GF_LOG_ERROR,
-                        "failed to fetch volume file (key:%s)",
-                        ctx->cmd_args.volfile_id);
-                cleanup_and_exit (0);
+static int
+glusterfs_volfile_fetch_one(glusterfs_ctx_t *ctx, char *volfile_id)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    if (!volfile_id) {
+        volfile_id = ctx->cmd_args.volfile_id;
+        if (!volfile_id) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "No volfile-id provided, erroring out");
+            return -1;
         }
-        return 0;
-}
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    req.key = volfile_id;
+    req.flags = 0;
+    /*
+     * We are only storing one variable in local, hence using the same
+     * variable. If multiple local variable is required, create a struct.
+     */
+    frame->local = gf_strdup(volfile_id);
+    if (!frame->local) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version"
+               " in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version"
+               " in request dict");
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    if (cmd_args->brick_name) {
+        ret = dict_set_dynstr_with_alloc(dict, "brick_name",
+                                         cmd_args->brick_name);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Failed to set brick_name in request dict");
+            goto out;
+        }
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to serialize dictionary");
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 
+out:
+    GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
+    if (ret && frame) {
+        /* Free the frame->local fast, because we have not used memget
+         */
+        GF_FREE(frame->local);
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
+
+    return ret;
+}
 
 int
-glusterfs_volfile_fetch (glusterfs_ctx_t *ctx)
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t       *cmd_args = NULL;
-        gf_getspec_req    req = {0, };
-        int               ret = 0;
-        call_frame_t     *frame = NULL;
-
-        cmd_args = &ctx->cmd_args;
+    xlator_t *server_xl = NULL;
+    xlator_list_t *trav;
+    gf_volfile_t *volfile_obj = NULL;
+    int ret = 0;
+
+    LOCK(&ctx->volfile_lock);
+    {
+        if (ctx->active &&
+            mgmt_is_multiplexed_daemon(ctx->cmd_args.process_name)) {
+            list_for_each_entry(volfile_obj, &ctx->volfile_list, volfile_list)
+            {
+                ret |= glusterfs_volfile_fetch_one(ctx, volfile_obj->vol_id);
+            }
+            UNLOCK(&ctx->volfile_lock);
+            return ret;
+        }
 
-        frame = create_frame (THIS, ctx->pool);
+        if (ctx->active) {
+            server_xl = ctx->active->first;
+            if (strcmp(server_xl->type, "protocol/server") != 0) {
+                server_xl = NULL;
+            }
+        }
+        if (!server_xl) {
+            /* Startup (ctx->active not set) or non-server. */
+            UNLOCK(&ctx->volfile_lock);
+            return glusterfs_volfile_fetch_one(ctx, ctx->cmd_args.volfile_id);
+        }
 
-        req.key = cmd_args->volfile_id;
-        req.flags = 0;
+        ret = 0;
+        for (trav = server_xl->children; trav; trav = trav->next) {
+            ret |= glusterfs_volfile_fetch_one(ctx, trav->xlator->volfile_id);
+        }
+    }
+    UNLOCK(&ctx->volfile_lock);
+    return ret;
+}
 
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GETSPEC, xdr_from_getspec_req,
-                                   mgmt_getspec_cbk);
-        return ret;
+int32_t
+mgmt_event_notify_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the rsp from server");
+        ret = -1;
+        goto out;
+    }
+out:
+    free(rsp.dict.dict_val);  // malloced by xdr
+    return ret;
 }
 
+int32_t
+glusterfs_rebalance_event_notify_cbk(struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get the rsp from server");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Received error (%s) from server", strerror(rsp.op_errno));
+        ret = -1;
+        goto out;
+    }
+out:
+    free(rsp.dict.dict_val);  // malloced by xdr
 
-static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                 void *data)
+    if (frame) {
+        STACK_DESTROY(frame->root);
+    }
+
+    return ret;
+}
+
+int32_t
+glusterfs_rebalance_event_notify(dict_t *dict)
 {
-        xlator_t        *this = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        int              ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    gf_event_notify_req req = {
+        0,
+    };
+    int32_t ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    call_frame_t *frame = NULL;
 
-        this = mydata;
-        ctx = this->ctx;
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
+
+    frame = create_frame(THIS, ctx->pool);
+
+    req.op = GF_EN_DEFRAG_STATUS;
+
+    if (dict) {
+        ret = dict_set_str(dict, "volname", cmd_args->volfile_id);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "failed to set volname");
+        }
+        ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                          &req.dict.dict_len);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "failed to serialize dict");
+        }
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_EVENT_NOTIFY,
+                              glusterfs_rebalance_event_notify_cbk,
+                              (xdrproc_t)xdr_gf_event_notify_req);
+
+    GF_FREE(req.dict.dict_val);
+    return ret;
+}
 
-        switch (event) {
+static int
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
+{
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    int need_term = 0;
+    int emval = 0;
+    static int log_ctr1;
+    static int log_ctr2;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+    ctx = this->ctx;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (rpc_trans->connect_failed) {
+                GF_LOG_OCCASIONALLY(log_ctr1, "glusterfsd-mgmt", GF_LOG_ERROR,
+                                    "failed to connect to remote-"
+                                    "host: %s",
+                                    ctx->cmd_args.volfile_server);
+            } else {
+                GF_LOG_OCCASIONALLY(log_ctr1, "glusterfsd-mgmt", GF_LOG_INFO,
+                                    "disconnected from remote-"
+                                    "host: %s",
+                                    ctx->cmd_args.volfile_server);
+            }
+
+            if (!rpc->disabled) {
+                /*
+                 * Check if dnscache is exhausted for current server
+                 * and continue until cache is exhausted
+                 */
+                dnscache = rpc_trans->dnscache;
+                if (dnscache && dnscache->next) {
+                    break;
+                }
+            }
+            server = ctx->cmd_args.curr_server;
+            if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                if (!ctx->active) {
+                    need_term = 1;
+                }
+                emval = ENOTCONN;
+                GF_LOG_OCCASIONALLY(log_ctr2, "glusterfsd-mgmt", GF_LOG_INFO,
+                                    "Exhausted all volfile servers");
+                break;
+            }
+            server = list_entry(server->list.next, typeof(*server), list);
+            ctx->cmd_args.curr_server = server;
+            ctx->cmd_args.volfile_server = server->volfile_server;
+
+            ret = dict_set_str(rpc_trans->options, "remote-host",
+                               server->volfile_server);
+            if (ret != 0) {
+                gf_log("glusterfsd-mgmt", GF_LOG_ERROR,
+                       "failed to set remote-host: %s", server->volfile_server);
+                if (!ctx->active) {
+                    need_term = 1;
+                }
+                emval = ENOTCONN;
+                break;
+            }
+            gf_log("glusterfsd-mgmt", GF_LOG_INFO,
+                   "connecting to next volfile server %s",
+                   server->volfile_server);
+            break;
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-                ret = glusterfs_volfile_fetch (ctx);
-                if (ret && ctx && (ctx->active == NULL)) {
-                        /* Do it only for the first time */
-                        /* Exit the process.. there is some wrong options */
-                        gf_log ("mgmt", GF_LOG_ERROR,
-                                "failed to fetch volume file (key:%s)",
-                                ctx->cmd_args.volfile_id);
-                        cleanup_and_exit (0);
+            ret = glusterfs_volfile_fetch(ctx);
+            if (ret) {
+                emval = ret;
+                if (!ctx->active) {
+                    need_term = 1;
+                    gf_log("glusterfsd-mgmt", GF_LOG_ERROR,
+                           "failed to fetch volume file (key:%s)",
+                           ctx->cmd_args.volfile_id);
+                    break;
                 }
+            }
 
-                if (is_mgmt_rpc_reconnect)
-                        glusterfs_mgmt_pmap_signin (ctx);
-                break;
+            if (is_mgmt_rpc_reconnect)
+                glusterfs_mgmt_pmap_signin(ctx);
+
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        return 0;
-}
+    if (need_term) {
+        emancipate(ctx, emval);
+        cleanup_and_exit(1);
+    }
 
+    return 0;
+}
 
 int
-glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
+glusterfs_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                        void *data)
 {
-        cmd_args_t              *cmd_args = NULL;
-        struct rpc_clnt         *rpc = NULL;
-        struct rpc_clnt_config  rpc_cfg = {0,};
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-        int                     port = GF_DEFAULT_BASE_PORT;
-        char                    *host = NULL;
+    if (!xl || !data) {
+        goto out;
+    }
 
-        cmd_args = &ctx->cmd_args;
+    switch (event) {
+        case RPCSVC_EVENT_ACCEPT: {
+            break;
+        }
+        case RPCSVC_EVENT_DISCONNECT: {
+            break;
+        }
 
-        if (ctx->mgmt)
-                return 0;
+        default:
+            break;
+    }
 
-        options = dict_new ();
-        if (!options)
-                goto out;
+out:
+    return 0;
+}
 
-        if (cmd_args->volfile_server_port)
-                port = cmd_args->volfile_server_port;
+int
+glusterfs_listener_init(glusterfs_ctx_t *ctx)
+{
+    cmd_args_t *cmd_args = NULL;
+    rpcsvc_t *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
 
-        host = "localhost";
-        if (cmd_args->volfile_server)
-                host = cmd_args->volfile_server;
+    cmd_args = &ctx->cmd_args;
 
-        rpc_cfg.remote_host = host;
-        rpc_cfg.remote_port = port;
+    if (ctx->listener)
+        return 0;
 
-        ret = dict_set_int32 (options, "remote-port", port);
-        if (ret)
-                goto out;
+    if (!cmd_args->sock_file)
+        return 0;
 
-        ret = dict_set_str (options, "remote-host", host);
-        if (ret)
-                goto out;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        ret = dict_set_str (options, "transport.address-family", "inet");
-        if (ret)
-                goto out;
+    ret = rpcsvc_transport_unix_options_build(options, cmd_args->sock_file);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
+    rpc = rpcsvc_init(THIS, ctx, options, 8);
+    if (rpc == NULL) {
+        goto out;
+    }
 
-        rpc = rpc_clnt_init (&rpc_cfg, options, THIS->ctx, THIS->name);
-        if (!rpc) {
-                ret = -1;
-                goto out;
-        }
+    ret = rpcsvc_register_notify(rpc, glusterfs_rpcsvc_notify, THIS);
+    if (ret) {
+        goto out;
+    }
 
-        ctx->mgmt = rpc;
+    ret = rpcsvc_create_listeners(rpc, options, "glusterfsd");
+    if (ret < 1) {
+        goto out;
+    }
 
-        ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-        if (ret)
-                goto out;
+    ret = rpcsvc_program_register(rpc, &glusterfs_mop_prog, _gf_false);
+    if (ret) {
+        goto out;
+    }
 
-        ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog);
-        if (ret)
-                goto out;
+    ctx->listener = rpc;
 
 out:
-        return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
 
-
-static int
-mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+int
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-        pmap_signin_rsp  rsp   = {0,};
-        call_frame_t    *frame = NULL;
-        int              ret   = 0;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    int ret = 0;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            ret = glusterfs_rebalance_event_notify((dict_t *)data);
+            break;
 
-        ret = xdr_to_pmap_signin_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to register the port with glusterd");
-                goto out;
-        }
-out:
+        default:
+            gf_log("", GF_LOG_ERROR, "Invalid op");
+            break;
+    }
 
-        STACK_DESTROY (frame->root);
-        return 0;
+    return ret;
 }
 
 int
-glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
+glusterfs_mgmt_init(glusterfs_ctx_t *ctx)
 {
-        call_frame_t     *frame = NULL;
-        pmap_signin_req   req = {0, };
-        int               ret = -1;
-        cmd_args_t       *cmd_args = NULL;
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    xlator_cmdline_option_t *opt = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args->volfile_server, out);
+
+    if (ctx->mgmt)
+        return 0;
 
-        frame = create_frame (THIS, ctx->pool);
-        cmd_args = &ctx->cmd_args;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        if (!cmd_args->brick_port || !cmd_args->brick_name) {
-                gf_log ("fsd-mgmt", GF_LOG_DEBUG,
-                        "portmapper signin arguments not given");
-                goto out;
-        }
+    LOCK_INIT(&ctx->volfile_lock);
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    host = cmd_args->volfile_server;
 
-        req.port  = cmd_args->brick_port;
-        req.brick = cmd_args->brick_name;
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        opt = find_xlator_option_in_cmd_args_t("address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+    if (ret)
+        goto out;
 
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
-                                   GF_PMAP_SIGNIN, xdr_from_pmap_signin_req,
-                                   mgmt_pmap_signin_cbk);
+    /* Explicitly turn on encrypted transport. */
+    if (ctx->secure_mgmt) {
+        ret = dict_set_dynstr_with_alloc(options,
+                                         "transport.socket.ssl-enabled", "yes");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set 'transport.socket.ssl-enabled' "
+                   "in options dict");
+            goto out;
+        }
 
+        ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to create rpc clnt");
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "failed to register notify function");
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "failed to register callback function");
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-        return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
 
-
 static int
-mgmt_pmap_signout_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+mgmt_pmap_signin2_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        pmap_signout_rsp  rsp   = {0,};
-        call_frame_t    *frame = NULL;
-        int              ret   = 0;
-	glusterfs_ctx_t	 *ctx = NULL;
-
-        frame = myframe;
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    ctx = glusterfsd_ctx;
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decode error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to register the port with glusterd");
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (need_emancipate)
+        emancipate(ctx, ret);
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
-        ctx = glusterfs_ctx_get ();
-        ret = xdr_to_pmap_signout_rsp (*iov, &rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+static int
+mgmt_pmap_signin_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    int emancipate_ret = -1;
+    pmap_signin_req pmap_req = {
+        0,
+    };
+    cmd_args_t *cmd_args = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char brick_name[PATH_MAX] = {
+        0,
+    };
+
+    frame = myframe;
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_signin_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decode error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to register the port with glusterd");
+        ret = -1;
+        goto out;
+    }
+
+    if (!cmd_args->brick_port2) {
+        /* We are done with signin process */
+        emancipate_ret = 0;
+        goto out;
+    }
+
+    snprintf(brick_name, PATH_MAX, "%s.rdma", cmd_args->brick_name);
+    pmap_req.port = cmd_args->brick_port2;
+    pmap_req.brick = brick_name;
+
+    ret = mgmt_submit_request(&pmap_req, frame, ctx, &clnt_pmap_prog,
+                              GF_PMAP_SIGNIN, mgmt_pmap_signin2_cbk,
+                              (xdrproc_t)xdr_pmap_signin_req);
+    if (ret)
+        goto out;
+
+    return 0;
 
-        if (-1 == rsp.op_ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "failed to register the port with glusterd");
-                goto out;
-        }
 out:
-//        if (frame)
-//                STACK_DESTROY (frame->root);
-        return 0;
-}
+    if (need_emancipate && (ret < 0 || !cmd_args->brick_port2))
+        emancipate(ctx, emancipate_ret);
 
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
 int
-glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx)
 {
-        int               ret = 0;
-        pmap_signout_req  req = {0, };
-        call_frame_t     *frame = NULL;
-        cmd_args_t       *cmd_args = NULL;
-
-        frame = create_frame (THIS, ctx->pool);
-        cmd_args = &ctx->cmd_args;
-
-        if (!cmd_args->brick_port || !cmd_args->brick_name) {
-                gf_log ("fsd-mgmt", GF_LOG_DEBUG,
-                        "portmapper signout arguments not given");
-                goto out;
+    call_frame_t *frame = NULL;
+    xlator_list_t **trav_p;
+    xlator_t *top;
+    pmap_signin_req req = {
+        0,
+    };
+    int ret = -1;
+    int emancipate_ret = -1;
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args->brick_port || !cmd_args->brick_name) {
+        gf_log("fsd-mgmt", GF_LOG_DEBUG,
+               "portmapper signin arguments not given");
+        emancipate_ret = 0;
+        goto out;
+    }
+
+    req.port = cmd_args->brick_port;
+    req.pid = (int)getpid(); /* only glusterd2 consumes this */
+
+    if (ctx->active) {
+        top = ctx->active->first;
+        for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+            frame = create_frame(THIS, ctx->pool);
+            req.brick = (*trav_p)->xlator->name;
+            ret = mgmt_submit_request(&req, frame, ctx, &clnt_pmap_prog,
+                                      GF_PMAP_SIGNIN, mgmt_pmap_signin_cbk,
+                                      (xdrproc_t)xdr_pmap_signin_req);
+            if (ret < 0) {
+                gf_log(THIS->name, GF_LOG_WARNING,
+                       "failed to send sign in request; brick = %s", req.brick);
+            }
         }
+    }
 
-        req.port  = cmd_args->brick_port;
-        req.brick = cmd_args->brick_name;
+    /* unfortunately, the caller doesn't care about the returned value */
 
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
-                                   GF_PMAP_SIGNOUT, xdr_from_pmap_signout_req,
-                                   mgmt_pmap_signout_cbk);
 out:
-        return ret;
+    if (need_emancipate && ret < 0)
+        emancipate(ctx, emancipate_ret);
+    return ret;
 }
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 8cfe4f4a9ec..dae41f33fef 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -25,9 +15,11 @@
 #include <sys/types.h>
 #include <sys/resource.h>
 #include <sys/file.h>
+#include <sys/wait.h>
 #include <netdb.h>
 #include <signal.h>
 #include <libgen.h>
+#include <dlfcn.h>
 
 #include <sys/utsname.h>
 
@@ -38,1383 +30,2709 @@
 #include <time.h>
 #include <semaphore.h>
 #include <errno.h>
+#include <pwd.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
+#ifdef GF_LINUX_HOST_OS
+#ifdef HAVE_LINUX_OOM_H
+#include <linux/oom.h>
+#else
+#define OOM_SCORE_ADJ_MIN (-1000)
+#define OOM_SCORE_ADJ_MAX 1000
+#define OOM_DISABLE (-17)
+#define OOM_ADJUST_MAX 15
+#endif
 #endif
 
 #ifdef HAVE_MALLOC_H
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "list.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include "glusterfsd-messages.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
 #include "glusterfsd.h"
-#include "stack.h"
-#include "revision.h"
-#include "common-utils.h"
-#include "event.h"
-#include "globals.h"
-#include "statedump.h"
-#include "latency.h"
+#include <glusterfs/revision.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/latency.h>
 #include "glusterfsd-mem-types.h"
-#include "syscall.h"
-#include "call-stub.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/call-stub.h>
 #include <fnmatch.h>
 #include "rpc-clnt.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/client_t.h>
+#include "netgroups.h"
+#include "exports.h"
+#include <glusterfs/monitoring.h>
 
-#ifdef GF_DARWIN_HOST_OS
-#include "daemon.h"
-#else
-#define os_daemon(u, v) daemon (u, v)
-#endif
-
+#include <glusterfs/daemon.h>
 
 /* using argp for command line parsing */
 static char gf_doc[] = "";
-static char argp_doc[] = "--volfile-server=SERVER [MOUNT-POINT]\n"       \
-        "--volfile=VOLFILE [MOUNT-POINT]";
-const char *argp_program_version = "" \
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__ \
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"  \
-        "Copyright (c) 2006-2010 Gluster Inc. "             \
-        "<http://www.gluster.com>\n"                                \
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"              \
-        "You may redistribute copies of GlusterFS under the terms of "\
-        "the GNU Affero General Public License.";
+static char argp_doc[] =
+    "--volfile-server=SERVER [MOUNT-POINT]\n"
+    "--volfile=VOLFILE [MOUNT-POINT]";
+const char *argp_program_version =
+    "" PACKAGE_NAME " " PACKAGE_VERSION
+    "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION
+    "\n"
+    "Copyright (c) 2006-2016 Red Hat, Inc. "
+    "<https://www.gluster.org/>\n"
+    "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+    "It is licensed to you under your choice of the GNU Lesser\n"
+    "General Public License, version 3 or any later version (LGPLv3\n"
+    "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+    "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
-static error_t parse_opts (int32_t key, char *arg, struct argp_state *_state);
+static error_t
+parse_opts(int32_t key, char *arg, struct argp_state *_state);
 
 static struct argp_option gf_options[] = {
-        {0, 0, 0, 0, "Basic options:"},
-        {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
-         "Server to get the volume file from.  This option overrides "
-         "--volfile option"},
-        {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS,
-         "MAX-ATTEMPTS", 0, "Maximum number of connect attempts to server. "
-         "This option should be provided with --volfile-server option"
-         "[default: 1]"},
-        {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
-         "File to use as VOLUME_FILE"},
-        {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
-         "File to use as VOLUME FILE"},
-        {"log-server", ARGP_LOG_SERVER_KEY, "LOGSERVER", 0,
-         "Server to use as the central log server"},
-
-        {"log-level", ARGP_LOG_LEVEL_KEY, "LOGLEVEL", 0,
-         "Logging severity.  Valid options are DEBUG, NORMAL, WARNING, ERROR, "
-         "CRITICAL and NONE [default: NORMAL]"},
-        {"log-file", ARGP_LOG_FILE_KEY, "LOGFILE", 0,
-         "File to use for logging [default: "
-         DEFAULT_LOG_FILE_DIRECTORY "/" PACKAGE_NAME ".log" "]"},
-
-        {0, 0, 0, 0, "Advanced Options:"},
-        {"volfile-server-port", ARGP_VOLFILE_SERVER_PORT_KEY, "PORT", 0,
-         "Listening port number of volfile server"},
-        {"volfile-server-transport", ARGP_VOLFILE_SERVER_TRANSPORT_KEY,
-         "TRANSPORT", 0,
-         "Transport type to get volfile from server [default: socket]"},
-        {"volfile-id", ARGP_VOLFILE_ID_KEY, "KEY", 0,
-         "'key' of the volfile to be fetched from server"},
-        {"log-server-port", ARGP_LOG_SERVER_PORT_KEY, "PORT", 0,
-         "Listening port number of log server"},
-        {"pid-file", ARGP_PID_FILE_KEY, "PIDFILE", 0,
-         "File to use as pid file"},
-        {"no-daemon", ARGP_NO_DAEMON_KEY, 0, 0,
-         "Run in foreground"},
-        {"run-id", ARGP_RUN_ID_KEY, "RUN-ID", OPTION_HIDDEN,
-         "Run ID for the process, used by scripts to keep track of process "
-         "they started, defaults to none"},
-        {"debug", ARGP_DEBUG_KEY, 0, 0,
-         "Run in debug mode.  This option sets --no-daemon, --log-level "
-         "to DEBUG and --log-file to console"},
-        {"volume-name", ARGP_VOLUME_NAME_KEY, "VOLUME-NAME", 0,
-         "Volume name to be used for MOUNT-POINT [default: top most volume "
-         "in VOLFILE]"},
-        {"xlator-option", ARGP_XLATOR_OPTION_KEY,"VOLUME-NAME.OPTION=VALUE", 0,
-         "Add/override a translator option for a volume with specified value"},
-        {"read-only", ARGP_READ_ONLY_KEY, 0, 0,
-         "Mount the filesystem in 'read-only' mode"},
-        {"mac-compat", ARGP_MAC_COMPAT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Provide stubs for attributes needed for seamless operation on Macs "
+    {0, 0, 0, 0, "Basic options:"},
+    {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
+     "Server to get the volume file from. Unix domain socket path when "
+     "transport type 'unix'. This option overrides --volfile option"},
+    {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
+     "File to use as VOLUME_FILE"},
+    {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
+     "File to use as VOLUME FILE"},
+
+    {"log-level", ARGP_LOG_LEVEL_KEY, "LOGLEVEL", 0,
+     "Logging severity.  Valid options are DEBUG, INFO, WARNING, ERROR, "
+     "CRITICAL, TRACE and NONE [default: INFO]"},
+    {"log-file", ARGP_LOG_FILE_KEY, "LOGFILE", 0,
+     "File to use for logging [default: " DEFAULT_LOG_FILE_DIRECTORY
+     "/" PACKAGE_NAME ".log"
+     "]"},
+    {"logger", ARGP_LOGGER, "LOGGER", 0,
+     "Set which logging sub-system to "
+     "log to, valid options are: gluster-log and syslog, "
+     "[default: \"gluster-log\"]"},
+    {"log-format", ARGP_LOG_FORMAT, "LOG-FORMAT", 0,
+     "Set log format, valid"
+     " options are: no-msg-id and with-msg-id, [default: \"with-msg-id\"]"},
+    {"log-buf-size", ARGP_LOG_BUF_SIZE, "LOG-BUF-SIZE", 0,
+     "Set logging "
+     "buffer size, [default: 5]"},
+    {"log-flush-timeout", ARGP_LOG_FLUSH_TIMEOUT, "LOG-FLUSH-TIMEOUT", 0,
+     "Set log flush timeout, [default: 2 minutes]"},
+
+    {0, 0, 0, 0, "Advanced Options:"},
+    {"volfile-server-port", ARGP_VOLFILE_SERVER_PORT_KEY, "PORT", 0,
+     "Listening port number of volfile server"},
+    {"volfile-server-transport", ARGP_VOLFILE_SERVER_TRANSPORT_KEY, "TRANSPORT",
+     0, "Transport type to get volfile from server [default: socket]"},
+    {"volfile-id", ARGP_VOLFILE_ID_KEY, "KEY", 0,
+     "'key' of the volfile to be fetched from server"},
+    {"pid-file", ARGP_PID_FILE_KEY, "PIDFILE", 0, "File to use as pid file"},
+    {"socket-file", ARGP_SOCK_FILE_KEY, "SOCKFILE", 0,
+     "File to use as unix-socket"},
+    {"no-daemon", ARGP_NO_DAEMON_KEY, 0, 0, "Run in foreground"},
+    {"run-id", ARGP_RUN_ID_KEY, "RUN-ID", OPTION_HIDDEN,
+     "Run ID for the process, used by scripts to keep track of process "
+     "they started, defaults to none"},
+    {"debug", ARGP_DEBUG_KEY, 0, 0,
+     "Run in debug mode.  This option sets --no-daemon, --log-level "
+     "to DEBUG and --log-file to console"},
+    {"volume-name", ARGP_VOLUME_NAME_KEY, "XLATOR-NAME", 0,
+     "Translator name to be used for MOUNT-POINT [default: top most volume "
+     "definition in VOLFILE]"},
+    {"xlator-option", ARGP_XLATOR_OPTION_KEY, "XLATOR-NAME.OPTION=VALUE", 0,
+     "Add/override an option for a translator in volume file with specified"
+     " value"},
+    {"read-only", ARGP_READ_ONLY_KEY, 0, 0,
+     "Mount the filesystem in 'read-only' mode"},
+    {"acl", ARGP_ACL_KEY, 0, 0, "Mount the filesystem with POSIX ACL support"},
+    {"selinux", ARGP_SELINUX_KEY, 0, 0,
+     "Enable SELinux label (extended attributes) support on inodes"},
+    {"capability", ARGP_CAPABILITY_KEY, 0, 0,
+     "Enable Capability (extended attributes) support on inodes"},
+    {"subdir-mount", ARGP_SUBDIR_MOUNT_KEY, "SUBDIR-PATH", 0,
+     "Mount subdirectory given [default: NULL]"},
+
+    {"print-netgroups", ARGP_PRINT_NETGROUPS, "NETGROUP-FILE", 0,
+     "Validate the netgroups file and print it out"},
+    {"print-exports", ARGP_PRINT_EXPORTS, "EXPORTS-FILE", 0,
+     "Validate the exports file and print it out"},
+    {"print-xlatordir", ARGP_PRINT_XLATORDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print xlator directory path"},
+    {"print-statedumpdir", ARGP_PRINT_STATEDUMPDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print directory path in which statedumps shall be generated"},
+    {"print-logdir", ARGP_PRINT_LOGDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print path of default log directory"},
+    {"print-libexecdir", ARGP_PRINT_LIBEXECDIR_KEY, 0, OPTION_ARG_OPTIONAL,
+     "Print path of default libexec directory"},
+
+    {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS, "0",
+     OPTION_HIDDEN, "Maximum number of attempts to fetch the volfile"},
+    {"aux-gfid-mount", ARGP_AUX_GFID_MOUNT_KEY, 0, 0,
+     "Enable access to filesystem through gfid directly"},
+    {"enable-ino32", ARGP_INODE32_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use 32-bit inodes when mounting to workaround broken applications"
+     "that don't support 64-bit inodes"},
+    {"worm", ARGP_WORM_KEY, 0, 0, "Mount the filesystem in 'worm' mode"},
+    {"mac-compat", ARGP_MAC_COMPAT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Provide stubs for attributes needed for seamless operation on Macs "
 #ifdef GF_DARWIN_HOST_OS
-         "[default: \"on\" on client side, else \"off\"]"
+     "[default: \"on\" on client side, else \"off\"]"
 #else
-         "[default: \"off\"]"
+     "[default: \"off\"]"
 #endif
-        },
-        {"brick-name", ARGP_BRICK_NAME_KEY, "BRICK-NAME", OPTION_HIDDEN,
-         "Brick name to be registered with Gluster portmapper" },
-        {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN,
-         "Brick Port to be registered with Gluster portmapper" },
-
-        {0, 0, 0, 0, "Fuse options:"},
-        {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
-         "Use direct I/O mode in fuse kernel module"
-         " [default: \"off\" if big writes are supported, else "
-         "\"on\" for fds not opened with O_RDONLY]"},
-        {"entry-timeout", ARGP_ENTRY_TIMEOUT_KEY, "SECONDS", 0,
-         "Set entry timeout to SECONDS in fuse kernel module [default: 1]"},
-        {"attribute-timeout", ARGP_ATTRIBUTE_TIMEOUT_KEY, "SECONDS", 0,
-         "Set attribute timeout to SECONDS for inodes in fuse kernel module "
-         "[default: 1]"},
-        {"dump-fuse", ARGP_DUMP_FUSE_KEY, "PATH", 0,
-         "Dump fuse traffic to PATH"},
-        {"volfile-check", ARGP_VOLFILE_CHECK_KEY, 0, 0,
-         "Enable strict volume file checking"},
-        {0, 0, 0, 0, "Miscellaneous Options:"},
-        {0, }
-};
-
-
-static struct argp argp = { gf_options, parse_opts, argp_doc, gf_doc };
-
-int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx);
-int glusterfs_volumes_init (glusterfs_ctx_t *ctx);
-int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
+    },
+    {"brick-name", ARGP_BRICK_NAME_KEY, "BRICK-NAME", OPTION_HIDDEN,
+     "Brick name to be registered with Gluster portmapper"},
+    {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN,
+     "Brick Port to be registered with Gluster portmapper"},
+    {"fopen-keep-cache", ARGP_FOPEN_KEEP_CACHE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Do not purge the cache on file open [default: false]"},
+    {"global-timer-wheel", ARGP_GLOBAL_TIMER_WHEEL, "BOOL", OPTION_ARG_OPTIONAL,
+     "Instantiate process global timer-wheel"},
+    {"thin-client", ARGP_THIN_CLIENT_KEY, 0, 0,
+     "Enables thin mount and connects via gfproxyd daemon"},
+    {"global-threading", ARGP_GLOBAL_THREADING_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use the global thread pool instead of io-threads"},
+    {0, 0, 0, 0, "Fuse options:"},
+    {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL|auto",
+     OPTION_ARG_OPTIONAL, "Specify direct I/O strategy [default: \"auto\"]"},
+    {"entry-timeout", ARGP_ENTRY_TIMEOUT_KEY, "SECONDS", 0,
+     "Set entry timeout to SECONDS in fuse kernel module [default: 1]"},
+    {"negative-timeout", ARGP_NEGATIVE_TIMEOUT_KEY, "SECONDS", 0,
+     "Set negative timeout to SECONDS in fuse kernel module [default: 0]"},
+    {"attribute-timeout", ARGP_ATTRIBUTE_TIMEOUT_KEY, "SECONDS", 0,
+     "Set attribute timeout to SECONDS for inodes in fuse kernel module "
+     "[default: 1]"},
+    {"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
+     "Set auxiliary group list timeout to SECONDS for fuse translator "
+     "[default: 300]"},
+    {"resolve-gids", ARGP_RESOLVE_GIDS_KEY, 0, 0,
+     "Resolve all auxiliary groups in fuse translator (max 32 otherwise)"},
+    {"lru-limit", ARGP_FUSE_LRU_LIMIT_KEY, "N", 0,
+     "Set fuse module's limit for number of inodes kept in LRU list to N "
+     "[default: 65536]"},
+    {"invalidate-limit", ARGP_FUSE_INVALIDATE_LIMIT_KEY, "N", 0,
+     "Suspend inode invalidations implied by 'lru-limit' if the number of "
+     "outstanding invalidations reaches N"},
+    {"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
+     "Set fuse module's background queue length to N "
+     "[default: 64]"},
+    {"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
+     "Set fuse module's congestion threshold to N "
+     "[default: 48]"},
+#ifdef GF_LINUX_HOST_OS
+    {"oom-score-adj", ARGP_OOM_SCORE_ADJ_KEY, "INTEGER", 0,
+     "Set oom_score_adj value for process"
+     "[default: 0]"},
+#endif
+    {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
+     "client will authenticate itself with process id PID to server"},
+    {"no-root-squash", ARGP_FUSE_NO_ROOT_SQUASH_KEY, "BOOL",
+     OPTION_ARG_OPTIONAL,
+     "disable/enable root squashing for the trusted "
+     "client"},
+    {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
+     "replace USER with root in messages"},
+    {"dump-fuse", ARGP_DUMP_FUSE_KEY, "PATH", 0, "Dump fuse traffic to PATH"},
+    {"volfile-check", ARGP_VOLFILE_CHECK_KEY, 0, 0,
+     "Enable strict volume file checking"},
+    {"no-mem-accounting", ARGP_MEM_ACCOUNTING_KEY, 0, OPTION_HIDDEN,
+     "disable internal memory accounting"},
+    {"fuse-mountopts", ARGP_FUSE_MOUNTOPTS_KEY, "OPTIONS", OPTION_HIDDEN,
+     "Extra mount options to pass to FUSE"},
+    {"use-readdirp", ARGP_FUSE_USE_READDIRP_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Use readdirp mode in fuse kernel module"
+     " [default: \"yes\"]"},
+    {"secure-mgmt", ARGP_SECURE_MGMT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "Override default for secure (SSL) management connections"},
+    {"localtime-logging", ARGP_LOCALTIME_LOGGING_KEY, 0, 0,
+     "Enable localtime logging"},
+    {"process-name", ARGP_PROCESS_NAME_KEY, "PROCESS-NAME", OPTION_HIDDEN,
+     "option to specify the process type"},
+    {"event-history", ARGP_FUSE_EVENT_HISTORY_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "disable/enable fuse event-history"},
+    {"reader-thread-count", ARGP_READER_THREAD_COUNT_KEY, "INTEGER",
+     OPTION_ARG_OPTIONAL, "set fuse reader thread count"},
+    {"kernel-writeback-cache", ARGP_KERNEL_WRITEBACK_CACHE_KEY, "BOOL",
+     OPTION_ARG_OPTIONAL, "enable fuse in-kernel writeback cache"},
+    {"attr-times-granularity", ARGP_ATTR_TIMES_GRANULARITY_KEY, "NS",
+     OPTION_ARG_OPTIONAL,
+     "declare supported granularity of file attribute"
+     " times in nanoseconds"},
+    {"fuse-flush-handle-interrupt", ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY,
+     "BOOL", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
+     "handle interrupt in fuse FLUSH handler"},
+    {"auto-invalidation", ARGP_FUSE_AUTO_INVAL_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "controls whether fuse-kernel can auto-invalidate "
+     "attribute, dentry and page-cache. "
+     "Disable this only if same files/directories are not accessed across "
+     "two different mounts concurrently [default: \"on\"]"},
+    {"fuse-dev-eperm-ratelimit-ns", ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY,
+     "OPTIONS", OPTION_HIDDEN,
+     "rate limit reading from fuse device upon EPERM failure"},
+    {"brick-mux", ARGP_BRICK_MUX_KEY, 0, 0, "Enable brick mux. "},
+    {0, 0, 0, 0, "Miscellaneous Options:"},
+    {
+        0,
+    }};
+
+static struct argp argp = {gf_options, parse_opts, argp_doc, gf_doc};
 
 int
-create_fuse_mount (glusterfs_ctx_t *ctx)
-{
-        int              ret = 0;
-        cmd_args_t      *cmd_args = NULL;
-        xlator_t        *master = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        if (!cmd_args->mount_point)
-                return 0;
-
-        master = GF_CALLOC (1, sizeof (*master),
-                            gfd_mt_xlator_t);
-        if (!master)
-                goto err;
-
-        master->name = gf_strdup ("fuse");
-        if (!master->name)
-                goto err;
-
-        if (xlator_set_type (master, "mount/fuse") == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "MOUNT-POINT %s initialization failed",
-                        cmd_args->mount_point);
-                goto err;
-        }
+glusterfs_pidfile_cleanup(glusterfs_ctx_t *ctx);
+int
+glusterfs_volumes_init(glusterfs_ctx_t *ctx);
+int
+glusterfs_mgmt_init(glusterfs_ctx_t *ctx);
+int
+glusterfs_listener_init(glusterfs_ctx_t *ctx);
 
-        master->ctx      = ctx;
-        master->options  = get_new_dict ();
+#define DICT_SET_VAL(method, dict, key, val, msgid)                            \
+    if (method(dict, key, val)) {                                              \
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, msgid, "key=%s", key);          \
+        goto err;                                                              \
+    }
 
-        ret = dict_set_static_ptr (master->options, ZR_MOUNTPOINT_OPT,
-                                   cmd_args->mount_point);
-        if (ret < 0) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "failed to set mount-point to options dictionary");
+static int
+set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
+{
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+    char *mount_point = NULL;
+    char cwd[PATH_MAX] = {
+        0,
+    };
+
+    cmd_args = &ctx->cmd_args;
+
+    /* Check if mount-point is absolute path,
+     * if not convert to absolute path by concatenating with CWD
+     */
+    if (cmd_args->mount_point[0] != '/') {
+        if (getcwd(cwd, PATH_MAX) != NULL) {
+            ret = gf_asprintf(&mount_point, "%s/%s", cwd,
+                              cmd_args->mount_point);
+            if (ret == -1) {
+                gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_1,
+                        "gf_asprintf failed", NULL);
                 goto err;
+            }
+        } else {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_2,
+                    "getcwd failed", NULL);
+            goto err;
         }
 
-        if (cmd_args->fuse_attribute_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ATTR_TIMEOUT_OPT,
-                                       cmd_args->fuse_attribute_timeout);
-
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->fuse_entry_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ENTRY_TIMEOUT_OPT,
-                                       cmd_args->fuse_entry_timeout);
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->volfile_check) {
-                ret = dict_set_int32 (master->options, ZR_STRICT_VOLFILE_CHECK,
-                                      cmd_args->volfile_check);
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-        }
-
-        if (cmd_args->dump_fuse) {
-                ret = dict_set_static_ptr (master->options, ZR_DUMP_FUSE,
-                                           cmd_args->dump_fuse);
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-        }
-
-        switch (cmd_args->fuse_direct_io_mode) {
+    } else {
+        mount_point = gf_strdup(cmd_args->mount_point);
+    }
+    DICT_SET_VAL(dict_set_dynstr_sizen, options, ZR_MOUNTPOINT_OPT, mount_point,
+                 glusterfsd_msg_3);
+
+    if (cmd_args->fuse_attribute_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_ATTR_TIMEOUT_OPT,
+                     cmd_args->fuse_attribute_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->fuse_entry_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_ENTRY_TIMEOUT_OPT,
+                     cmd_args->fuse_entry_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->fuse_negative_timeout >= 0) {
+        DICT_SET_VAL(dict_set_double, options, ZR_NEGATIVE_TIMEOUT_OPT,
+                     cmd_args->fuse_negative_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->client_pid_set) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "client-pid",
+                     cmd_args->client_pid, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->uid_map_root) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "uid-map-root",
+                     cmd_args->uid_map_root, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->volfile_check) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, ZR_STRICT_VOLFILE_CHECK,
+                     cmd_args->volfile_check, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->dump_fuse) {
+        DICT_SET_VAL(dict_set_static_ptr, options, ZR_DUMP_FUSE,
+                     cmd_args->dump_fuse, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->acl) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "acl", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->selinux) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "selinux", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->capability) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "capability", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->aux_gfid_mount) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "virtual-gfid-access", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->enable_ino32) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "enable-ino32", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->read_only) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "read-only", "on",
+                     glusterfsd_msg_3);
+    }
+
+    switch (cmd_args->fopen_keep_cache) {
+        case GF_OPTION_ENABLE:
+
+            DICT_SET_VAL(dict_set_static_ptr, options, "fopen-keep-cache", "on",
+                         glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "fopen-keep-cache",
+                         "off", glusterfsd_msg_3);
+            break;
+        default:
+            gf_msg_debug("glusterfsd", 0, "fopen-keep-cache mode %d",
+                         cmd_args->fopen_keep_cache);
+            break;
+    }
+
+    if (cmd_args->gid_timeout_set) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "gid-timeout",
+                     cmd_args->gid_timeout, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->resolve_gids) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "resolve-gids", "on",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->lru_limit >= 0) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "lru-limit",
+                     cmd_args->lru_limit, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->invalidate_limit >= 0) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "invalidate-limit",
+                     cmd_args->invalidate_limit, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->background_qlen) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "background-qlen",
+                     cmd_args->background_qlen, glusterfsd_msg_3);
+    }
+    if (cmd_args->congestion_threshold) {
+        DICT_SET_VAL(dict_set_int32_sizen, options, "congestion-threshold",
+                     cmd_args->congestion_threshold, glusterfsd_msg_3);
+    }
+
+    switch (cmd_args->fuse_direct_io_mode) {
         case GF_OPTION_DISABLE: /* disable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
-                                           "disable");
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-                break;
+            DICT_SET_VAL(dict_set_static_ptr, options, ZR_DIRECT_IO_OPT,
+                         "disable", glusterfsd_msg_3);
+            break;
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
-                                           "enable");
-                if (ret < 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "failed to set dict value.");
-                        goto err;
-                }
-                break;
-        case GF_OPTION_DEFERRED: /* default */
+            DICT_SET_VAL(dict_set_static_ptr, options, ZR_DIRECT_IO_OPT,
+                         "enable", glusterfsd_msg_3);
+            break;
         default:
-                break;
+            gf_msg_debug("glusterfsd", 0, "fuse direct io type %d",
+                         cmd_args->fuse_direct_io_mode);
+            break;
+    }
+
+    switch (cmd_args->no_root_squash) {
+        case GF_OPTION_ENABLE: /* enable */
+            DICT_SET_VAL(dict_set_static_ptr, options, "no-root-squash",
+                         "enable", glusterfsd_msg_3);
+            break;
+        default:
+            DICT_SET_VAL(dict_set_static_ptr, options, "no-root-squash",
+                         "disable", glusterfsd_msg_3);
+            gf_msg_debug("glusterfsd", 0, "fuse no-root-squash mode %d",
+                         cmd_args->no_root_squash);
+            break;
+    }
+
+    if (!cmd_args->no_daemon_mode) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "sync-to-mount", "enable",
+                     glusterfsd_msg_3);
+    }
+
+    if (cmd_args->use_readdirp) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "use-readdirp",
+                     cmd_args->use_readdirp, glusterfsd_msg_3);
+    }
+    if (cmd_args->event_history) {
+        ret = dict_set_str(options, "event-history", cmd_args->event_history);
+        DICT_SET_VAL(dict_set_static_ptr, options, "event-history",
+                     cmd_args->event_history, glusterfsd_msg_3);
+    }
+    if (cmd_args->thin_client) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "thin-client", "on",
+                     glusterfsd_msg_3);
+    }
+    if (cmd_args->reader_thread_count) {
+        DICT_SET_VAL(dict_set_uint32, options, "reader-thread-count",
+                     cmd_args->reader_thread_count, glusterfsd_msg_3);
+    }
+
+    DICT_SET_VAL(dict_set_uint32, options, "auto-invalidation",
+                 cmd_args->fuse_auto_inval, glusterfsd_msg_3);
+
+    switch (cmd_args->kernel_writeback_cache) {
+        case GF_OPTION_ENABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "kernel-writeback-cache",
+                         "on", glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "kernel-writeback-cache",
+                         "off", glusterfsd_msg_3);
+            break;
+        default:
+            gf_msg_debug("glusterfsd", 0, "kernel-writeback-cache mode %d",
+                         cmd_args->kernel_writeback_cache);
+            break;
+    }
+    if (cmd_args->attr_times_granularity) {
+        DICT_SET_VAL(dict_set_uint32, options, "attr-times-granularity",
+                     cmd_args->attr_times_granularity, glusterfsd_msg_3);
+    }
+    switch (cmd_args->fuse_flush_handle_interrupt) {
+        case GF_OPTION_ENABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "flush-handle-interrupt",
+                         "on", glusterfsd_msg_3);
+            break;
+        case GF_OPTION_DISABLE:
+            DICT_SET_VAL(dict_set_static_ptr, options, "flush-handle-interrupt",
+                         "off", glusterfsd_msg_3);
+            break;
+        default:
+            gf_msg_debug("glusterfsd", 0, "fuse-flush-handle-interrupt mode %d",
+                         cmd_args->fuse_flush_handle_interrupt);
+            break;
+    }
+    if (cmd_args->global_threading) {
+        DICT_SET_VAL(dict_set_static_ptr, options, "global-threading", "on",
+                     glusterfsd_msg_3);
+    }
+    if (cmd_args->fuse_dev_eperm_ratelimit_ns) {
+        DICT_SET_VAL(dict_set_uint32, options, "fuse-dev-eperm-ratelimit-ns",
+                     cmd_args->fuse_dev_eperm_ratelimit_ns, glusterfsd_msg_3);
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
+
+int
+create_fuse_mount(glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+    xlator_t *master = NULL;
+
+    cmd_args = &ctx->cmd_args;
+    if (!cmd_args->mount_point) {
+        gf_msg_trace("glusterfsd", 0,
+                     "mount point not found, not a client process");
+        return 0;
+    }
+
+    if (ctx->process_mode != GF_CLIENT_PROCESS) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_7, NULL);
+        return -1;
+    }
+
+    master = GF_CALLOC(1, sizeof(*master), gfd_mt_xlator_t);
+    if (!master)
+        goto err;
+
+    master->name = gf_strdup("fuse");
+    if (!master->name)
+        goto err;
+
+    if (xlator_set_type(master, "mount/fuse") == -1) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_8,
+                "MOUNT-POINT=%s", cmd_args->mount_point, NULL);
+        goto err;
+    }
+
+    master->ctx = ctx;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
+
+    ret = set_fuse_mount_options(ctx, master->options);
+    if (ret)
+        goto err;
+
+    if (cmd_args->fuse_mountopts) {
+        ret = dict_set_static_ptr(master->options, ZR_FUSE_MOUNTOPTS,
+                                  cmd_args->fuse_mountopts);
+        if (ret < 0) {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_3,
+                    ZR_FUSE_MOUNTOPTS, NULL);
+            goto err;
         }
+    }
 
-        ret = xlator_init (master);
-        if (ret)
-                goto err;
+    ret = xlator_init(master);
+    if (ret) {
+        gf_msg_debug("glusterfsd", 0, "failed to initialize fuse translator");
+        goto err;
+    }
 
-        ctx->master = master;
+    ctx->master = master;
 
-        return 0;
+    return 0;
 
 err:
-        if (master) {
-                xlator_destroy (master);
-        }
+    if (master) {
+        xlator_destroy(master);
+    }
 
-        return -1;
+    return 1;
 }
 
-
 static FILE *
-get_volfp (glusterfs_ctx_t *ctx)
+get_volfp(glusterfs_ctx_t *ctx)
 {
-        int          ret = 0;
-        cmd_args_t  *cmd_args = NULL;
-        FILE        *specfp = NULL;
-        struct stat  statbuf;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-        cmd_args = &ctx->cmd_args;
-
-        ret = sys_lstat (cmd_args->volfile, &statbuf);
-        if (ret == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "%s: %s", cmd_args->volfile, strerror (errno));
-                return NULL;
-        }
+    cmd_args = &ctx->cmd_args;
 
-        if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "volume file %s: %s",
-                        cmd_args->volfile,
-                        strerror (errno));
-                return NULL;
-        }
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_9,
+                "volume_file=%s", cmd_args->volfile, NULL);
+        return NULL;
+    }
 
-        gf_log ("glusterfsd", GF_LOG_DEBUG,
-                "loading volume file %s", cmd_args->volfile);
+    gf_msg_debug("glusterfsd", 0, "loading volume file %s", cmd_args->volfile);
 
-        return specfp;
+    return specfp;
 }
 
+static int
+gf_remember_backup_volfile_server(char *arg)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+
+    ctx = glusterfsd_ctx;
+    if (!ctx)
+        goto out;
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args)
+        goto out;
+
+    ret = gf_set_volfile_server_common(
+        cmd_args, arg, GF_DEFAULT_VOLFILE_TRANSPORT, GF_DEFAULT_BASE_PORT);
+    if (ret) {
+        gf_log("glusterfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
+out:
+    return ret;
+}
 
 static int
-gf_remember_xlator_option (struct list_head *options, char *arg)
+gf_remember_xlator_option(char *arg)
 {
-        glusterfs_ctx_t         *ctx = NULL;
-        cmd_args_t              *cmd_args  = NULL;
-        xlator_cmdline_option_t *option = NULL;
-        int                      ret = -1;
-        char                    *dot = NULL;
-        char                    *equals = NULL;
-
-        ctx = glusterfs_ctx_get ();
-        cmd_args = &ctx->cmd_args;
-
-        option = GF_CALLOC (1, sizeof (xlator_cmdline_option_t),
-                            gfd_mt_xlator_cmdline_option_t);
-        if (!option)
-                goto out;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    xlator_cmdline_option_t *option = NULL;
+    int ret = -1;
+    char *dot = NULL;
+    char *equals = NULL;
 
-        INIT_LIST_HEAD (&option->cmd_args);
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
 
-        dot = strchr (arg, '.');
-        if (!dot)
-                goto out;
+    option = GF_CALLOC(1, sizeof(xlator_cmdline_option_t),
+                       gfd_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto out;
 
-        option->volume = GF_CALLOC ((dot - arg) + 1, sizeof (char),
-                                    gfd_mt_char);
-        strncpy (option->volume, arg, (dot - arg));
+    INIT_LIST_HEAD(&option->cmd_args);
 
-        equals = strchr (arg, '=');
-        if (!equals)
-                goto out;
+    dot = strchr(arg, '.');
+    if (!dot) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
 
-        option->key = GF_CALLOC ((equals - dot) + 1, sizeof (char),
-                                 gfd_mt_char);
-        if (!option->key)
-                goto out;
+    option->volume = GF_MALLOC((dot - arg) + 1, gfd_mt_char);
+    if (!option->volume)
+        goto out;
 
-        strncpy (option->key, dot + 1, (equals - dot - 1));
+    strncpy(option->volume, arg, (dot - arg));
+    option->volume[(dot - arg)] = '\0';
 
-        if (!*(equals + 1))
-                goto out;
+    equals = strchr(arg, '=');
+    if (!equals) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
 
-        option->value = gf_strdup (equals + 1);
+    option->key = GF_MALLOC((equals - dot) + 1, gfd_mt_char);
+    if (!option->key)
+        goto out;
 
-        list_add (&option->cmd_args, &cmd_args->xlator_options);
+    strncpy(option->key, dot + 1, (equals - dot - 1));
+    option->key[(equals - dot - 1)] = '\0';
 
-        ret = 0;
+    if (!*(equals + 1)) {
+        gf_smsg("", GF_LOG_WARNING, 0, glusterfsd_msg_10, "arg=%s", arg, NULL);
+        goto out;
+    }
+
+    option->value = gf_strdup(equals + 1);
+
+    list_add(&option->cmd_args, &cmd_args->xlator_options);
+
+    ret = 0;
 out:
-        if (ret == -1) {
-                if (option) {
-                        if (option->volume)
-                                GF_FREE (option->volume);
-                        if (option->key)
-                                GF_FREE (option->key);
-                        if (option->value)
-                                GF_FREE (option->value);
-
-                        GF_FREE (option);
-                }
+    if (ret == -1) {
+        if (option) {
+            GF_FREE(option->volume);
+            GF_FREE(option->key);
+            GF_FREE(option->value);
+
+            GF_FREE(option);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
+#ifdef GF_LINUX_HOST_OS
+static struct oom_api_info {
+    char *oom_api_file;
+    int32_t oom_min;
+    int32_t oom_max;
+} oom_api_info[] = {
+    {"/proc/self/oom_score_adj", OOM_SCORE_ADJ_MIN, OOM_SCORE_ADJ_MAX},
+    {"/proc/self/oom_adj", OOM_DISABLE, OOM_ADJUST_MAX},
+    {NULL, 0, 0}};
+
+static struct oom_api_info *
+get_oom_api_info(void)
+{
+    struct oom_api_info *api = NULL;
+
+    for (api = oom_api_info; api->oom_api_file; api++) {
+        if (sys_access(api->oom_api_file, F_OK) != -1) {
+            return api;
+        }
+    }
 
+    return NULL;
+}
+#endif
 
 static error_t
-parse_opts (int key, char *arg, struct argp_state *state)
+parse_opts(int key, char *arg, struct argp_state *state)
 {
-        cmd_args_t   *cmd_args      = NULL;
-        uint32_t      n             = 0;
-        double        d             = 0.0;
-        gf_boolean_t  b             = _gf_false;
-        char         *pwd           = NULL;
-        char          tmp_buf[2048] = {0,};
+    cmd_args_t *cmd_args = NULL;
+    uint32_t n = 0;
+#ifdef GF_LINUX_HOST_OS
+    int32_t k = 0;
+    struct oom_api_info *api = NULL;
+#endif
+    double d = 0.0;
+    gf_boolean_t b = _gf_false;
+    char *pwd = NULL;
+    char *tmp_str = NULL;
+    char *port_str = NULL;
+    struct passwd *pw = NULL;
+    int ret = 0;
 
-        cmd_args = state->input;
+    cmd_args = state->input;
 
-        switch (key) {
+    switch (key) {
         case ARGP_VOLFILE_SERVER_KEY:
-                cmd_args->volfile_server = gf_strdup (arg);
-                break;
+            gf_remember_backup_volfile_server(arg);
 
-        case ARGP_VOLFILE_MAX_FETCH_ATTEMPTS:
-                n = 0;
+            break;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->max_connect_attempts = n;
-                        break;
-                }
+        case ARGP_READ_ONLY_KEY:
+            cmd_args->read_only = 1;
+            break;
 
-                argp_failure (state, -1, 0,
-                              "Invalid limit on connect attempts %s", arg);
-                break;
+        case ARGP_ACL_KEY:
+            cmd_args->acl = 1;
+            gf_remember_xlator_option("*-md-cache.cache-posix-acl=true");
+            break;
 
-        case ARGP_READ_ONLY_KEY:
-                cmd_args->read_only = 1;
-                break;
+        case ARGP_SELINUX_KEY:
+            cmd_args->selinux = 1;
+            gf_remember_xlator_option("*-md-cache.cache-selinux=true");
+            break;
 
-        case ARGP_MAC_COMPAT_KEY:
-                if (!arg)
-                        arg = "on";
+        case ARGP_CAPABILITY_KEY:
+            cmd_args->capability = 1;
+            break;
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->mac_compat = b;
+        case ARGP_AUX_GFID_MOUNT_KEY:
+            cmd_args->aux_gfid_mount = 1;
+            break;
 
-                        break;
-                }
+        case ARGP_INODE32_KEY:
+            cmd_args->enable_ino32 = 1;
+            break;
 
-                argp_failure (state, -1, 0,
-                              "invalid value \"%s\" for mac-compat", arg);
-                break;
+        case ARGP_WORM_KEY:
+            cmd_args->worm = 1;
+            break;
 
-        case ARGP_VOLUME_FILE_KEY:
-                if (cmd_args->volfile)
-                        GF_FREE (cmd_args->volfile);
-
-                if (arg[0] != '/') {
-                        pwd = getcwd (NULL, 0);
-                        snprintf (tmp_buf, 1024, "%s/%s", pwd, arg);
-                        cmd_args->volfile = gf_strdup (tmp_buf);
-                        free (pwd);
-                } else {
-                        cmd_args->volfile = gf_strdup (arg);
-                }
+        case ARGP_PRINT_NETGROUPS:
+            cmd_args->print_netgroups = arg;
+            break;
 
-                break;
+        case ARGP_PRINT_EXPORTS:
+            cmd_args->print_exports = arg;
+            break;
 
-        case ARGP_LOG_SERVER_KEY:
-                if (cmd_args->log_server)
-                        GF_FREE (cmd_args->log_server);
+        case ARGP_PRINT_XLATORDIR_KEY:
+            cmd_args->print_xlatordir = _gf_true;
+            break;
 
-                cmd_args->log_server = gf_strdup (arg);
-                break;
+        case ARGP_PRINT_STATEDUMPDIR_KEY:
+            cmd_args->print_statedumpdir = _gf_true;
+            break;
 
-        case ARGP_LOG_LEVEL_KEY:
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_NONE_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_NONE;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_CRITICAL_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_CRITICAL;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_ERROR_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_ERROR;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_WARNING_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_WARNING;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_NORMAL_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_NORMAL;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_DEBUG_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_DEBUG;
-                        break;
-                }
-                if (strcasecmp (arg, ARGP_LOG_LEVEL_TRACE_OPTION) == 0) {
-                        cmd_args->log_level = GF_LOG_TRACE;
-                        break;
-                }
+        case ARGP_PRINT_LOGDIR_KEY:
+            cmd_args->print_logdir = _gf_true;
+            break;
 
-                argp_failure (state, -1, 0, "unknown log level %s", arg);
-                break;
+        case ARGP_PRINT_LIBEXECDIR_KEY:
+            cmd_args->print_libexecdir = _gf_true;
+            break;
+
+        case ARGP_MAC_COMPAT_KEY:
+            if (!arg)
+                arg = "on";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->mac_compat = b;
 
-        case ARGP_LOG_FILE_KEY:
-                cmd_args->log_file = gf_strdup (arg);
                 break;
+            }
 
-        case ARGP_VOLFILE_SERVER_PORT_KEY:
-                n = 0;
+            argp_failure(state, -1, 0, "invalid value \"%s\" for mac-compat",
+                         arg);
+            break;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->volfile_server_port = n;
-                        break;
+        case ARGP_VOLUME_FILE_KEY:
+            GF_FREE(cmd_args->volfile);
+
+            if (arg[0] != '/') {
+                pwd = getcwd(NULL, PATH_MAX);
+                if (!pwd) {
+                    argp_failure(state, -1, errno,
+                                 "getcwd failed with error no %d", errno);
+                    break;
                 }
+                char tmp_buf[1024];
+                snprintf(tmp_buf, sizeof(tmp_buf), "%s/%s", pwd, arg);
+                cmd_args->volfile = gf_strdup(tmp_buf);
+                free(pwd);
+            } else {
+                cmd_args->volfile = gf_strdup(arg);
+            }
+
+            break;
 
-                argp_failure (state, -1, 0,
-                              "unknown volfile server port %s", arg);
+        case ARGP_LOG_LEVEL_KEY:
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_NONE_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_NONE;
                 break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_CRITICAL_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_CRITICAL;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_ERROR_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_ERROR;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_WARNING_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_WARNING;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_INFO_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_INFO;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_DEBUG_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_DEBUG;
+                break;
+            }
+            if (strcasecmp(arg, ARGP_LOG_LEVEL_TRACE_OPTION) == 0) {
+                cmd_args->log_level = GF_LOG_TRACE;
+                break;
+            }
 
-        case ARGP_LOG_SERVER_PORT_KEY:
-                n = 0;
+            argp_failure(state, -1, 0, "unknown log level %s", arg);
+            break;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->log_server_port = n;
-                        break;
-                }
+        case ARGP_LOG_FILE_KEY:
+            cmd_args->log_file = gf_strdup(arg);
+            break;
+
+        case ARGP_VOLFILE_SERVER_PORT_KEY:
+            n = 0;
 
-                argp_failure (state, -1, 0,
-                              "unknown log server port %s", arg);
+            if (gf_string2uint_base10(arg, &n) == 0) {
+                cmd_args->volfile_server_port = n;
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown volfile server port %s", arg);
+            break;
 
         case ARGP_VOLFILE_SERVER_TRANSPORT_KEY:
-                cmd_args->volfile_server_transport = gf_strdup (arg);
-                break;
+            cmd_args->volfile_server_transport = gf_strdup(arg);
+            break;
 
         case ARGP_VOLFILE_ID_KEY:
-                cmd_args->volfile_id = gf_strdup (arg);
-                break;
+            cmd_args->volfile_id = gf_strdup(arg);
+            break;
+
+        case ARGP_THIN_CLIENT_KEY:
+            cmd_args->thin_client = _gf_true;
+            break;
+
+        case ARGP_BRICK_MUX_KEY:
+            cmd_args->brick_mux = _gf_true;
+            break;
 
         case ARGP_PID_FILE_KEY:
-                cmd_args->pid_file = gf_strdup (arg);
-                break;
+            cmd_args->pid_file = gf_strdup(arg);
+            break;
+
+        case ARGP_SOCK_FILE_KEY:
+            cmd_args->sock_file = gf_strdup(arg);
+            break;
 
         case ARGP_NO_DAEMON_KEY:
-                cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
-                break;
+            cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
+            break;
 
         case ARGP_RUN_ID_KEY:
-                cmd_args->run_id = gf_strdup (arg);
-                break;
+            cmd_args->run_id = gf_strdup(arg);
+            break;
 
         case ARGP_DEBUG_KEY:
-                cmd_args->debug_mode = ENABLE_DEBUG_MODE;
-                break;
+            cmd_args->debug_mode = ENABLE_DEBUG_MODE;
+            break;
+        case ARGP_VOLFILE_MAX_FETCH_ATTEMPTS:
+            cmd_args->max_connect_attempts = 1;
+            break;
 
         case ARGP_DIRECT_IO_MODE_KEY:
-                if (!arg)
-                        arg = "on";
+            if (!arg)
+                arg = "on";
 
-                if (gf_string2boolean (arg, &b) == 0) {
-                        cmd_args->fuse_direct_io_mode = b;
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_direct_io_mode = b;
 
-                        break;
-                }
+                break;
+            }
 
-                argp_failure (state, -1, 0,
-                              "unknown direct I/O mode setting \"%s\"", arg);
+            if (strcmp(arg, "auto") == 0)
                 break;
 
+            argp_failure(state, -1, 0, "unknown direct I/O mode setting \"%s\"",
+                         arg);
+            break;
+
+        case ARGP_FUSE_NO_ROOT_SQUASH_KEY:
+            cmd_args->no_root_squash = _gf_true;
+            break;
+
         case ARGP_ENTRY_TIMEOUT_KEY:
-                d = 0.0;
+            d = 0.0;
 
-                gf_string2double (arg, &d);
-                if (!(d < 0.0)) {
-                        cmd_args->fuse_entry_timeout = d;
-                        break;
-                }
+            gf_string2double(arg, &d);
+            if (!(d < 0.0)) {
+                cmd_args->fuse_entry_timeout = d;
+                break;
+            }
 
-                argp_failure (state, -1, 0, "unknown entry timeout %s", arg);
+            argp_failure(state, -1, 0, "unknown entry timeout %s", arg);
+            break;
+
+        case ARGP_NEGATIVE_TIMEOUT_KEY:
+            d = 0.0;
+
+            ret = gf_string2double(arg, &d);
+            if ((ret == 0) && !(d < 0.0)) {
+                cmd_args->fuse_negative_timeout = d;
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown negative timeout %s", arg);
+            break;
 
         case ARGP_ATTRIBUTE_TIMEOUT_KEY:
-                d = 0.0;
+            d = 0.0;
 
-                gf_string2double (arg, &d);
-                if (!(d < 0.0)) {
-                        cmd_args->fuse_attribute_timeout = d;
-                        break;
-                }
+            gf_string2double(arg, &d);
+            if (!(d < 0.0)) {
+                cmd_args->fuse_attribute_timeout = d;
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown attribute timeout %s", arg);
+            break;
 
-                argp_failure (state, -1, 0,
-                              "unknown attribute timeout %s", arg);
+        case ARGP_CLIENT_PID_KEY:
+            if (gf_string2int(arg, &cmd_args->client_pid) == 0) {
+                cmd_args->client_pid_set = 1;
                 break;
+            }
+
+            argp_failure(state, -1, 0, "unknown client pid %s", arg);
+            break;
+
+        case ARGP_USER_MAP_ROOT_KEY:
+            pw = getpwnam(arg);
+            if (pw)
+                cmd_args->uid_map_root = pw->pw_uid;
+            else
+                argp_failure(state, -1, 0, "user %s does not exist", arg);
+            break;
 
         case ARGP_VOLFILE_CHECK_KEY:
-                cmd_args->volfile_check = 1;
-                break;
+            cmd_args->volfile_check = 1;
+            break;
 
         case ARGP_VOLUME_NAME_KEY:
-                cmd_args->volume_name = gf_strdup (arg);
-                break;
+            cmd_args->volume_name = gf_strdup(arg);
+            break;
 
         case ARGP_XLATOR_OPTION_KEY:
-                gf_remember_xlator_option (&cmd_args->xlator_options, arg);
-                break;
+            if (gf_remember_xlator_option(arg))
+                argp_failure(state, -1, 0, "invalid xlator option  %s", arg);
+
+            break;
 
         case ARGP_KEY_NO_ARGS:
-                break;
+            break;
 
         case ARGP_KEY_ARG:
-                if (state->arg_num >= 1)
-                        argp_usage (state);
+            if (state->arg_num >= 1)
+                argp_usage(state);
+            cmd_args->mount_point = gf_strdup(arg);
+            break;
+
+        case ARGP_DUMP_FUSE_KEY:
+            cmd_args->dump_fuse = gf_strdup(arg);
+            break;
+        case ARGP_BRICK_NAME_KEY:
+            cmd_args->brick_name = gf_strdup(arg);
+            break;
+        case ARGP_BRICK_PORT_KEY:
+            n = 0;
+
+            if (arg != NULL) {
+                port_str = strtok_r(arg, ",", &tmp_str);
+                if (gf_string2uint_base10(port_str, &n) == 0) {
+                    cmd_args->brick_port = n;
+                    port_str = strtok_r(NULL, ",", &tmp_str);
+                    if (port_str) {
+                        if (gf_string2uint_base10(port_str, &n) == 0) {
+                            cmd_args->brick_port2 = n;
+                            break;
+                        }
+                        argp_failure(state, -1, 0,
+                                     "wrong brick (listen) port %s", arg);
+                    }
+                    break;
+                }
+            }
+
+            argp_failure(state, -1, 0, "unknown brick (listen) port %s", arg);
+            break;
+
+        case ARGP_MEM_ACCOUNTING_KEY:
+            /* TODO: it should have got handled much earlier */
+            // gf_mem_acct_enable_set (THIS->ctx);
+            break;
+
+        case ARGP_FOPEN_KEEP_CACHE_KEY:
+            if (!arg)
+                arg = "on";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fopen_keep_cache = b;
 
-                cmd_args->mount_point = gf_strdup (arg);
                 break;
+            }
 
-        case ARGP_DUMP_FUSE_KEY:
-                cmd_args->dump_fuse = gf_strdup (arg);
+            argp_failure(state, -1, 0, "unknown cache setting \"%s\"", arg);
+
+            break;
+
+        case ARGP_GLOBAL_TIMER_WHEEL:
+            cmd_args->global_timer_wheel = 1;
+            break;
+
+        case ARGP_GID_TIMEOUT_KEY:
+            if (!gf_string2int(arg, &cmd_args->gid_timeout)) {
+                cmd_args->gid_timeout_set = _gf_true;
                 break;
-        case ARGP_BRICK_NAME_KEY:
-                cmd_args->brick_name = gf_strdup (arg);
+            }
+
+            argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
+            break;
+
+        case ARGP_RESOLVE_GIDS_KEY:
+            cmd_args->resolve_gids = 1;
+            break;
+
+        case ARGP_FUSE_LRU_LIMIT_KEY:
+            if (!gf_string2int32(arg, &cmd_args->lru_limit))
                 break;
-        case ARGP_BRICK_PORT_KEY:
-                n = 0;
 
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->brick_port = n;
-                        break;
+            argp_failure(state, -1, 0, "unknown LRU limit option %s", arg);
+            break;
+
+        case ARGP_FUSE_INVALIDATE_LIMIT_KEY:
+            if (!gf_string2int32(arg, &cmd_args->invalidate_limit))
+                break;
+
+            argp_failure(state, -1, 0, "unknown invalidate limit option %s",
+                         arg);
+            break;
+
+        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
+            if (!gf_string2int(arg, &cmd_args->background_qlen))
+                break;
+
+            argp_failure(state, -1, 0, "unknown background qlen option %s",
+                         arg);
+            break;
+        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
+            if (!gf_string2int(arg, &cmd_args->congestion_threshold))
+                break;
+
+            argp_failure(state, -1, 0, "unknown congestion threshold option %s",
+                         arg);
+            break;
+
+#ifdef GF_LINUX_HOST_OS
+        case ARGP_OOM_SCORE_ADJ_KEY:
+            k = 0;
+
+            api = get_oom_api_info();
+            if (!api)
+                goto no_oom_api;
+
+            if (gf_string2int(arg, &k) == 0 && k >= api->oom_min &&
+                k <= api->oom_max) {
+                cmd_args->oom_score_adj = gf_strdup(arg);
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown oom_score_adj value %s", arg);
+
+        no_oom_api:
+            break;
+#endif
+
+        case ARGP_FUSE_MOUNTOPTS_KEY:
+            cmd_args->fuse_mountopts = gf_strdup(arg);
+            break;
+
+        case ARGP_FUSE_USE_READDIRP_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                if (b) {
+                    cmd_args->use_readdirp = "yes";
+                } else {
+                    cmd_args->use_readdirp = "no";
                 }
 
-                argp_failure (state, -1, 0,
-                              "unknown brick (listen) port %s", arg);
                 break;
-        }
+            }
+
+            argp_failure(state, -1, 0, "unknown use-readdirp setting \"%s\"",
+                         arg);
+            break;
+
+        case ARGP_LOGGER:
+            if (strcasecmp(arg, GF_LOGGER_GLUSTER_LOG) == 0)
+                cmd_args->logger = gf_logger_glusterlog;
+            else if (strcasecmp(arg, GF_LOGGER_SYSLOG) == 0)
+                cmd_args->logger = gf_logger_syslog;
+            else
+                argp_failure(state, -1, 0, "unknown logger %s", arg);
+
+            break;
+
+        case ARGP_LOG_FORMAT:
+            if (strcasecmp(arg, GF_LOG_FORMAT_NO_MSG_ID) == 0)
+                cmd_args->log_format = gf_logformat_traditional;
+            else if (strcasecmp(arg, GF_LOG_FORMAT_WITH_MSG_ID) == 0)
+                cmd_args->log_format = gf_logformat_withmsgid;
+            else
+                argp_failure(state, -1, 0, "unknown log format %s", arg);
+
+            break;
+
+        case ARGP_LOG_BUF_SIZE:
+            if (gf_string2uint32(arg, &cmd_args->log_buf_size)) {
+                argp_failure(state, -1, 0, "unknown log buf size option %s",
+                             arg);
+            } else if (cmd_args->log_buf_size > GF_LOG_LRU_BUFSIZE_MAX) {
+                argp_failure(state, -1, 0,
+                             "Invalid log buf size %s. "
+                             "Valid range: [" GF_LOG_LRU_BUFSIZE_MIN_STR
+                             "," GF_LOG_LRU_BUFSIZE_MAX_STR "]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_LOG_FLUSH_TIMEOUT:
+            if (gf_string2uint32(arg, &cmd_args->log_flush_timeout)) {
+                argp_failure(state, -1, 0,
+                             "unknown log flush timeout option %s", arg);
+            } else if ((cmd_args->log_flush_timeout <
+                        GF_LOG_FLUSH_TIMEOUT_MIN) ||
+                       (cmd_args->log_flush_timeout >
+                        GF_LOG_FLUSH_TIMEOUT_MAX)) {
+                argp_failure(state, -1, 0,
+                             "Invalid log flush timeout %s. "
+                             "Valid range: [" GF_LOG_FLUSH_TIMEOUT_MIN_STR
+                             "," GF_LOG_FLUSH_TIMEOUT_MAX_STR "]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_SECURE_MGMT_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->secure_mgmt = b ? 1 : 0;
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown secure-mgmt setting \"%s\"",
+                         arg);
+            break;
+
+        case ARGP_LOCALTIME_LOGGING_KEY:
+            cmd_args->localtime_logging = 1;
+            break;
+        case ARGP_PROCESS_NAME_KEY:
+            cmd_args->process_name = gf_strdup(arg);
+            break;
+        case ARGP_SUBDIR_MOUNT_KEY:
+            if (arg[0] != '/') {
+                argp_failure(state, -1, 0, "expect '/%s', provided just \"%s\"",
+                             arg, arg);
+                break;
+            }
+            cmd_args->subdir_mount = gf_strdup(arg);
+            break;
+        case ARGP_FUSE_EVENT_HISTORY_KEY:
+            if (!arg)
+                arg = "no";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                if (b) {
+                    cmd_args->event_history = "yes";
+                } else {
+                    cmd_args->event_history = "no";
+                }
 
-        return 0;
+                break;
+            }
+
+            argp_failure(state, -1, 0, "unknown event-history setting \"%s\"",
+                         arg);
+            break;
+        case ARGP_READER_THREAD_COUNT_KEY:
+            if (gf_string2uint32(arg, &cmd_args->reader_thread_count)) {
+                argp_failure(state, -1, 0,
+                             "unknown reader thread count option %s", arg);
+            } else if ((cmd_args->reader_thread_count < 1) ||
+                       (cmd_args->reader_thread_count > 64)) {
+                argp_failure(state, -1, 0,
+                             "Invalid reader thread count %s. "
+                             "Valid range: [\"1, 64\"]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_KERNEL_WRITEBACK_CACHE_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->kernel_writeback_cache = b;
+
+                break;
+            }
+
+            argp_failure(state, -1, 0,
+                         "unknown kernel writeback cache setting \"%s\"", arg);
+            break;
+        case ARGP_ATTR_TIMES_GRANULARITY_KEY:
+            if (gf_string2uint32(arg, &cmd_args->attr_times_granularity)) {
+                argp_failure(state, -1, 0,
+                             "unknown attribute times granularity option %s",
+                             arg);
+            } else if (cmd_args->attr_times_granularity > 1000000000) {
+                argp_failure(state, -1, 0,
+                             "Invalid attribute times granularity value %s. "
+                             "Valid range: [\"0, 1000000000\"]",
+                             arg);
+            }
+
+            break;
+
+        case ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_flush_handle_interrupt = b;
+
+                break;
+            }
+
+            argp_failure(state, -1, 0,
+                         "unknown fuse flush handle interrupt setting \"%s\"",
+                         arg);
+            break;
+
+        case ARGP_FUSE_AUTO_INVAL_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_auto_inval = b;
+                break;
+            }
+
+            break;
+
+        case ARGP_GLOBAL_THREADING_KEY:
+            if (!arg || (*arg == 0)) {
+                arg = "yes";
+            }
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->global_threading = b;
+                break;
+            }
+
+            argp_failure(state, -1, 0,
+                         "Invalid value for global threading \"%s\"", arg);
+            break;
+
+        case ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY:
+            if (gf_string2uint32(arg, &cmd_args->fuse_dev_eperm_ratelimit_ns)) {
+                argp_failure(state, -1, 0,
+                             "Non-numerical value for "
+                             "'fuse-dev-eperm-ratelimit-ns' option %s",
+                             arg);
+            } else if (cmd_args->fuse_dev_eperm_ratelimit_ns > 1000000000) {
+                argp_failure(state, -1, 0,
+                             "Invalid 'fuse-dev-eperm-ratelimit-ns' value %s. "
+                             "Valid range: [\"0, 1000000000\"]",
+                             arg);
+            }
+
+            break;
+    }
+    return 0;
 }
 
+gf_boolean_t
+should_call_fini(glusterfs_ctx_t *ctx, xlator_t *trav)
+{
+    /* There's nothing to call, so the other checks don't matter. */
+    if (!trav->fini) {
+        return _gf_false;
+    }
+
+    /* This preserves previous behavior in glusterd. */
+    if (ctx->process_mode == GF_GLUSTERD_PROCESS) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
 
 void
-cleanup_and_exit (int signum)
+cleanup_and_exit(int signum)
 {
-        glusterfs_ctx_t *ctx      = NULL;
-        xlator_t        *trav     = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *trav = NULL;
+    xlator_t *top;
+    xlator_t *victim;
+    xlator_list_t **trav_p;
 
-        ctx = glusterfs_ctx_get ();
+    ctx = glusterfsd_ctx;
 
-        /* TODO: is this the right place? */
-        if (!ctx)
-                return;
-        if (ctx->cleanup_started)
-                return;
+    if (!ctx)
+        return;
 
+    /* To take or not to take the mutex here and in the other
+     * signal handler - gf_print_trace() - is the big question here.
+     *
+     * Taking mutex in signal handler would mean that if the process
+     * receives a fatal signal while another thread is holding
+     * ctx->log.log_buf_lock to perhaps log a message in _gf_msg_internal(),
+     * the offending thread hangs on the mutex lock forever without letting
+     * the process exit.
+     *
+     * On the other hand. not taking the mutex in signal handler would cause
+     * it to modify the lru_list of buffered log messages in a racy manner,
+     * corrupt the list and potentially give rise to an unending
+     * cascade of SIGSEGVs and other re-entrancy issues.
+     */
+
+    gf_log_disable_suppression_before_exit(ctx);
+
+    gf_msg_callingfn("", GF_LOG_WARNING, 0, glusterfsd_msg_32,
+                     "received signum (%d), shutting down", signum);
+
+    if (ctx->cleanup_started)
+        return;
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    {
         ctx->cleanup_started = 1;
-        glusterfs_mgmt_pmap_signout (ctx);
+
+        /* signout should be sent to all the bricks in case brick mux is enabled
+         * and multiple brick instances are attached to this process
+         */
+        if (ctx->active) {
+            top = ctx->active->first;
+            for (trav_p = &top->children; *trav_p; trav_p = &(*trav_p)->next) {
+                victim = (*trav_p)->xlator;
+                rpc_clnt_mgmt_pmap_signout(ctx, victim->name);
+            }
+        } else {
+            rpc_clnt_mgmt_pmap_signout(ctx, NULL);
+        }
+
+        /* below part is a racy code where the rpcsvc object is freed.
+         * But in another thread (epoll thread), upon poll error in the
+         * socket the transports are cleaned up where again rpcsvc object
+         * is accessed (which is already freed by the below function).
+         * Since the process is about to be killed don't execute the function
+         * below.
+         */
+        /* if (ctx->listener) { */
+        /*         (void) glusterfs_listener_stop (ctx); */
+        /* } */
 
         /* Call fini() of FUSE xlator first:
          * so there are no more requests coming and
          * 'umount' of mount point is done properly */
         trav = ctx->master;
         if (trav && trav->fini) {
-                THIS = trav;
-                trav->fini (trav);
+            THIS = trav;
+            trav->fini(trav);
         }
 
-        gf_log ("glusterfsd", GF_LOG_NORMAL, "shutting down");
-
-        glusterfs_pidfile_cleanup (ctx);
+        glusterfs_pidfile_cleanup(ctx);
 
-        exit (0);
 #if 0
-        /* TODO: Properly do cleanup_and_exit(), with synchronisations */
-        if (ctx->mgmt)
+        /* TODO: Properly do cleanup_and_exit(), with synchronization */
+        if (ctx->mgmt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&ctx->mgmt->conn);
                 rpc_clnt_unref (ctx->mgmt);
-
-        /* call fini() of each xlator */
-        trav = NULL;
-        if (ctx->active)
-                trav = ctx->active->top;
-        while (trav) {
-                if (trav->fini) {
-                        THIS = trav;
-                        trav->fini (trav);
-                }
-                trav = trav->next;
         }
 #endif
-}
 
+        trav = NULL;
+
+        /* previously we were releasing the cleanup mutex lock before the
+           process exit. As we are releasing the cleanup mutex lock, before
+           the process can exit some other thread which is blocked on
+           cleanup mutex lock is acquiring the cleanup mutex lock and
+           trying to acquire some resources which are already freed as a
+           part of cleanup. To avoid this, we are exiting the process without
+           releasing the cleanup mutex lock. This will not cause any lock
+           related issues as the process which acquired the lock is going down
+         */
+        /* NOTE: Only the least significant 8 bits i.e (signum & 255)
+           will be available to parent process on calling exit() */
+        exit(abs(signum));
+    }
+}
 
 static void
-reincarnate (int signum)
+reincarnate(int signum)
 {
-        int                 ret = 0;
-        glusterfs_ctx_t    *ctx = NULL;
-        cmd_args_t         *cmd_args = NULL;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-        ctx = glusterfs_ctx_get ();
-        cmd_args = &ctx->cmd_args;
+    ctx = glusterfsd_ctx;
+    cmd_args = &ctx->cmd_args;
 
+    gf_msg_trace("gluster", 0, "received reincarnate request (sig:HUP)");
 
-        if (cmd_args->volfile_server) {
-                gf_log ("glusterfsd", GF_LOG_NORMAL,
-                        "Fetching the volume file from server...");
-                ret = glusterfs_volfile_fetch (ctx);
-        } else {
-                gf_log ("glusterfsd", GF_LOG_NORMAL,
-                        "Reloading volfile ...");
-                ret = glusterfs_volumes_init (ctx);
-        }
+    if (cmd_args->volfile_server) {
+        gf_smsg("glusterfsd", GF_LOG_INFO, 0, glusterfsd_msg_11, NULL);
+        ret = glusterfs_volfile_fetch(ctx);
+    }
 
-        if (ret < 0)
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "volume initialization failed.");
+    /* Also, SIGHUP should do logrotate */
+    gf_log_logrotate(1);
 
-        /* Also, SIGHUP should do logroate */
-        gf_log_logrotate (1);
+    if (ret < 0)
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_12, NULL);
 
-        return;
+    return;
 }
 
-
-static char *
-generate_uuid ()
+void
+emancipate(glusterfs_ctx_t *ctx, int ret)
 {
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
-
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
-
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
-        }
-
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (tmp_str, 1024, "%s-%d-%s:%" GF_PRI_SUSECONDS,
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
+    /* break free from the parent */
+    if (ctx->daemon_pipe[1] != -1) {
+        sys_write(ctx->daemon_pipe[1], (void *)&ret, sizeof(ret));
+        sys_close(ctx->daemon_pipe[1]);
+        ctx->daemon_pipe[1] = -1;
+    }
 }
 
-#define GF_SERVER_PROCESS   0
-#define GF_CLIENT_PROCESS   1
-#define GF_GLUSTERD_PROCESS 2
-
 static uint8_t
-gf_get_process_mode (char *exec_name)
+gf_get_process_mode(char *exec_name)
 {
-        char *dup_execname = NULL, *base = NULL;
-        uint8_t ret = 0;
+    char *dup_execname = NULL, *base = NULL;
+    uint8_t ret = 0;
 
-        dup_execname = gf_strdup (exec_name);
-        base = basename (dup_execname);
+    dup_execname = gf_strdup(exec_name);
+    base = basename(dup_execname);
 
-        if (!strncmp (base, "glusterfsd", 10)) {
-                ret = GF_SERVER_PROCESS;
-        } else if (!strncmp (base, "glusterd", 8)) {
-                ret = GF_GLUSTERD_PROCESS;
-        } else {
-                ret = GF_CLIENT_PROCESS;
-        }
+    if (!strncmp(base, "glusterfsd", 10)) {
+        ret = GF_SERVER_PROCESS;
+    } else if (!strncmp(base, "glusterd", 8)) {
+        ret = GF_GLUSTERD_PROCESS;
+    } else {
+        ret = GF_CLIENT_PROCESS;
+    }
 
-        GF_FREE (dup_execname);
+    GF_FREE(dup_execname);
 
-        return ret;
+    return ret;
 }
 
-
-
 static int
-set_log_file_path (cmd_args_t *cmd_args)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        int   i = 0;
-        int   j = 0;
-        int   ret = 0;
-        int   port = 0;
-        char *tmp_ptr = NULL;
-        char  tmp_str[1024] = {0,};
-
-        if (cmd_args->mount_point) {
-                j = 0;
-                i = 0;
-                if (cmd_args->mount_point[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->mount_point); i++,j++) {
-                        tmp_str[j] = cmd_args->mount_point[i];
-                        if (cmd_args->mount_point[i] == '/')
-                                tmp_str[j] = '-';
-                }
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    int ret = -1;
 
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   tmp_str);
-                if (ret == -1) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "asprintf failed while setting up log-file");
-                }
-                goto done;
-        }
+    if (!ctx)
+        return ret;
 
-        if (cmd_args->volfile) {
-                j = 0;
-                i = 0;
-                if (cmd_args->volfile[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->volfile); i++,j++) {
-                        tmp_str[j] = cmd_args->volfile[i];
-                        if (cmd_args->volfile[i] == '/')
-                                tmp_str[j] = '-';
-                }
-                ret = gf_asprintf (&cmd_args->log_file,
-                                DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                tmp_str);
-                if (ret == -1) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "asprintf failed while setting up log-file");
-                }
-                goto done;
-        }
+    ret = xlator_mem_acct_init(THIS, gfd_mt_end);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_CRITICAL, 0, glusterfsd_msg_34, NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+    ret = -1;
+
+    /* monitoring should be enabled by default */
+    ctx->measure_latency = true;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_13, NULL);
+        goto out;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "iobuf", NULL);
+        goto out;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "event", NULL);
+        goto out;
+    }
+
+    ctx->pool = GF_CALLOC(1, sizeof(call_pool_t), gfd_mt_call_pool_t);
+    if (!ctx->pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "call", NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&ctx->pool->all_frames);
+    LOCK_INIT(&ctx->pool->lock);
+
+    /* frame_mem_pool size 112 * 4k */
+    ctx->pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!ctx->pool->frame_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "frame", NULL);
+        goto out;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    ctx->pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!ctx->pool->stack_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stack", NULL);
+        goto out;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        gf_smsg("", GF_LOG_CRITICAL, 0, glusterfsd_msg_14, "stub", NULL);
+        goto out;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto out;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto out;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto out;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto out;
+
+    pthread_mutex_init(&ctx->notify_lock, NULL);
+    pthread_mutex_init(&ctx->cleanup_lock, NULL);
+    pthread_cond_init(&ctx->notify_cond, NULL);
+
+    ctx->clienttable = gf_clienttable_alloc();
+    if (!ctx->clienttable)
+        goto out;
+
+    cmd_args = &ctx->cmd_args;
+
+    /* parsing command line arguments */
+    cmd_args->log_level = DEFAULT_LOG_LEVEL;
+    cmd_args->logger = gf_logger_glusterlog;
+    cmd_args->log_format = gf_logformat_withmsgid;
+    cmd_args->log_buf_size = GF_LOG_LRU_BUFSIZE_DEFAULT;
+    cmd_args->log_flush_timeout = GF_LOG_FLUSH_TIMEOUT_DEFAULT;
+
+    cmd_args->mac_compat = GF_OPTION_DISABLE;
+#ifdef GF_DARWIN_HOST_OS
+    /* On Darwin machines, O_APPEND is not handled,
+     * which may corrupt the data
+     */
+    cmd_args->fuse_direct_io_mode = GF_OPTION_DISABLE;
+#else
+    cmd_args->fuse_direct_io_mode = GF_OPTION_DEFERRED;
+#endif
+    cmd_args->fuse_attribute_timeout = -1;
+    cmd_args->fuse_entry_timeout = -1;
+    cmd_args->fopen_keep_cache = GF_OPTION_DEFERRED;
+    cmd_args->kernel_writeback_cache = GF_OPTION_DEFERRED;
+    cmd_args->fuse_flush_handle_interrupt = GF_OPTION_DEFERRED;
+
+    if (ctx->mem_acct_enable)
+        cmd_args->mem_acct = 1;
+
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
+    INIT_LIST_HEAD(&cmd_args->volfile_servers);
+    ctx->pxl_count = 0;
+    pthread_mutex_init(&ctx->fd_lock, NULL);
+    pthread_cond_init(&ctx->fd_cond, NULL);
+    INIT_LIST_HEAD(&ctx->janitor_fds);
+
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
+
+    ret = 0;
+out:
 
-        if (cmd_args->volfile_server) {
-                port = 1;
-                tmp_ptr = "default";
-
-                if (cmd_args->volfile_server_port)
-                        port = cmd_args->volfile_server_port;
-                if (cmd_args->volfile_id)
-                        tmp_ptr = cmd_args->volfile_id;
-
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s-%s-%d.log",
-                                   cmd_args->volfile_server, tmp_ptr, port);
-                if (-1 == ret) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "asprintf failed while setting up log-file");
-                }
+    if (ret) {
+        if (ctx->pool) {
+            mem_pool_destroy(ctx->pool->frame_mem_pool);
+            mem_pool_destroy(ctx->pool->stack_mem_pool);
         }
-done:
-        return ret;
+        GF_FREE(ctx->pool);
+        mem_pool_destroy(ctx->stub_mem_pool);
+        mem_pool_destroy(ctx->dict_pool);
+        mem_pool_destroy(ctx->dict_data_pool);
+        mem_pool_destroy(ctx->dict_pair_pool);
+        mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+logging_init(glusterfs_ctx_t *ctx, const char *progpath)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-
-        xlator_mem_acct_init (THIS, gfd_mt_end);
-
-        ctx->process_uuid = generate_uuid ();
-        if (!ctx->process_uuid)
-                return -1;
-
-        ctx->page_size  = 128 * GF_UNIT_KB;
-
-        ctx->iobuf_pool = iobuf_pool_new (8 * GF_UNIT_MB, ctx->page_size);
-        if (!ctx->iobuf_pool)
-                return -1;
-
-        ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
-        if (!ctx->event_pool)
-                return -1;
-
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          gfd_mt_call_pool_t);
-        if (!pool)
-                return -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-        /* frame_mem_pool size 112 * 16k */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384);
+    cmd_args = &ctx->cmd_args;
 
-        if (!pool->frame_mem_pool)
-                return -1;
+    if (cmd_args->log_file == NULL) {
+        ret = gf_set_log_file_path(cmd_args, ctx);
+        if (ret == -1) {
+            fprintf(stderr,
+                    "ERROR: failed to set the log file "
+                    "path\n");
+            return -1;
+        }
+    }
 
-        /* stack_mem_pool size 256 * 8k */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192);
+    if (cmd_args->log_ident == NULL) {
+        ret = gf_set_log_ident(cmd_args);
+        if (ret == -1) {
+            fprintf(stderr,
+                    "ERROR: failed to set the log "
+                    "identity\n");
+            return -1;
+        }
+    }
 
-        if (!pool->stack_mem_pool)
-                return -1;
+    /* finish log set parameters before init */
+    gf_log_set_loglevel(ctx, cmd_args->log_level);
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-        if (!ctx->stub_mem_pool)
-                return -1;
+    gf_log_set_localtime(cmd_args->localtime_logging);
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+    gf_log_set_logger(cmd_args->logger);
 
-        pthread_mutex_init (&(ctx->lock), NULL);
+    gf_log_set_logformat(cmd_args->log_format);
 
-        cmd_args = &ctx->cmd_args;
+    gf_log_set_log_buf_size(cmd_args->log_buf_size);
 
-        /* parsing command line arguments */
-        cmd_args->log_level = DEFAULT_LOG_LEVEL;
-#ifdef GF_DARWIN_HOST_OS
-        cmd_args->mac_compat = GF_OPTION_DEFERRED;
-        /* On Darwin machines, O_APPEND is not handled,
-         * which may corrupt the data
-         */
-        cmd_args->fuse_direct_io_mode = GF_OPTION_DISABLE;
-#else
-        cmd_args->mac_compat = GF_OPTION_DISABLE;
-        cmd_args->fuse_direct_io_mode = GF_OPTION_DEFERRED;
-#endif
-        cmd_args->fuse_attribute_timeout = -1;
-        cmd_args->fuse_entry_timeout = -1;
+    gf_log_set_log_flush_timeout(cmd_args->log_flush_timeout);
 
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    if (gf_log_init(ctx, cmd_args->log_file, cmd_args->log_ident) == -1) {
+        fprintf(stderr, "ERROR: failed to open logfile %s\n",
+                cmd_args->log_file);
+        return -1;
+    }
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    /* At this point, all the logging related parameters are initialised
+     * except for the log flush timer, which will be injected post fork(2)
+     * in daemonize() . During this time, any log message that is logged
+     * will be kept buffered. And if the list that holds these messages
+     * overflows, then the same lru policy is used to drive out the least
+     * recently used message and displace it with the message just logged.
+     */
 
-        return 0;
+    return 0;
 }
 
-
-static int
-logging_init (glusterfs_ctx_t *ctx)
+void
+gf_check_and_set_mem_acct(int argc, char *argv[])
 {
-        cmd_args_t *cmd_args = NULL;
-        int         ret = 0;
-
-        cmd_args = &ctx->cmd_args;
+    int i = 0;
 
-        if (cmd_args->log_file == NULL) {
-                ret = set_log_file_path (cmd_args);
-                if (ret == -1) {
-                        fprintf (stderr, "failed to set the log file path.. "
-                                 "exiting\n");
-                        return -1;
-                }
-        }
-
-        if (gf_log_init (cmd_args->log_file) == -1) {
-                fprintf (stderr,
-                         "failed to open logfile %s.  exiting\n",
-                         cmd_args->log_file);
-                return -1;
+    for (i = 0; i < argc; i++) {
+        if (strcmp(argv[i], "--no-mem-accounting") == 0) {
+            gf_global_mem_acct_enable_set(0);
+            break;
         }
+    }
+}
 
-        gf_log_set_loglevel (cmd_args->log_level);
+/**
+ * print_exports_file - Print out & verify the syntax
+ *                      of the exports file specified
+ *                      in the parameter.
+ *
+ * @exports_file : Path of the exports file to print & verify
+ *
+ * @return : success: 0 when successfully parsed
+ *           failure: 1 when failed to parse one or more lines
+ *                   -1 when other critical errors (dlopen () etc)
+ * Critical errors are treated differently than parse errors. Critical
+ * errors terminate the program immediately here and print out different
+ * error messages. Hence there are different return values.
+ */
+int
+print_exports_file(const char *exports_file)
+{
+    void *libhandle = NULL;
+    char *libpathfull = NULL;
+    struct exports_file *file = NULL;
+    int ret = 0;
+
+    int (*exp_file_parse)(const char *filepath, struct exports_file **expfile,
+                          struct mount3_state *ms) = NULL;
+    void (*exp_file_print)(const struct exports_file *file) = NULL;
+    void (*exp_file_deinit)(struct exports_file * ptr) = NULL;
+
+    /* XLATORDIR passed through a -D flag to GCC */
+    ret = gf_asprintf(&libpathfull, "%s/%s/server.so", XLATORDIR, "nfs");
+    if (ret < 0) {
+        gf_log("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the library */
+    libhandle = dlopen(libpathfull, RTLD_NOW);
+    if (!libhandle) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error loading NFS server library : "
+               "%s\n",
+               dlerror());
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the function */
+    exp_file_parse = dlsym(libhandle, "exp_file_parse");
+    if (!exp_file_parse) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_parse "
+               "in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Parse the file */
+    ret = exp_file_parse(exports_file, &file, NULL);
+    if (ret < 0) {
+        ret = 1; /* This means we failed to parse */
+        goto out;
+    }
+
+    /* Load up the function */
+    exp_file_print = dlsym(libhandle, "exp_file_print");
+    if (!exp_file_print) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_print in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Print it out to screen */
+    exp_file_print(file);
+
+    /* Load up the function */
+    exp_file_deinit = dlsym(libhandle, "exp_file_deinit");
+    if (!exp_file_deinit) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function exp_file_deinit in lib.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Free the file */
+    exp_file_deinit(file);
 
-        return 0;
+out:
+    if (libhandle)
+        dlclose(libhandle);
+    GF_FREE(libpathfull);
+    return ret;
 }
 
+/**
+ * print_netgroups_file - Print out & verify the syntax
+ *                        of the netgroups file specified
+ *                        in the parameter.
+ *
+ * @netgroups_file : Path of the netgroups file to print & verify
+ * @return : success: 0 when successfully parsed
+ *           failure: 1 when failed to parse one more more lines
+ *                   -1 when other critical errors (dlopen () etc)
+ *
+ * We have multiple returns here because for critical errors, we abort
+ * operations immediately and exit. For example, if we can't load the
+ * NFS server library, then we have a real bad problem so we don't continue.
+ * Or if we cannot allocate anymore memory, we don't want to continue. Also,
+ * we want to print out a different error messages based on the ret value.
+ */
+int
+print_netgroups_file(const char *netgroups_file)
+{
+    void *libhandle = NULL;
+    char *libpathfull = NULL;
+    struct netgroups_file *file = NULL;
+    int ret = 0;
+
+    struct netgroups_file *(*ng_file_parse)(const char *file_path) = NULL;
+    void (*ng_file_print)(const struct netgroups_file *file) = NULL;
+    void (*ng_file_deinit)(struct netgroups_file * ptr) = NULL;
+
+    /* XLATORDIR passed through a -D flag to GCC */
+    ret = gf_asprintf(&libpathfull, "%s/%s/server.so", XLATORDIR, "nfs");
+    if (ret < 0) {
+        gf_log("glusterfs", GF_LOG_CRITICAL, "asprintf () failed.");
+        ret = -1;
+        goto out;
+    }
+    /* Load up the library */
+    libhandle = dlopen(libpathfull, RTLD_NOW);
+    if (!libhandle) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error loading NFS server library : %s\n", dlerror());
+        ret = -1;
+        goto out;
+    }
+
+    /* Load up the function */
+    ng_file_parse = dlsym(libhandle, "ng_file_parse");
+    if (!ng_file_parse) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_parse in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Parse the file */
+    file = ng_file_parse(netgroups_file);
+    if (!file) {
+        ret = 1; /* This means we failed to parse */
+        goto out;
+    }
+
+    /* Load up the function */
+    ng_file_print = dlsym(libhandle, "ng_file_print");
+    if (!ng_file_print) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_print in symbol.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Print it out to screen */
+    ng_file_print(file);
+
+    /* Load up the function */
+    ng_file_deinit = dlsym(libhandle, "ng_file_deinit");
+    if (!ng_file_deinit) {
+        gf_log("glusterfs", GF_LOG_CRITICAL,
+               "Error finding function ng_file_deinit in lib.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Free the file */
+    ng_file_deinit(file);
+
+out:
+    if (libhandle)
+        dlclose(libhandle);
+    GF_FREE(libpathfull);
+    return ret;
+}
 
 int
-parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx)
+parse_cmdline(int argc, char *argv[], glusterfs_ctx_t *ctx)
 {
-        int               process_mode = 0;
-        int               ret = 0;
-        struct stat       stbuf = {0, };
-        struct tm        *tm = NULL;
-        time_t            utime;
-        char              timestr[256];
-        char              tmp_logfile[1024] = { 0 };
-        char              *tmp_logfile_dyn = NULL;
-        char              *tmp_logfilebase = NULL;
-        cmd_args_t        *cmd_args = NULL;
-
-        cmd_args = &ctx->cmd_args;
-
-        argp_parse (&argp, argc, argv, ARGP_IN_ORDER, NULL, cmd_args);
-
-        if (ENABLE_DEBUG_MODE == cmd_args->debug_mode) {
-                cmd_args->log_level = GF_LOG_DEBUG;
-                cmd_args->log_file = "/dev/stderr";
-                cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
+    int process_mode = 0;
+    int ret = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE];
+    char tmp_logfile[1024] = {0};
+    char *tmp_logfile_dyn = NULL;
+    char *tmp_logfilebase = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int len = 0;
+    char *thin_volfileid = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    /* Do this before argp_parse so it can be overridden. */
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        cmd_args->secure_mgmt = 1;
+        ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    /* Need to set lru_limit to below 0 to indicate there was nothing
+       specified. This is needed as 0 is a valid option, and may not be
+       default value. */
+    cmd_args->lru_limit = -1;
+
+    argp_parse(&argp, argc, argv, ARGP_IN_ORDER, NULL, cmd_args);
+
+    if (cmd_args->print_xlatordir || cmd_args->print_statedumpdir ||
+        cmd_args->print_logdir || cmd_args->print_libexecdir) {
+        /* Just print, nothing else to do */
+        goto out;
+    }
+
+    if (cmd_args->print_netgroups) {
+        /* When this option is set we don't want to do anything else
+         * except for printing & verifying the netgroups file.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    if (cmd_args->print_exports) {
+        /* When this option is set we don't want to do anything else
+         * except for printing & verifying the exports file.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    ctx->secure_mgmt = cmd_args->secure_mgmt;
+
+    if (ENABLE_DEBUG_MODE == cmd_args->debug_mode) {
+        cmd_args->log_level = GF_LOG_DEBUG;
+        cmd_args->log_file = gf_strdup("/dev/stderr");
+        cmd_args->no_daemon_mode = ENABLE_NO_DAEMON_MODE;
+    }
+
+    process_mode = gf_get_process_mode(argv[0]);
+    ctx->process_mode = process_mode;
+
+    if (cmd_args->process_name) {
+        ctx->cmd_args.process_name = cmd_args->process_name;
+    }
+    /* Make sure after the parsing cli, if '--volfile-server' option is
+       given, then '--volfile-id' is mandatory */
+    if (cmd_args->volfile_server && !cmd_args->volfile_id) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_15, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd_args->volfile_server == NULL) && (cmd_args->volfile == NULL)) {
+        if (process_mode == GF_SERVER_PROCESS)
+            cmd_args->volfile = gf_strdup(DEFAULT_SERVER_VOLFILE);
+        else if (process_mode == GF_GLUSTERD_PROCESS)
+            cmd_args->volfile = gf_strdup(DEFAULT_GLUSTERD_VOLFILE);
+        else
+            cmd_args->volfile = gf_strdup(DEFAULT_CLIENT_VOLFILE);
+
+        /* Check if the volfile exists, if not give usage output
+           and exit */
+        ret = sys_stat(cmd_args->volfile, &stbuf);
+        if (ret) {
+            gf_smsg("glusterfs", GF_LOG_CRITICAL, errno, glusterfsd_msg_16,
+                    NULL);
+            /* argp_usage (argp.) */
+            fprintf(stderr, "USAGE: %s [options] [mountpoint]\n", argv[0]);
+            goto out;
         }
+    }
+
+    if (cmd_args->thin_client) {
+        len = strlen(cmd_args->volfile_id) + SLEN("gfproxy-client/");
+        thin_volfileid = GF_MALLOC(len + 1, gf_common_mt_char);
+        snprintf(thin_volfileid, len + 1, "gfproxy-client/%s",
+                 cmd_args->volfile_id);
+        GF_FREE(cmd_args->volfile_id);
+        cmd_args->volfile_id = thin_volfileid;
+    }
+
+    if (cmd_args->run_id) {
+        ret = sys_lstat(cmd_args->log_file, &stbuf);
+        /* If its /dev/null, or /dev/stdout, /dev/stderr,
+         * let it use the same, no need to alter
+         */
+        if (((ret == 0) &&
+             (S_ISREG(stbuf.st_mode) || S_ISLNK(stbuf.st_mode))) ||
+            (ret == -1)) {
+            /* Have separate logfile per run. */
+            gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT);
+            sprintf(tmp_logfile, "%s.%s.%d", cmd_args->log_file, timestr,
+                    getpid());
+
+            /* Create symlink to actual log file */
+            sys_unlink(cmd_args->log_file);
+
+            tmp_logfile_dyn = gf_strdup(tmp_logfile);
+            tmp_logfilebase = basename(tmp_logfile_dyn);
+            ret = sys_symlink(tmp_logfilebase, cmd_args->log_file);
+            if (ret == -1) {
+                fprintf(stderr, "ERROR: symlink of logfile failed\n");
+                goto out;
+            }
 
-        process_mode = gf_get_process_mode (argv[0]);
+            GF_FREE(cmd_args->log_file);
+            cmd_args->log_file = gf_strdup(tmp_logfile);
 
-        if ((cmd_args->volfile_server == NULL)
-            && (cmd_args->volfile == NULL)) {
-                if (process_mode == GF_SERVER_PROCESS)
-                        cmd_args->volfile = gf_strdup (DEFAULT_SERVER_VOLFILE);
-                else if (process_mode == GF_GLUSTERD_PROCESS)
-                        cmd_args->volfile = gf_strdup (DEFAULT_GLUSTERD_VOLFILE);
-                else
-                        cmd_args->volfile = gf_strdup (DEFAULT_CLIENT_VOLFILE);
+            GF_FREE(tmp_logfile_dyn);
         }
+    }
 
-        if (cmd_args->run_id) {
-                ret = sys_lstat (cmd_args->log_file, &stbuf);
-                /* If its /dev/null, or /dev/stdout, /dev/stderr,
-                 * let it use the same, no need to alter
-                 */
-                if (((ret == 0) &&
-                     (S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode))) ||
-                    (ret == -1)) {
-                        /* Have seperate logfile per run */
-                        tm = localtime (&utime);
-                        strftime (timestr, 256, "%Y%m%d.%H%M%S", tm);
-                        sprintf (tmp_logfile, "%s.%s.%d",
-                                 cmd_args->log_file, timestr, getpid ());
-
-                        /* Create symlink to actual log file */
-                        sys_unlink (cmd_args->log_file);
-
-                        tmp_logfile_dyn = gf_strdup (tmp_logfile);
-                        tmp_logfilebase = basename (tmp_logfile_dyn);
-                        ret = sys_symlink (tmp_logfilebase,
-                                           cmd_args->log_file);
-                        if (ret == -1) {
-                                fprintf (stderr, "symlink of logfile failed");
-                        } else {
-                                GF_FREE (cmd_args->log_file);
-                                cmd_args->log_file = gf_strdup (tmp_logfile);
-                        }
+    /*
+       This option was made obsolete but parsing it for backward
+       compatibility with third party applications
+     */
+    if (cmd_args->max_connect_attempts) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, 0, glusterfsd_msg_33, NULL);
+    }
 
-                        GF_FREE (tmp_logfile_dyn);
-                }
-        }
+#ifdef GF_DARWIN_HOST_OS
+    if (cmd_args->mount_point)
+        cmd_args->mac_compat = GF_OPTION_DEFERRED;
+#endif
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
-
 int
-glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_setup(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t  *cmd_args = NULL;
-        int          ret = 0;
-        FILE        *pidfp = NULL;
-
-        cmd_args = &ctx->cmd_args;
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    FILE *pidfp = NULL;
 
-        if (!cmd_args->pid_file)
-                return 0;
+    cmd_args = &ctx->cmd_args;
 
-        pidfp = fopen (cmd_args->pid_file, "a+");
-        if (!pidfp) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s error (%s)",
-                        cmd_args->pid_file, strerror (errno));
-                return -1;
-        }
-
-        ret = lockf (fileno (pidfp), F_TLOCK, 0);
-        if (ret) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s lock error (%s)",
-                        cmd_args->pid_file, strerror (errno));
-                return ret;
-        }
+    if (!cmd_args->pid_file)
+        return 0;
 
-        gf_log ("glusterfsd", GF_LOG_TRACE,
-                "pidfile %s lock acquired",
-                cmd_args->pid_file);
+    pidfp = fopen(cmd_args->pid_file, "a+");
+    if (!pidfp) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_17,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        goto out;
+    }
 
-        ret = lockf (fileno (pidfp), F_ULOCK, 0);
-        if (ret) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s unlock error (%s)",
-                        cmd_args->pid_file, strerror (errno));
-                return ret;
-        }
+    ctx->pidfp = pidfp;
 
-        ctx->pidfp = pidfp;
+    ret = 0;
+out:
 
-        return 0;
+    return ret;
 }
 
-
 int
-glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_cleanup(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t  *cmd_args = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (!ctx->pidfp)
-                return 0;
+    if (!ctx->pidfp)
+        return 0;
 
-        gf_log ("glusterfsd", GF_LOG_TRACE,
-                "pidfile %s unlocking",
-                cmd_args->pid_file);
+    gf_msg_trace("glusterfsd", 0, "pidfile %s cleanup", cmd_args->pid_file);
 
-        lockf (fileno (ctx->pidfp), F_ULOCK, 0);
-        fclose (ctx->pidfp);
-        ctx->pidfp = NULL;
+    if (ctx->cmd_args.pid_file) {
+        GF_FREE(ctx->cmd_args.pid_file);
+        ctx->cmd_args.pid_file = NULL;
+    }
 
-        if (ctx->cmd_args.pid_file) {
-                unlink (ctx->cmd_args.pid_file);
-                ctx->cmd_args.pid_file = NULL;
-        }
+    lockf(fileno(ctx->pidfp), F_ULOCK, 0);
+    fclose(ctx->pidfp);
+    ctx->pidfp = NULL;
 
-        return 0;
+    return 0;
 }
 
 int
-glusterfs_pidfile_update (glusterfs_ctx_t *ctx)
+glusterfs_pidfile_update(glusterfs_ctx_t *ctx, pid_t pid)
 {
-        cmd_args_t  *cmd_args = NULL;
-        int          ret = 0;
-        FILE        *pidfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+    FILE *pidfp = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        pidfp = ctx->pidfp;
-        if (!pidfp)
-                return 0;
+    pidfp = ctx->pidfp;
+    if (!pidfp)
+        return 0;
 
-        ret = lockf (fileno (pidfp), F_TLOCK, 0);
-        if (ret) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s lock failed",
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = lockf(fileno(pidfp), F_TLOCK, 0);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_18,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = ftruncate (fileno (pidfp), 0);
-        if (ret) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s truncation failed",
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = sys_ftruncate(fileno(pidfp), 0);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_20,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = fprintf (pidfp, "%d\n", getpid ());
-        if (ret <= 0) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s write failed",
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = fprintf(pidfp, "%d\n", pid);
+    if (ret <= 0) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        ret = fflush (pidfp);
-        if (ret) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "pidfile %s write failed",
-                        cmd_args->pid_file);
-                return ret;
-        }
+    ret = fflush(pidfp);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, errno, glusterfsd_msg_21,
+                "pidfile=%s", cmd_args->pid_file, NULL);
+        return ret;
+    }
 
-        gf_log ("glusterfsd", GF_LOG_DEBUG,
-                "pidfile %s updated with pid %d",
-                cmd_args->pid_file, getpid ());
+    gf_msg_debug("glusterfsd", 0, "pidfile %s updated with pid %d",
+                 cmd_args->pid_file, pid);
 
-        return 0;
+    return 0;
 }
 
-
 void *
-glusterfs_sigwaiter (void *arg)
+glusterfs_sigwaiter(void *arg)
 {
-        sigset_t  set;
-        int       ret = 0;
-        int       sig = 0;
-
-
-        sigaddset (&set, SIGINT);   /* cleanup_and_exit */
-        sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
-        sigaddset (&set, SIGHUP);   /* reincarnate */
-        sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
-
-        for (;;) {
-                ret = sigwait (&set, &sig);
-                if (ret) 
-                        continue;
-                                
-
-                switch (sig) {
-                case SIGINT:
-                case SIGTERM:
-                        cleanup_and_exit (sig);
-                        break;
-                case SIGHUP:
-                        reincarnate (sig);
-                        break;
-                case SIGUSR1:
-                        gf_proc_dump_info (sig);
-                        break;
-                case SIGUSR2:
-                        gf_latency_toggle (sig);
-                        break;
-                default:
-                        
-                        break;
-                }
-        }
+    sigset_t set;
+    int ret = 0;
+    int sig = 0;
+    char *file = NULL;
+
+    sigemptyset(&set);
+    sigaddset(&set, SIGINT);  /* cleanup_and_exit */
+    sigaddset(&set, SIGTERM); /* cleanup_and_exit */
+    sigaddset(&set, SIGHUP);  /* reincarnate */
+    sigaddset(&set, SIGUSR1); /* gf_proc_dump_info */
+    sigaddset(&set, SIGUSR2);
+
+    for (;;) {
+        ret = sigwait(&set, &sig);
+        if (ret)
+            continue;
 
-        return NULL;
-}
+        switch (sig) {
+            case SIGINT:
+            case SIGTERM:
+                cleanup_and_exit(sig);
+                break;
+            case SIGHUP:
+                reincarnate(sig);
+                break;
+            case SIGUSR1:
+                gf_proc_dump_info(sig, glusterfsd_ctx);
+                break;
+            case SIGUSR2:
+                file = gf_monitor_metrics(glusterfsd_ctx);
 
+                /* Nothing needed to be done here */
+                GF_FREE(file);
 
-int
-glusterfs_signals_setup (glusterfs_ctx_t *ctx)
-{
-        sigset_t  set;
-        int       ret = 0;
-
-        sigemptyset (&set);
-
-        /* common setting for all threads */
-        signal (SIGSEGV, gf_print_trace);
-        signal (SIGABRT, gf_print_trace);
-        signal (SIGILL, gf_print_trace);
-        signal (SIGTRAP, gf_print_trace);
-        signal (SIGFPE, gf_print_trace);
-        signal (SIGBUS, gf_print_trace);
-        signal (SIGINT, cleanup_and_exit);
-        signal (SIGPIPE, SIG_IGN);
-
-        /* block these signals from non-sigwaiter threads */
-        sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
-        sigaddset (&set, SIGHUP);   /* reincarnate */
-        sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
-
-        ret = pthread_sigmask (SIG_BLOCK, &set, NULL);
-        if (ret)
-                return ret;
+                break;
+            default:
 
-        ret = pthread_create (&ctx->sigwaiter, NULL, glusterfs_sigwaiter,
-                              (void *) &set);
-        if (ret) {
-                /*
-                  TODO:
-                  fallback to signals getting handled by other threads.
-                  setup the signal handlers
-                */
-                return ret;
+                break;
         }
+    }
 
-        return ret;
+    return NULL;
 }
 
+void
+glusterfsd_print_trace(int signum)
+{
+    gf_print_trace(signum, glusterfsd_ctx);
+}
 
 int
-daemonize (glusterfs_ctx_t *ctx)
+glusterfs_signals_setup(glusterfs_ctx_t *ctx)
 {
-        int            ret = 0;
-        cmd_args_t    *cmd_args = NULL;
+    sigset_t set;
+    int ret = 0;
+
+    sigemptyset(&set);
+
+    /* common setting for all threads */
+    signal(SIGSEGV, glusterfsd_print_trace);
+    signal(SIGABRT, glusterfsd_print_trace);
+    signal(SIGILL, glusterfsd_print_trace);
+    signal(SIGTRAP, glusterfsd_print_trace);
+    signal(SIGFPE, glusterfsd_print_trace);
+    signal(SIGBUS, glusterfsd_print_trace);
+    signal(SIGINT, cleanup_and_exit);
+    signal(SIGPIPE, SIG_IGN);
+
+    /* block these signals from non-sigwaiter threads */
+    sigaddset(&set, SIGTERM); /* cleanup_and_exit */
+    sigaddset(&set, SIGHUP);  /* reincarnate */
+    sigaddset(&set, SIGUSR1); /* gf_proc_dump_info */
+    sigaddset(&set, SIGUSR2);
+
+    /* Signals needed for asynchronous framework. */
+    sigaddset(&set, GF_ASYNC_SIGQUEUE);
+    sigaddset(&set, GF_ASYNC_SIGCTRL);
+
+    ret = pthread_sigmask(SIG_BLOCK, &set, NULL);
+    if (ret) {
+        gf_smsg("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_22, NULL);
+        return ret;
+    }
+
+    ret = gf_thread_create(&ctx->sigwaiter, NULL, glusterfs_sigwaiter,
+                           (void *)&set, "sigwait");
+    if (ret) {
+        /*
+          TODO:
+          fallback to signals getting handled by other threads.
+          setup the signal handlers
+        */
+        gf_smsg("glusterfsd", GF_LOG_WARNING, errno, glusterfsd_msg_23, NULL);
+        return ret;
+    }
 
+    return ret;
+}
 
-        cmd_args = &ctx->cmd_args;
+int
+daemonize(glusterfs_ctx_t *ctx)
+{
+    int ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int cstatus = 0;
+    int err = 1;
+    int child_pid = 0;
+
+    cmd_args = &ctx->cmd_args;
+
+    ret = glusterfs_pidfile_setup(ctx);
+    if (ret)
+        goto out;
+
+    if (cmd_args->no_daemon_mode) {
+        goto postfork;
+    }
+
+    if (cmd_args->debug_mode)
+        goto postfork;
+
+    ret = pipe(ctx->daemon_pipe);
+    if (ret) {
+        /* If pipe() fails, retain daemon_pipe[] = {-1, -1}
+           and parent will just not wait for child status
+        */
+        ctx->daemon_pipe[0] = -1;
+        ctx->daemon_pipe[1] = -1;
+    }
+
+    ret = os_daemon_return(0, 0);
+    switch (ret) {
+        case -1:
+            if (ctx->daemon_pipe[0] != -1) {
+                sys_close(ctx->daemon_pipe[0]);
+                sys_close(ctx->daemon_pipe[1]);
+            }
+
+            gf_smsg("daemonize", GF_LOG_ERROR, errno, glusterfsd_msg_24, NULL);
+            goto out;
+        case 0:
+            /* child */
+            /* close read */
+            sys_close(ctx->daemon_pipe[0]);
+            break;
+        default:
+            /* parent */
+            /* close write */
+            child_pid = ret;
+            sys_close(ctx->daemon_pipe[1]);
+
+            if (ctx->mnt_pid > 0) {
+                ret = waitpid(ctx->mnt_pid, &cstatus, 0);
+                if (!(ret == ctx->mnt_pid)) {
+                    if (WIFEXITED(cstatus)) {
+                        err = WEXITSTATUS(cstatus);
+                    } else {
+                        err = cstatus;
+                    }
+                    gf_smsg("daemonize", GF_LOG_ERROR, 0, glusterfsd_msg_25,
+                            NULL);
+                    exit(err);
+                }
+            }
+            sys_read(ctx->daemon_pipe[0], (void *)&err, sizeof(err));
+            /* NOTE: Only the least significant 8 bits i.e (err & 255)
+               will be available to parent process on calling exit() */
+            if (err)
+                _exit(abs(err));
+
+            /* Update pid in parent only for glusterd process */
+            if (ctx->process_mode == GF_GLUSTERD_PROCESS) {
+                ret = glusterfs_pidfile_update(ctx, child_pid);
+                if (ret)
+                    exit(1);
+            }
+            _exit(0);
+    }
 
-        ret = glusterfs_pidfile_setup (ctx);
+postfork:
+    /* Update pid in child either process_mode is not belong to glusterd
+       or process is spawned in no daemon mode
+    */
+    if ((ctx->process_mode != GF_GLUSTERD_PROCESS) ||
+        (cmd_args->no_daemon_mode)) {
+        ret = glusterfs_pidfile_update(ctx, getpid());
         if (ret)
-                return ret;
+            goto out;
+    }
+    gf_log("glusterfs", GF_LOG_INFO, "Pid of current running process is %d",
+           getpid());
+    ret = gf_log_inject_timer_event(ctx);
 
-        if (cmd_args->no_daemon_mode)
-                goto postfork;
+    glusterfs_signals_setup(ctx);
+out:
+    return ret;
+}
 
-        if (cmd_args->debug_mode)
-                goto postfork;
+#ifdef GF_LINUX_HOST_OS
+static int
+set_oom_score_adj(glusterfs_ctx_t *ctx)
+{
+    int ret = -1;
+    cmd_args_t *cmd_args = NULL;
+    int fd = -1;
+    size_t oom_score_len = 0;
+    struct oom_api_info *api = NULL;
 
-        ret = os_daemon (0, 0);
-        if (ret == -1) {
-                gf_log ("daemonize", GF_LOG_ERROR,
-                        "Daemonization failed: %s", strerror(errno));
-                return ret;
-        }
+    cmd_args = &ctx->cmd_args;
 
-postfork:
-        ret = glusterfs_pidfile_update (ctx);
-        if (ret)
-                return ret;
+    if (!cmd_args->oom_score_adj)
+        goto success;
 
-        glusterfs_signals_setup (ctx);
+    api = get_oom_api_info();
+    if (!api)
+        goto out;
 
-        return ret;
-}
+    fd = open(api->oom_api_file, O_WRONLY);
+    if (fd < 0)
+        goto out;
 
+    oom_score_len = strlen(cmd_args->oom_score_adj);
+    if (sys_write(fd, cmd_args->oom_score_adj, oom_score_len) !=
+        oom_score_len) {
+        sys_close(fd);
+        goto out;
+    }
+
+    if (sys_close(fd) < 0)
+        goto out;
+
+success:
+    ret = 0;
+
+out:
+    return ret;
+}
+#endif
 
 int
-glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp)
+glusterfs_process_volfp(glusterfs_ctx_t *ctx, FILE *fp)
 {
-        glusterfs_graph_t  *graph = NULL;
-        int                 ret = 0;
-        xlator_t           *trav = NULL;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
 
-        graph = glusterfs_graph_construct (fp);
+    if (!ctx)
+        return -1;
 
-        if (!graph) {
-                ret = -1;
-                goto out;
-        }
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("", GF_LOG_ERROR, 0, glusterfsd_msg_26, NULL);
+        goto out;
+    }
 
-        for (trav = graph->first; trav; trav = trav->next) {
-                if (strcmp (trav->type, "mount/fuse") == 0) {
-                        gf_log ("glusterfsd", GF_LOG_ERROR,
-                                "fuse xlator cannot be specified "
-                                "in volume file");
-                        ret = -1;
-                        goto out;
-                }
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/fuse") == 0) {
+            gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_27, NULL);
+            goto out;
         }
+    }
 
-        ret = glusterfs_graph_prepare (graph, ctx);
+    xlator_t *xl = graph->first;
+    if (xl && (strcmp(xl->type, "protocol/server") == 0)) {
+        (void)copy_opts_to_child(xl, FIRST_CHILD(xl), "*auth*");
+    }
 
-        if (ret) {
-                glusterfs_graph_destroy (graph);
-                ret = -1;
-                goto out;
-        }
+    ret = glusterfs_graph_prepare(graph, ctx, ctx->cmd_args.volume_name);
+    if (ret) {
+        goto out;
+    }
 
-        ret = glusterfs_graph_activate (graph, ctx);
+    ret = glusterfs_graph_activate(graph, ctx);
 
-        if (ret) {
-                glusterfs_graph_destroy (graph);
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        goto out;
+    }
 
-        gf_log_volume_file (fp);
+    gf_log_dump_graph(fp, graph);
 
+    ret = 0;
 out:
-        if (fp)
-                fclose (fp);
+    if (fp)
+        fclose(fp);
+
+    if (ret) {
+        /* TODO This code makes to generic for all graphs
+           client as well as servers.For now it destroys
+           graph only for server-side xlators not for client-side
+           xlators, before destroying a graph call xlator fini for
+           xlators those call xlator_init to avoid leak
+        */
+        if (graph) {
+            xl = graph->first;
+            if ((ctx->active != graph) &&
+                (xl && !strcmp(xl->type, "protocol/server"))) {
+                /* Take dict ref for every graph xlator to avoid dict leak
+                   at the time of graph destroying
+                */
+                glusterfs_graph_fini(graph);
+                glusterfs_graph_destroy(graph);
+            }
+        }
 
-        if (ret && !ctx->active) {
-                /* there is some error in setting up the first graph itself */
-                cleanup_and_exit (0);
+        /* there is some error in setting up the first graph itself */
+        if (!ctx->active) {
+            emancipate(ctx, ret);
+            cleanup_and_exit(ret);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-glusterfs_volumes_init (glusterfs_ctx_t *ctx)
+glusterfs_volumes_init(glusterfs_ctx_t *ctx)
 {
-        FILE               *fp = NULL;
-        cmd_args_t         *cmd_args = NULL;
-        int                 ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (cmd_args->volfile_server) {
-                ret = glusterfs_mgmt_init (ctx);
-                goto out;
-        }
+    if (cmd_args->sock_file) {
+        ret = glusterfs_listener_init(ctx);
+        if (ret)
+            goto out;
+    }
 
-        fp = get_volfp (ctx);
+    if (cmd_args->volfile_server) {
+        ret = glusterfs_mgmt_init(ctx);
+        /* return, do not emancipate() yet */
+        return ret;
+    }
 
-        if (!fp) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "Cannot reach volume specification file");
-                ret = -1;
-                goto out;
-        }
+    fp = get_volfp(ctx);
 
-        ret = glusterfs_process_volfp (ctx, fp);
-        if (ret)
-                goto out;
+    if (!fp) {
+        gf_smsg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_28, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterfs_process_volfp(ctx, fp);
+    if (ret)
+        goto out;
 
 out:
-        return ret;
+    emancipate(ctx, ret);
+    return ret;
 }
 
+/* This is the only legal global pointer  */
+glusterfs_ctx_t *glusterfsd_ctx;
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        int               ret = -1;
-
-        ret = glusterfs_globals_init ();
-        if (ret)
-                return ret;
-
-        ctx = glusterfs_ctx_get ();
-        if (!ctx)
-                return ENOMEM;
-
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto out;
-
-        ret = parse_cmdline (argc, argv, ctx);
-        if (ret)
-                goto out;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    char cmdlinestr[PATH_MAX] = {
+        0,
+    };
+    cmd_args_t *cmd = NULL;
+
+    gf_check_and_set_mem_acct(argc, argv);
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_29, NULL);
+        return ENOMEM;
+    }
+    glusterfsd_ctx = ctx;
+
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        return ret;
 
-        ret = logging_init (ctx);
-        if (ret)
+    THIS->ctx = ctx;
+
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
+
+    ret = parse_cmdline(argc, argv, ctx);
+    if (ret)
+        goto out;
+    cmd = &ctx->cmd_args;
+
+    if (cmd->print_xlatordir) {
+        /* XLATORDIR passed through a -D flag to GCC */
+        printf("%s\n", XLATORDIR);
+        goto out;
+    }
+
+    if (cmd->print_statedumpdir) {
+        printf("%s\n", DEFAULT_VAR_RUN_DIRECTORY);
+        goto out;
+    }
+
+    if (cmd->print_logdir) {
+        printf("%s\n", DEFAULT_LOG_FILE_DIRECTORY);
+        goto out;
+    }
+
+    if (cmd->print_libexecdir) {
+        printf("%s\n", LIBEXECDIR);
+        goto out;
+    }
+
+    if (cmd->print_netgroups) {
+        /* If this option is set we want to print & verify the file,
+         * set the return value (exit code in this case) and exit.
+         */
+        ret = print_netgroups_file(cmd->print_netgroups);
+        goto out;
+    }
+
+    if (cmd->print_exports) {
+        /* If this option is set we want to print & verify the file,
+         * set the return value (exit code in this case)
+         * and exit.
+         */
+        ret = print_exports_file(cmd->print_exports);
+        goto out;
+    }
+
+    ret = logging_init(ctx, argv[0]);
+    if (ret)
+        goto out;
+
+    /* set brick_mux mode only for server process */
+    if ((ctx->process_mode != GF_SERVER_PROCESS) && cmd->brick_mux) {
+        gf_smsg("glusterfs", GF_LOG_CRITICAL, 0, glusterfsd_msg_43, NULL);
+        goto out;
+    }
+
+    /* log the version of glusterfs running here along with the actual
+       command line options. */
+    {
+        int i = 0;
+        int pos = 0;
+        int len = snprintf(cmdlinestr, sizeof(cmdlinestr), "%s", argv[0]);
+        for (i = 1; (i < argc) && (len > 0); i++) {
+            pos += len;
+            len = snprintf(cmdlinestr + pos, sizeof(cmdlinestr) - pos, " %s",
+                           argv[i]);
+            if ((len <= 0) || (len >= (sizeof(cmdlinestr) - pos))) {
+                gf_smsg("glusterfs", GF_LOG_ERROR, 0, glusterfsd_msg_029, NULL);
+                ret = -1;
                 goto out;
+            }
+        }
+        gf_smsg(argv[0], GF_LOG_INFO, 0, glusterfsd_msg_30, "arg=%s", argv[0],
+                "version=%s", PACKAGE_VERSION, "cmdlinestr=%s", cmdlinestr,
+                NULL);
+
+        ctx->cmdlinestr = gf_strdup(cmdlinestr);
+    }
+
+    gf_proc_dump_init();
+
+    ret = create_fuse_mount(ctx);
+    if (ret)
+        goto out;
+
+    ret = daemonize(ctx);
+    if (ret)
+        goto out;
+
+    /*
+     * If we do this before daemonize, the pool-sweeper thread dies with
+     * the parent, but we want to do it as soon as possible after that in
+     * case something else depends on pool allocations.
+     */
+    mem_pools_init();
+
+    ret = gf_async_init(ctx);
+    if (ret < 0) {
+        goto out;
+    }
+
+#ifdef GF_LINUX_HOST_OS
+    ret = set_oom_score_adj(ctx);
+    if (ret)
+        goto out;
+#endif
 
-        gf_proc_dump_init();
-
-        ret = create_fuse_mount (ctx);
-        if (ret)
-                goto out;
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        gf_smsg("", GF_LOG_ERROR, 0, glusterfsd_msg_31, NULL);
+        goto out;
+    }
 
-        ret = daemonize (ctx);
-        if (ret)
-                goto out;
+    /* do this _after_ daemonize() */
+    if (!glusterfs_ctx_tw_get(ctx)) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterfs_volumes_init (ctx);
-        if (ret)
-                goto out;
+    ret = glusterfs_volumes_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = event_dispatch (ctx->event_pool);
+    ret = gf_event_dispatch(ctx->event_pool);
 
 out:
-//        glusterfs_ctx_destroy (ctx);
-
-        return ret;
+    //    glusterfs_ctx_destroy (ctx);
+    gf_async_fini();
+    return ret;
 }
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 634eb867975..4e1413caa70 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -1,85 +1,142 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __GLUSTERFSD_H__
 #define __GLUSTERFSD_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "rpcsvc.h"
+#include "glusterd1-xdr.h"
+
+#define DEFAULT_GLUSTERD_VOLFILE CONFDIR "/glusterd.vol"
+#define DEFAULT_CLIENT_VOLFILE CONFDIR "/glusterfs.vol"
+#define DEFAULT_SERVER_VOLFILE CONFDIR "/glusterfsd.vol"
 
-#include "glusterfsd-common.h"
+#define DEFAULT_EVENT_POOL_SIZE 16384
 
-#define DEFAULT_GLUSTERD_VOLFILE              CONFDIR "/glusterd.vol"
-#define DEFAULT_CLIENT_VOLFILE                CONFDIR "/glusterfs.vol"
-#define DEFAULT_SERVER_VOLFILE                CONFDIR "/glusterfsd.vol"
-#define DEFAULT_LOG_FILE_DIRECTORY            DATADIR "/log/glusterfs"
-#define DEFAULT_LOG_LEVEL                     GF_LOG_NORMAL
+#define ARGP_LOG_LEVEL_NONE_OPTION "NONE"
+#define ARGP_LOG_LEVEL_TRACE_OPTION "TRACE"
+#define ARGP_LOG_LEVEL_CRITICAL_OPTION "CRITICAL"
+#define ARGP_LOG_LEVEL_ERROR_OPTION "ERROR"
+#define ARGP_LOG_LEVEL_WARNING_OPTION "WARNING"
+#define ARGP_LOG_LEVEL_INFO_OPTION "INFO"
+#define ARGP_LOG_LEVEL_DEBUG_OPTION "DEBUG"
 
-#define DEFAULT_EVENT_POOL_SIZE            16384
+#define ENABLE_NO_DAEMON_MODE 1
+#define ENABLE_DEBUG_MODE 1
 
-#define ARGP_LOG_LEVEL_NONE_OPTION        "NONE"
-#define ARGP_LOG_LEVEL_TRACE_OPTION       "TRACE"
-#define ARGP_LOG_LEVEL_CRITICAL_OPTION    "CRITICAL"
-#define ARGP_LOG_LEVEL_ERROR_OPTION       "ERROR"
-#define ARGP_LOG_LEVEL_WARNING_OPTION     "WARNING"
-#define ARGP_LOG_LEVEL_NORMAL_OPTION      "NORMAL"
-#define ARGP_LOG_LEVEL_DEBUG_OPTION       "DEBUG"
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+/* Considering 4 key/value pairs in a dictionary on an average */
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
 
-#define ENABLE_NO_DAEMON_MODE     1
-#define ENABLE_DEBUG_MODE         1
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
 
 enum argp_option_keys {
-	ARGP_VOLFILE_SERVER_KEY = 's',
-	ARGP_VOLUME_FILE_KEY = 'f',
-	ARGP_LOG_LEVEL_KEY = 'L',
-	ARGP_LOG_FILE_KEY = 'l',
-	ARGP_VOLFILE_SERVER_PORT_KEY = 131,
-	ARGP_VOLFILE_SERVER_TRANSPORT_KEY = 132,
-	ARGP_PID_FILE_KEY = 'p',
-	ARGP_NO_DAEMON_KEY = 'N',
-	ARGP_RUN_ID_KEY = 'r',
-	ARGP_DEBUG_KEY = 133,
-	ARGP_ENTRY_TIMEOUT_KEY = 135,
-	ARGP_ATTRIBUTE_TIMEOUT_KEY = 136,
-	ARGP_VOLUME_NAME_KEY = 137,
-	ARGP_XLATOR_OPTION_KEY = 138,
-	ARGP_DIRECT_IO_MODE_KEY = 139,
+    ARGP_VOLFILE_SERVER_KEY = 's',
+    ARGP_VOLUME_FILE_KEY = 'f',
+    ARGP_LOG_LEVEL_KEY = 'L',
+    ARGP_LOG_FILE_KEY = 'l',
+    ARGP_VOLFILE_SERVER_PORT_KEY = 131,
+    ARGP_VOLFILE_SERVER_TRANSPORT_KEY = 132,
+    ARGP_PID_FILE_KEY = 'p',
+    ARGP_SOCK_FILE_KEY = 'S',
+    ARGP_NO_DAEMON_KEY = 'N',
+    ARGP_RUN_ID_KEY = 'r',
+    ARGP_PRINT_NETGROUPS = 'n',
+    ARGP_PRINT_EXPORTS = 'e',
+    ARGP_DEBUG_KEY = 133,
+    ARGP_NEGATIVE_TIMEOUT_KEY = 134,
+    ARGP_ENTRY_TIMEOUT_KEY = 135,
+    ARGP_ATTRIBUTE_TIMEOUT_KEY = 136,
+    ARGP_VOLUME_NAME_KEY = 137,
+    ARGP_XLATOR_OPTION_KEY = 138,
+    ARGP_DIRECT_IO_MODE_KEY = 139,
 #ifdef GF_DARWIN_HOST_OS
-	ARGP_NON_LOCAL_KEY = 140,
+    ARGP_NON_LOCAL_KEY = 140,
 #endif /* DARWIN */
-	ARGP_VOLFILE_ID_KEY = 143,
-        ARGP_VOLFILE_CHECK_KEY = 144,
-        ARGP_VOLFILE_MAX_FETCH_ATTEMPTS = 145,
-        ARGP_LOG_SERVER_KEY = 146,
-        ARGP_LOG_SERVER_PORT_KEY = 147,
-        ARGP_READ_ONLY_KEY = 148,
-        ARGP_MAC_COMPAT_KEY = 149,
-        ARGP_DUMP_FUSE_KEY = 150,
-        ARGP_BRICK_NAME_KEY = 151,
-        ARGP_BRICK_PORT_KEY = 152,
+    ARGP_VOLFILE_ID_KEY = 143,
+    ARGP_VOLFILE_CHECK_KEY = 144,
+    ARGP_VOLFILE_MAX_FETCH_ATTEMPTS = 145,
+    ARGP_LOG_SERVER_KEY = 146,
+    ARGP_LOG_SERVER_PORT_KEY = 147,
+    ARGP_READ_ONLY_KEY = 148,
+    ARGP_MAC_COMPAT_KEY = 149,
+    ARGP_DUMP_FUSE_KEY = 150,
+    ARGP_BRICK_NAME_KEY = 151,
+    ARGP_BRICK_PORT_KEY = 152,
+    ARGP_CLIENT_PID_KEY = 153,
+    ARGP_ACL_KEY = 154,
+    ARGP_WORM_KEY = 155,
+    ARGP_USER_MAP_ROOT_KEY = 156,
+    ARGP_MEM_ACCOUNTING_KEY = 157,
+    ARGP_SELINUX_KEY = 158,
+    ARGP_FOPEN_KEEP_CACHE_KEY = 159,
+    ARGP_GID_TIMEOUT_KEY = 160,
+    ARGP_FUSE_BACKGROUND_QLEN_KEY = 161,
+    ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
+    ARGP_INODE32_KEY = 163,
+    ARGP_FUSE_MOUNTOPTS_KEY = 164,
+    ARGP_FUSE_USE_READDIRP_KEY = 165,
+    ARGP_AUX_GFID_MOUNT_KEY = 166,
+    ARGP_FUSE_NO_ROOT_SQUASH_KEY = 167,
+    ARGP_LOGGER = 168,
+    ARGP_LOG_FORMAT = 169,
+    ARGP_LOG_BUF_SIZE = 170,
+    ARGP_LOG_FLUSH_TIMEOUT = 171,
+    ARGP_SECURE_MGMT_KEY = 172,
+    ARGP_GLOBAL_TIMER_WHEEL = 173,
+    ARGP_RESOLVE_GIDS_KEY = 174,
+    ARGP_CAPABILITY_KEY = 175,
+#ifdef GF_LINUX_HOST_OS
+    ARGP_OOM_SCORE_ADJ_KEY = 176,
+#endif
+    ARGP_LOCALTIME_LOGGING_KEY = 177,
+    ARGP_SUBDIR_MOUNT_KEY = 178,
+    ARGP_PROCESS_NAME_KEY = 179,
+    ARGP_FUSE_EVENT_HISTORY_KEY = 180,
+    ARGP_THIN_CLIENT_KEY = 181,
+    ARGP_READER_THREAD_COUNT_KEY = 182,
+    ARGP_PRINT_XLATORDIR_KEY = 183,
+    ARGP_PRINT_STATEDUMPDIR_KEY = 184,
+    ARGP_PRINT_LOGDIR_KEY = 185,
+    ARGP_KERNEL_WRITEBACK_CACHE_KEY = 186,
+    ARGP_ATTR_TIMES_GRANULARITY_KEY = 187,
+    ARGP_PRINT_LIBEXECDIR_KEY = 188,
+    ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY = 189,
+    ARGP_FUSE_LRU_LIMIT_KEY = 190,
+    ARGP_FUSE_AUTO_INVAL_KEY = 191,
+    ARGP_GLOBAL_THREADING_KEY = 192,
+    ARGP_BRICK_MUX_KEY = 193,
+    ARGP_FUSE_DEV_EPERM_RATELIMIT_NS_KEY = 194,
+    ARGP_FUSE_INVALIDATE_LIMIT_KEY = 195,
+};
+
+struct _gfd_vol_top_priv {
+    rpcsvc_request_t *req;
+    gd1_mgmt_brick_op_req xlator_req;
+    uint32_t blk_count;
+    uint32_t blk_size;
+    double throughput;
+    double time;
+    int32_t ret;
 };
+typedef struct _gfd_vol_top_priv gfd_vol_top_priv_t;
+
+int
+glusterfs_mgmt_pmap_signin(glusterfs_ctx_t *ctx);
+int
+glusterfs_volfile_fetch(glusterfs_ctx_t *ctx);
+void
+cleanup_and_exit(int signum);
 
-int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx);
-int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
-int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
-void cleanup_and_exit (int signum);
+void
+xlator_mem_cleanup(xlator_t *this);
 
+extern glusterfs_ctx_t *glusterfsd_ctx;
 #endif /* __GLUSTERFSD_H__ */
diff --git a/xlators/features/access-control/Makefile.am b/heal/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/features/access-control/Makefile.am
+++ b/heal/Makefile.am
diff --git a/heal/src/Makefile.am b/heal/src/Makefile.am
new file mode 100644
index 00000000000..aa18d3eff88
--- /dev/null
+++ b/heal/src/Makefile.am
@@ -0,0 +1,29 @@
+if WITH_SERVER
+scriptdir = $(GLUSTERFS_LIBEXECDIR)
+script_PROGRAMS = glfsheal
+endif
+
+glfsheal_SOURCES = glfs-heal.c
+
+glfsheal_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\
+		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		$(top_builddir)/api/src/libgfapi.la \
+		$(XML_LIBS) $(GFAPI_LIBS) $(UUID_LIBS)
+
+glfsheal_LDFLAGS = $(GF_LDFLAGS)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/xlators/lib/src\
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
+	-I$(top_srcdir)/rpc/xdr/src\
+	-I$(top_builddir)/rpc/xdr/src\
+	-I$(top_srcdir)/api/src\
+	-DDATADIR=\"$(localstatedir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
+
+CLEANFILES =
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/heal/src/glfs-heal.c b/heal/src/glfs-heal.c
new file mode 100644
index 00000000000..bf4b47f8760
--- /dev/null
+++ b/heal/src/glfs-heal.c
@@ -0,0 +1,1793 @@
+/*
+ Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "glfs.h"
+#include "glfs-internal.h"
+#include "protocol-common.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
+#include <string.h>
+#include <glusterfs/glusterfs.h>
+#include <libgen.h>
+
+#if (HAVE_LIB_XML)
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+
+xmlTextWriterPtr glfsh_writer;
+xmlDocPtr glfsh_doc = NULL;
+#endif
+
+#define XML_RET_CHECK_AND_GOTO(ret, label)                                     \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        } else                                                                 \
+            ret = 0;                                                           \
+    } while (0)
+
+#define MODE_XML (1 << 0)
+#define MODE_NO_LOG (1 << 1)
+
+typedef struct num_entries {
+    uint64_t num_entries;
+    uint64_t pending_entries;
+    uint64_t spb_entries;
+    uint64_t possibly_healing_entries;
+} num_entries_t;
+
+typedef int (*print_status)(dict_t *, char *, uuid_t, num_entries_t *,
+                            gf_boolean_t flag);
+
+int
+glfsh_heal_splitbrain_file(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                           char *file, dict_t *xattr_req);
+
+typedef struct glfs_info {
+    int (*init)(void);
+    int (*print_brick_from_xl)(xlator_t *xl, loc_t *rootloc);
+    int (*print_heal_op_status)(int ret, uint64_t num_entries, char *fmt_str);
+    int (*print_heal_op_summary)(int ret, num_entries_t *num_entries);
+    int (*print_heal_status)(char *path, uuid_t gfid, char *status);
+    int (*print_spb_status)(char *path, uuid_t gfid, char *status);
+    int (*end)(int op_ret, char *op_errstr);
+} glfsh_info_t;
+
+glfsh_info_t *glfsh_output = NULL;
+int32_t is_xml;
+
+#define DEFAULT_HEAL_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
+#define USAGE_STR                                                              \
+    "Usage: %s <VOLNAME> [bigger-file <FILE> | "                               \
+    "latest-mtime <FILE> | "                                                   \
+    "source-brick <HOSTNAME:BRICKNAME> [<FILE>] | "                            \
+    "split-brain-info | info-summary] [glusterd-sock <FILE>"                   \
+    "]\n"
+
+typedef enum {
+    GLFSH_MODE_CONTINUE_ON_ERROR = 1,
+    GLFSH_MODE_EXIT_ON_FIRST_FAILURE,
+} glfsh_fail_mode_t;
+
+int
+glfsh_init()
+{
+    return 0;
+}
+
+int
+glfsh_end_op_granular_entry_heal(int op_ret, char *op_errstr)
+{
+    /* If error string is available, give it higher precedence.*/
+
+    if (op_errstr) {
+        printf("%s\n", op_errstr);
+    } else if (op_ret < 0) {
+        if (op_ret == -EAGAIN)
+            printf(
+                "One or more entries need heal. Please execute "
+                "the command again after there are no entries "
+                "to be healed\n");
+        else if (op_ret == -ENOTCONN)
+            printf(
+                "One or more bricks could be down. Please "
+                "execute the command again after bringing all "
+                "bricks online and finishing any pending "
+                "heals\n");
+        else
+            printf(
+                "Command failed - %s. Please check the logs for"
+                " more details\n",
+                strerror(-op_ret));
+    }
+    return 0;
+}
+
+int
+glfsh_end(int op_ret, char *op_errstr)
+{
+    if (op_errstr)
+        printf("%s\n", op_errstr);
+    return 0;
+}
+
+int
+glfsh_print_hr_spb_status(char *path, uuid_t gfid, char *status)
+{
+    printf("%s\n", path);
+    fflush(stdout);
+    return 0;
+}
+
+int
+glfsh_no_print_hr_status(char *path, uuid_t gfid, char *status)
+{
+    return 0;
+}
+
+int
+glfsh_print_hr_heal_status(char *path, uuid_t gfid, char *status)
+{
+    printf("%s%s\n", path, status);
+    fflush(stdout);
+    return 0;
+}
+
+#if (HAVE_LIB_XML)
+
+int
+glfsh_xml_init()
+{
+    int ret = -1;
+    glfsh_writer = xmlNewTextWriterDoc(&glfsh_doc, 0);
+    if (glfsh_writer == NULL) {
+        return -1;
+    }
+
+    ret = xmlTextWriterStartDocument(glfsh_writer, "1.0", "UTF-8", "yes");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* <cliOutput> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"cliOutput");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* <healInfo> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"healInfo");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    /* <bricks> */
+    ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"bricks");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    xmlTextWriterFlush(glfsh_writer);
+xml_out:
+    return ret;
+}
+
+int
+glfsh_xml_end(int op_ret, char *op_errstr)
+{
+    int ret = -1;
+    int op_errno = 0;
+    gf_boolean_t alloc = _gf_false;
+
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+        if (op_errstr == NULL) {
+            op_errstr = gf_strdup(strerror(op_errno));
+            alloc = _gf_true;
+        }
+    } else {
+        op_errstr = NULL;
+    }
+
+    /* </bricks> */
+    ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* </healInfo> */
+    ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    ret = xmlTextWriterWriteFormatElement(glfsh_writer, (xmlChar *)"opRet",
+                                          "%d", op_ret);
+
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    ret = xmlTextWriterWriteFormatElement(glfsh_writer, (xmlChar *)"opErrno",
+                                          "%d", op_errno);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    if (op_errstr)
+        ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"opErrstr", "%s", op_errstr);
+    else
+        ret = xmlTextWriterWriteFormatElement(glfsh_writer,
+                                              (xmlChar *)"opErrstr", "%s", "");
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+    ret = xmlTextWriterEndDocument(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(ret, xml_out);
+
+    /* Dump xml document to stdout and pretty format it */
+    xmlSaveFormatFileEnc("-", glfsh_doc, "UTF-8", 1);
+
+    xmlFreeTextWriter(glfsh_writer);
+    xmlFreeDoc(glfsh_doc);
+xml_out:
+    if (alloc)
+        GF_FREE(op_errstr);
+    return ret;
+}
+
+int
+glfsh_print_xml_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
+{
+    int x_ret = 0;
+    if (ret < 0 && num_entries == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", strerror(-ret));
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"numberOfEntries", "-");
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
+        }
+        goto out;
+    } else if (ret == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", "Connected");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+
+    if (ret < 0) {
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"status",
+                "Failed to process entries completely. "
+                "(%s)%s %" PRIu64 "",
+                strerror(-ret), fmt_str, num_entries);
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
+        }
+    } else {
+        if (fmt_str) {
+            x_ret = xmlTextWriterWriteFormatElement(
+                glfsh_writer, (xmlChar *)"numberOfEntries", "%" PRIu64 "",
+                num_entries);
+            XML_RET_CHECK_AND_GOTO(x_ret, out);
+        }
+    }
+out:
+    if (x_ret >= 0) {
+        x_ret = xmlTextWriterEndElement(glfsh_writer);
+        if (x_ret >= 0) {
+            xmlTextWriterFlush(glfsh_writer);
+            x_ret = 0;
+        } else {
+            x_ret = -1;
+        }
+    }
+    return x_ret;
+}
+
+int
+glfsh_print_xml_heal_op_summary(int ret, num_entries_t *num_entries)
+{
+    int x_ret = 0;
+
+    if (ret < 0 && num_entries == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", strerror(-ret));
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"totalNumberOfEntries", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInHealPending", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInSplitBrain", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesPossiblyHealing", "-");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        goto out;
+    } else if (ret == 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status", "%s", "Connected");
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+
+    if (ret < 0) {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"status",
+            "Failed to process entries"
+            " completely. "
+            "(%s)totalNumberOfEntries%" PRIu64 "",
+            strerror(-ret), num_entries->num_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    } else {
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"totalNumberOfEntries", "%" PRIu64 "",
+            num_entries->num_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInHealPending",
+            "%" PRIu64 "", num_entries->pending_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesInSplitBrain",
+            "%" PRIu64 "", num_entries->spb_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+        x_ret = xmlTextWriterWriteFormatElement(
+            glfsh_writer, (xmlChar *)"numberOfEntriesPossiblyHealing",
+            "%" PRIu64 "", num_entries->possibly_healing_entries);
+        XML_RET_CHECK_AND_GOTO(x_ret, out);
+    }
+out:
+    if (x_ret >= 0) {
+        x_ret = xmlTextWriterEndElement(glfsh_writer);
+    }
+    return x_ret;
+}
+
+int
+glfsh_print_xml_file_status(char *path, uuid_t gfid, char *status)
+{
+    int x_ret = 0;
+
+    x_ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterWriteFormatAttribute(glfsh_writer, (xmlChar *)"gfid",
+                                              "%s", uuid_utoa(gfid));
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterWriteFormatString(glfsh_writer, "%s", path);
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    x_ret = xmlTextWriterEndElement(glfsh_writer);
+    XML_RET_CHECK_AND_GOTO(x_ret, out);
+    xmlTextWriterFlush(glfsh_writer);
+out:
+    return x_ret;
+}
+
+int
+glfsh_print_xml_brick_from_xl(xlator_t *xl, loc_t *rootloc)
+{
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+    char *uuid = NULL;
+    int ret = 0;
+    int x_ret = 0;
+
+    ret = dict_get_str(xl->options, "remote-host", &remote_host);
+    if (ret < 0)
+        goto print;
+
+    ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+    if (ret < 0)
+        goto print;
+    ret = syncop_getxattr(xl, rootloc, &xl->options, GF_XATTR_NODE_UUID_KEY,
+                          NULL, NULL);
+    if (ret < 0)
+        goto print;
+
+    ret = dict_get_str(xl->options, GF_XATTR_NODE_UUID_KEY, &uuid);
+    if (ret < 0)
+        goto print;
+print:
+
+    x_ret = xmlTextWriterStartElement(glfsh_writer, (xmlChar *)"brick");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+    x_ret = xmlTextWriterWriteFormatAttribute(
+        glfsh_writer, (xmlChar *)"hostUuid", "%s", uuid ? uuid : "-");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+
+    x_ret = xmlTextWriterWriteFormatElement(
+        glfsh_writer, (xmlChar *)"name", "%s:%s",
+        remote_host ? remote_host : "-", remote_subvol ? remote_subvol : "-");
+    XML_RET_CHECK_AND_GOTO(x_ret, xml_out);
+    xmlTextWriterFlush(glfsh_writer);
+xml_out:
+    return ret;
+}
+#endif
+
+int
+glfsh_link_inode_update_loc(loc_t *loc, struct iatt *iattr)
+{
+    inode_t *link_inode = NULL;
+    int ret = -1;
+
+    link_inode = inode_link(loc->inode, NULL, NULL, iattr);
+    if (link_inode == NULL)
+        goto out;
+
+    inode_unref(loc->inode);
+    loc->inode = link_inode;
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glfsh_no_print_hr_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
+{
+    return 0;
+}
+
+int
+glfsh_print_hr_heal_op_summary(int ret, num_entries_t *num_entries)
+{
+    if (ret < 0 && num_entries->num_entries == 0) {
+        printf("Status: %s\n", strerror(-ret));
+        printf("Total Number of entries: -\n");
+        printf("Number of entries in heal pending: -\n");
+        printf("Number of entries in split-brain: -\n");
+        printf("Number of entries possibly healing: -\n");
+        goto out;
+    } else if (ret == 0) {
+        printf("Status: Connected\n");
+    }
+
+    if (ret < 0) {
+        printf(
+            "Status: Failed to process entries completely. "
+            "(%s)\nTotal Number of entries: %" PRIu64 "\n",
+            strerror(-ret), num_entries->num_entries);
+    } else {
+        printf("Total Number of entries: %" PRIu64 "\n",
+               num_entries->num_entries);
+        printf("Number of entries in heal pending: %" PRIu64 "\n",
+               num_entries->pending_entries);
+        printf("Number of entries in split-brain: %" PRIu64 "\n",
+               num_entries->spb_entries);
+        printf("Number of entries possibly healing: %" PRIu64 "\n",
+               num_entries->possibly_healing_entries);
+    }
+out:
+    printf("\n");
+    fflush(stdout);
+    return 0;
+}
+
+int
+glfsh_print_hr_heal_op_status(int ret, uint64_t num_entries, char *fmt_str)
+{
+    if (ret < 0 && num_entries == 0) {
+        printf("Status: %s\n", strerror(-ret));
+        if (fmt_str)
+            printf("%s -\n", fmt_str);
+        goto out;
+    } else if (ret == 0) {
+        printf("Status: Connected\n");
+    }
+
+    if (ret < 0) {
+        if (fmt_str)
+            printf(
+                "Status: Failed to process entries completely. "
+                "(%s)\n%s %" PRIu64 "\n",
+                strerror(-ret), fmt_str, num_entries);
+    } else {
+        if (fmt_str)
+            printf("%s %" PRIu64 "\n", fmt_str, num_entries);
+    }
+out:
+    printf("\n");
+    return 0;
+}
+
+int
+glfsh_print_info_summary(int ret, num_entries_t *num_entries)
+{
+    return glfsh_output->print_heal_op_summary(ret, num_entries);
+}
+
+int
+glfsh_print_heal_op_status(int ret, uint64_t num_entries,
+                           gf_xl_afr_op_t heal_op)
+{
+    char *fmt_str = NULL;
+
+    if (heal_op == GF_SHD_OP_INDEX_SUMMARY)
+        fmt_str = "Number of entries:";
+    else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES)
+        fmt_str = "Number of entries in split-brain:";
+    else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK)
+        fmt_str = "Number of healed entries:";
+
+    return glfsh_output->print_heal_op_status(ret, num_entries, fmt_str);
+}
+
+int
+glfsh_get_index_dir_loc(loc_t *rootloc, xlator_t *xl, loc_t *dirloc,
+                        int32_t *op_errno, char *vgfid)
+{
+    void *index_gfid = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    struct iatt iattr = {0};
+    struct iatt parent = {0};
+
+    ret = syncop_getxattr(xl, rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret < 0) {
+        *op_errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret < 0) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_uuid_copy(dirloc->gfid, index_gfid);
+    dirloc->path = "";
+    dirloc->inode = inode_new(rootloc->inode->table);
+    ret = syncop_lookup(xl, dirloc, &iattr, &parent, NULL, NULL);
+    dirloc->path = NULL;
+    if (ret < 0) {
+        *op_errno = -ret;
+        goto out;
+    }
+    ret = glfsh_link_inode_update_loc(dirloc, &iattr);
+    if (ret)
+        goto out;
+
+    ret = glfs_loc_touchup(dirloc);
+    if (ret < 0) {
+        *op_errno = errno;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
+}
+
+static xlator_t *
+_get_ancestor(xlator_t *xl, gf_xl_afr_op_t heal_op)
+{
+    static char *replica_xl[] = {"cluster/replicate", NULL};
+    static char *heal_xls[] = {"cluster/replicate", "cluster/disperse", NULL};
+    char **ancestors = NULL;
+
+    if (heal_op == GF_SHD_OP_INDEX_SUMMARY || heal_op == GF_SHD_OP_HEAL_SUMMARY)
+        ancestors = heal_xls;
+    else
+        ancestors = replica_xl;
+
+    if (!xl || !xl->parents)
+        return NULL;
+
+    while (xl->parents) {
+        xl = xl->parents->xlator;
+        if (!xl)
+            break;
+        if (gf_get_index_by_elem(ancestors, xl->type) != -1)
+            return xl;
+    }
+
+    return NULL;
+}
+
+int
+glfsh_index_purge(xlator_t *subvol, inode_t *inode, char *name)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+
+    loc.parent = inode_ref(inode);
+    loc.name = name;
+
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+
+    loc_wipe(&loc);
+    return ret;
+}
+
+int
+glfsh_print_summary_status(dict_t *dict, char *path, uuid_t gfid,
+                           num_entries_t *num_entries, gf_boolean_t flag)
+{
+    int ret = 0;
+    char *value = NULL;
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret)
+        goto out;
+
+    if ((!strcmp(value, "heal")) || (!strcmp(value, "heal-pending"))) {
+        (num_entries->pending_entries)++;
+    } else if ((!strcmp(value, "split-brain")) ||
+               (!strcmp(value, "split-brain-pending"))) {
+        (num_entries->spb_entries)++;
+    } else if ((!strcmp(value, "possibly-healing-pending")) ||
+               (!strcmp(value, "possibly-healing"))) {
+        (num_entries->possibly_healing_entries)++;
+    } else {
+        goto out;
+    }
+    (num_entries->num_entries)++;
+out:
+    return ret;
+}
+
+int
+glfsh_print_spb_status(dict_t *dict, char *path, uuid_t gfid,
+                       num_entries_t *num_entries, gf_boolean_t flag)
+{
+    int ret = 0;
+    gf_boolean_t pending = _gf_false;
+    gf_boolean_t split_b = _gf_false;
+    char *value = NULL;
+    char gfid_str[64] = {0};
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret)
+        return 0;
+
+    if (!strcmp(value, "split-brain")) {
+        split_b = _gf_true;
+    } else if (!strcmp(value, "split-brain-pending")) {
+        split_b = _gf_true;
+        pending = _gf_true;
+    }
+    /* Consider the entry only iff :
+     * 1) The dir being processed is not indices/dirty, indicated by
+     *    flag == _gf_false
+     * 2) The dir being processed is indices/dirty but the entry also
+     *    exists in indices/xattrop dir and has already been processed.
+     */
+    if (split_b) {
+        if (!flag || (flag && !pending)) {
+            (num_entries->num_entries)++;
+            glfsh_output->print_spb_status(
+                path ? path : uuid_utoa_r(gfid, gfid_str), gfid, NULL);
+        }
+    }
+    return 0;
+}
+
+int
+glfsh_print_heal_status(dict_t *dict, char *path, uuid_t gfid,
+                        num_entries_t *num_entries, gf_boolean_t ignore_dirty)
+{
+    int ret = 0;
+    gf_boolean_t pending = _gf_false;
+    char *status = NULL;
+    char *value = NULL;
+    char gfid_str[64] = {0};
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if (ret || (!strcmp(value, "no-heal")))
+        return 0;
+
+    if (!strcmp(value, "heal")) {
+        ret = gf_asprintf(&status, " ");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "possibly-healing")) {
+        ret = gf_asprintf(&status, " - Possibly undergoing heal");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "split-brain")) {
+        ret = gf_asprintf(&status, " - Is in split-brain");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "heal-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " ");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "split-brain-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " - Is in split-brain");
+        if (ret < 0)
+            goto out;
+    } else if (!strcmp(value, "possibly-healing-pending")) {
+        pending = _gf_true;
+        ret = gf_asprintf(&status, " - Possibly undergoing heal");
+        if (ret < 0)
+            goto out;
+    }
+out:
+    /* If ignore_dirty is set, it means indices/dirty directory is
+     * being processed. Ignore the entry if it also exists in
+     * indices/xattrop.
+     * Boolean pending is set to true if the entry also exists in
+     * indices/xattrop directory.
+     */
+    if (ignore_dirty) {
+        if (pending) {
+            GF_FREE(status);
+            status = NULL;
+            return 0;
+        }
+    }
+    if (ret == -1)
+        status = NULL;
+
+    (num_entries->num_entries)++;
+    glfsh_output->print_heal_status(path ? path : uuid_utoa_r(gfid, gfid_str),
+                                    gfid, status ? status : "");
+
+    GF_FREE(status);
+    return 0;
+}
+
+int
+glfsh_heal_status_boolean(dict_t *dict, char *path, uuid_t gfid,
+                          num_entries_t *num_entries, gf_boolean_t ignore_dirty)
+{
+    int ret = 0;
+    char *value = NULL;
+
+    ret = dict_get_str(dict, "heal-info", &value);
+    if ((!ret) && (!strcmp(value, "no-heal")))
+        return 0;
+    else
+        return -1;
+}
+
+static void
+glfsh_heal_entries(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                   gf_dirent_t *entries, uint64_t *offset,
+                   num_entries_t *num_entries, dict_t *xattr_req)
+{
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    int ret = 0;
+    char file[64] = {0};
+
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        *offset = entry->d_off;
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+        snprintf(file, sizeof(file), "gfid:%s", entry->d_name);
+        ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file,
+                                         xattr_req);
+        if (ret)
+            continue;
+        (num_entries->num_entries)++;
+    }
+}
+
+static int
+glfsh_process_entries(xlator_t *xl, fd_t *fd, gf_dirent_t *entries,
+                      uint64_t *offset, num_entries_t *num_entries,
+                      print_status glfsh_print_status,
+                      gf_boolean_t ignore_dirty, glfsh_fail_mode_t mode)
+{
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    int ret = 0;
+    int print_status = 0;
+    char *path = NULL;
+    uuid_t gfid = {0};
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    loc_t loc = {
+        0,
+    };
+    this = THIS;
+
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        *offset = entry->d_off;
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+
+        if (dict) {
+            dict_unref(dict);
+            dict = NULL;
+        }
+        gf_uuid_clear(gfid);
+        GF_FREE(path);
+        path = NULL;
+
+        gf_uuid_parse(entry->d_name, gfid);
+        gf_uuid_copy(loc.gfid, gfid);
+        ret = syncop_getxattr(this, &loc, &dict, GF_HEAL_INFO, NULL, NULL);
+        if (ret) {
+            if ((mode != GLFSH_MODE_CONTINUE_ON_ERROR) && (ret == -ENOTCONN))
+                goto out;
+            else
+                continue;
+        }
+
+        ret = syncop_gfid_to_path(this->itable, xl, gfid, &path);
+
+        if (ret == -ENOENT || ret == -ESTALE) {
+            glfsh_index_purge(xl, fd->inode, entry->d_name);
+            ret = 0;
+            continue;
+        }
+        if (dict) {
+            print_status = glfsh_print_status(dict, path, gfid, num_entries,
+                                              ignore_dirty);
+            if ((print_status) && (mode != GLFSH_MODE_CONTINUE_ON_ERROR)) {
+                ret = -EAGAIN;
+                goto out;
+            }
+        }
+    }
+    ret = 0;
+out:
+    GF_FREE(path);
+    if (dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return ret;
+}
+
+static int
+glfsh_crawl_directory(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                      xlator_t *readdir_xl, fd_t *fd, loc_t *loc,
+                      dict_t *xattr_req, num_entries_t *num_entries,
+                      gf_boolean_t ignore)
+{
+    int ret = 0;
+    int heal_op = -1;
+    uint64_t offset = 0;
+    gf_dirent_t entries;
+    gf_boolean_t free_entries = _gf_false;
+    glfsh_fail_mode_t mode = GLFSH_MODE_CONTINUE_ON_ERROR;
+
+    INIT_LIST_HEAD(&entries.list);
+    ret = dict_get_int32(xattr_req, "heal-op", &heal_op);
+    if (ret)
+        return ret;
+
+    if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        mode = GLFSH_MODE_EXIT_ON_FIRST_FAILURE;
+
+    while (1) {
+        ret = syncop_readdir(readdir_xl, fd, 131072, offset, &entries, NULL,
+                             NULL);
+        if (ret <= 0)
+            break;
+        ret = 0;
+        free_entries = _gf_true;
+
+        if (list_empty(&entries.list))
+            goto out;
+
+        if (heal_op == GF_SHD_OP_INDEX_SUMMARY) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_heal_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_spb_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_HEAL_SUMMARY) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_print_summary_status,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
+            glfsh_heal_entries(fs, top_subvol, rootloc, &entries, &offset,
+                               num_entries, xattr_req);
+        } else if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) {
+            ret = glfsh_process_entries(readdir_xl, fd, &entries, &offset,
+                                        num_entries, glfsh_heal_status_boolean,
+                                        ignore, mode);
+            if (ret < 0)
+                goto out;
+        }
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+    }
+    ret = 0;
+out:
+    if (free_entries)
+        gf_dirent_free(&entries);
+    return ret;
+}
+
+static int
+glfsh_no_print_brick_from_xl(xlator_t *xl, loc_t *rootloc)
+{
+    return 0;
+}
+
+static int
+glfsh_print_brick_from_xl(xlator_t *xl, loc_t *rootloc)
+{
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+    int ret = 0;
+
+    ret = dict_get_str(xl->options, "remote-host", &remote_host);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+    if (ret < 0)
+        goto out;
+out:
+    if (ret < 0)
+        printf("Brick - Not able to get brick information\n");
+    else
+        printf("Brick %s:%s\n", remote_host, remote_subvol);
+    return ret;
+}
+
+int
+glfsh_print_pending_heals_type(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                               xlator_t *xl, gf_xl_afr_op_t heal_op,
+                               dict_t *xattr_req, char *vgfid,
+                               num_entries_t *num_entries)
+{
+    int ret = 0;
+    loc_t dirloc = {0};
+    fd_t *fd = NULL;
+    int32_t op_errno = 0;
+    gf_boolean_t ignore = _gf_false;
+
+    if (!strcmp(vgfid, GF_XATTROP_DIRTY_GFID))
+        ignore = _gf_true;
+
+    ret = glfsh_get_index_dir_loc(rootloc, xl, &dirloc, &op_errno, vgfid);
+    if (ret < 0) {
+        if (op_errno == ESTALE || op_errno == ENOENT || op_errno == ENOTSUP)
+            ret = 0;
+        else
+            ret = -op_errno;
+        goto out;
+    }
+
+    ret = syncop_dirfd(xl, &dirloc, &fd, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret)
+        goto out;
+
+    ret = glfsh_crawl_directory(fs, top_subvol, rootloc, xl, fd, &dirloc,
+                                xattr_req, num_entries, ignore);
+    if (fd)
+        fd_unref(fd);
+out:
+    loc_wipe(&dirloc);
+    return ret;
+}
+
+int
+glfsh_print_pending_heals(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                          xlator_t *xl, gf_xl_afr_op_t heal_op,
+                          gf_boolean_t is_parent_replicate)
+{
+    int ret = 0;
+    num_entries_t num_entries = {
+        0,
+    };
+    num_entries_t total = {
+        0,
+    };
+
+    dict_t *xattr_req = NULL;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op", heal_op);
+    if (ret)
+        goto out;
+
+    if ((!is_parent_replicate) &&
+        ((heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||
+         (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE))) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = glfsh_output->print_brick_from_xl(xl, rootloc);
+    if (ret < 0)
+        goto out;
+
+    ret = glfsh_print_pending_heals_type(fs, top_subvol, rootloc, xl, heal_op,
+                                         xattr_req, GF_XATTROP_INDEX_GFID,
+                                         &num_entries);
+
+    if (ret < 0 && heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        goto out;
+
+    total.num_entries += num_entries.num_entries;
+    total.pending_entries += num_entries.pending_entries;
+    total.spb_entries += num_entries.spb_entries;
+    total.possibly_healing_entries += num_entries.possibly_healing_entries;
+    num_entries.num_entries = 0;
+    num_entries.pending_entries = 0;
+    num_entries.spb_entries = 0;
+    num_entries.possibly_healing_entries = 0;
+    if (ret == -ENOTCONN)
+        goto out;
+
+    if (is_parent_replicate) {
+        ret = glfsh_print_pending_heals_type(
+            fs, top_subvol, rootloc, xl, heal_op, xattr_req,
+            GF_XATTROP_DIRTY_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        total.pending_entries += num_entries.pending_entries;
+        total.spb_entries += num_entries.spb_entries;
+        total.possibly_healing_entries += num_entries.possibly_healing_entries;
+    }
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (heal_op == GF_SHD_OP_HEAL_SUMMARY) {
+        glfsh_print_info_summary(ret, &total);
+    } else {
+        glfsh_print_heal_op_status(ret, total.num_entries, heal_op);
+    }
+    return ret;
+}
+
+static int
+glfsh_set_heal_options(glfs_t *fs, gf_xl_afr_op_t heal_op)
+{
+    int ret = 0;
+
+    ret = glfs_set_xlator_option(fs, "*-replicate-*",
+                                 "background-self-heal-count", "0");
+    if (ret)
+        goto out;
+
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "halo-enabled", "off");
+    if (ret)
+        goto out;
+
+    if ((heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) &&
+        (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) &&
+        (heal_op != GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME))
+        return 0;
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "data-self-heal", "on");
+    if (ret)
+        goto out;
+
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "metadata-self-heal",
+                                 "on");
+    if (ret)
+        goto out;
+
+    ret = glfs_set_xlator_option(fs, "*-replicate-*", "entry-self-heal", "on");
+
+out:
+    return ret;
+}
+
+static int
+glfsh_validate_volume(xlator_t *xl, gf_xl_afr_op_t heal_op)
+{
+    xlator_t *heal_xl = NULL;
+    int ret = -1;
+
+    while (xl->next)
+        xl = xl->next;
+
+    while (xl) {
+        if (strcmp(xl->type, "protocol/client") == 0) {
+            heal_xl = _get_ancestor(xl, heal_op);
+            if (heal_xl) {
+                ret = 0;
+                break;
+            }
+        }
+
+        xl = xl->prev;
+    }
+
+    return ret;
+}
+
+static xlator_t *
+_brick_path_to_client_xlator(xlator_t *top_subvol, char *hostname,
+                             char *brickpath)
+{
+    int ret = 0;
+    xlator_t *xl = NULL;
+    char *remote_host = NULL;
+    char *remote_subvol = NULL;
+
+    xl = top_subvol;
+
+    while (xl->next)
+        xl = xl->next;
+
+    while (xl) {
+        if (!strcmp(xl->type, "protocol/client")) {
+            ret = dict_get_str(xl->options, "remote-host", &remote_host);
+            if (ret < 0)
+                goto out;
+            ret = dict_get_str(xl->options, "remote-subvolume", &remote_subvol);
+            if (ret < 0)
+                goto out;
+            if (!strcmp(hostname, remote_host) &&
+                !strcmp(brickpath, remote_subvol))
+                return xl;
+        }
+        xl = xl->prev;
+    }
+
+out:
+    return NULL;
+}
+
+int
+glfsh_gather_heal_info(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                       gf_xl_afr_op_t heal_op)
+{
+    int ret = 0;
+    xlator_t *xl = NULL;
+    xlator_t *heal_xl = NULL;
+    xlator_t *old_THIS = NULL;
+
+    xl = top_subvol;
+    while (xl->next)
+        xl = xl->next;
+    while (xl) {
+        if (strcmp(xl->type, "protocol/client") == 0 &&
+            !strstr(xl->name, "-ta-")) {
+            heal_xl = _get_ancestor(xl, heal_op);
+            if (heal_xl) {
+                old_THIS = THIS;
+                THIS = heal_xl;
+                ret = glfsh_print_pending_heals(
+                    fs, top_subvol, rootloc, xl, heal_op,
+                    !strcmp(heal_xl->type, "cluster/replicate"));
+                THIS = old_THIS;
+
+                if ((ret < 0) &&
+                    (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE))
+                    goto out;
+            }
+        }
+
+        xl = xl->prev;
+    }
+
+out:
+    if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        ret = 0;
+
+    return ret;
+}
+
+int
+_validate_directory(dict_t *xattr_req, char *file)
+{
+    int heal_op = -1;
+    int ret = 0;
+
+    ret = dict_get_int32(xattr_req, "heal-op", &heal_op);
+    if (ret)
+        return ret;
+
+    if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) {
+        printf("'bigger-file' not a valid option for directories.\n");
+        ret = -1;
+    } else if (heal_op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) {
+        printf(
+            "'source-brick' option used on a directory (%s). "
+            "Performing conservative merge.\n",
+            file);
+    }
+
+    return ret;
+}
+
+int
+glfsh_heal_splitbrain_file(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                           char *file, dict_t *xattr_req)
+{
+    int ret = -1;
+    int reval = 0;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    char *filename = NULL;
+    char *filename1 = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *xl = top_subvol;
+    dict_t *xattr_rsp = NULL;
+    char *sh_fail_msg = NULL;
+    char *gfid_heal_msg = NULL;
+    int32_t op_errno = 0;
+    gf_boolean_t flag = _gf_false;
+
+    if (!strncmp(file, "gfid:", 5)) {
+        filename = gf_strdup(file);
+        if (!filename) {
+            printf("Error allocating memory to filename\n");
+            goto out;
+        }
+        path = strtok(filename, ":");
+        path = strtok(NULL, ";");
+        gf_uuid_parse(path, loc.gfid);
+        loc.path = gf_strdup(uuid_utoa(loc.gfid));
+        if (!loc.path) {
+            printf("Error allocating memory to path\n");
+            goto out;
+        }
+        loc.inode = inode_new(rootloc->inode->table);
+        if (!loc.inode) {
+            printf("Error getting inode\n");
+            goto out;
+        }
+        ret = syncop_lookup(xl, &loc, &iatt, 0, xattr_req, &xattr_rsp);
+        if (ret) {
+            op_errno = -ret;
+            printf("Lookup failed on %s:%s.\n", file, strerror(op_errno));
+            goto out;
+        }
+    } else {
+        if (file[0] != '/') {
+            printf(
+                "<FILE> must be absolute path w.r.t. the "
+                "volume, starting with '/'\n");
+            ret = -1;
+            goto out;
+        }
+        path1 = gf_strdup(file);
+        if (!path1) {
+            printf("Error allocating memory to path\n");
+            ret = -1;
+            goto out;
+        }
+        path2 = gf_strdup(file);
+        if (!path2) {
+            printf("Error allocating memory to path\n");
+            ret = -1;
+            goto out;
+        }
+        path = dirname(path1);
+        filename1 = basename(path2);
+    retry1:
+        ret = glfs_resolve(fs, xl, path, &loc, &iatt, reval);
+        ESTALE_RETRY(ret, errno, reval, &loc, retry1);
+        if (ret) {
+            printf("Lookup failed on %s:%s\n", path, strerror(errno));
+            goto out;
+        }
+        GF_FREE((char *)loc.path);
+        loc.path = gf_strdup(file);
+        if (!loc.path) {
+            printf("Error allocating memory for path\n");
+            ret = -1;
+            goto out;
+        }
+        loc.parent = inode_unref(loc.parent);
+        loc.parent = inode_ref(loc.inode);
+        loc.inode = inode_unref(loc.inode);
+        loc.inode = inode_new(rootloc->inode->table);
+        if (!loc.inode) {
+            printf("Error getting inode\n");
+            ret = -1;
+            goto out;
+        }
+        loc.name = filename1;
+        gf_uuid_copy(loc.pargfid, loc.gfid);
+        gf_uuid_clear(loc.gfid);
+
+        ret = syncop_lookup(xl, &loc, &iatt, 0, xattr_req, &xattr_rsp);
+        if (ret) {
+            op_errno = -ret;
+            printf("Lookup failed on %s:%s.\n", file, strerror(op_errno));
+            flag = _gf_true;
+        }
+
+        ret = dict_get_str(xattr_rsp, "gfid-heal-msg", &gfid_heal_msg);
+        if (!ret) {
+            printf("%s for file %s\n", gfid_heal_msg, file);
+            loc_wipe(&loc);
+            goto out;
+        }
+        if (flag)
+            goto out;
+
+        reval = 0;
+        loc_wipe(&loc);
+        memset(&iatt, 0, sizeof(iatt));
+
+    retry2:
+        ret = glfs_resolve(fs, xl, file, &loc, &iatt, reval);
+        ESTALE_RETRY(ret, errno, reval, &loc, retry2);
+        if (ret) {
+            printf("Lookup failed on %s:%s\n", file, strerror(errno));
+            goto out;
+        }
+    }
+
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = _validate_directory(xattr_req, file);
+        if (ret)
+            goto out;
+    }
+    ret = syncop_getxattr(xl, &loc, &xattr_rsp, GF_AFR_HEAL_SBRAIN, xattr_req,
+                          NULL);
+    if (ret) {
+        op_errno = -ret;
+        printf("Healing %s failed:%s.\n", file, strerror(op_errno));
+        goto out;
+    }
+    ret = dict_get_str(xattr_rsp, "sh-fail-msg", &sh_fail_msg);
+    if (!ret) {
+        printf("Healing %s failed: %s.\n", file, sh_fail_msg);
+        ret = -1;
+        goto out;
+    }
+    printf("Healed %s.\n", file);
+    ret = 0;
+out:
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+    if (path1)
+        GF_FREE(path1);
+    if (path2)
+        GF_FREE(path2);
+    if (filename)
+        GF_FREE(filename);
+    loc_wipe(&loc);
+    return ret;
+}
+
+int
+glfsh_heal_from_brick_type(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                           char *hostname, char *brickpath, xlator_t *client,
+                           dict_t *xattr_req, char *vgfid,
+                           num_entries_t *num_entries)
+{
+    fd_t *fd = NULL;
+    loc_t dirloc = {0};
+    int32_t op_errno = 0;
+    int ret = -1;
+
+    ret = glfsh_get_index_dir_loc(rootloc, client, &dirloc, &op_errno, vgfid);
+    if (ret < 0) {
+        if (op_errno == ESTALE || op_errno == ENOENT)
+            ret = 0;
+        else
+            ret = -op_errno;
+        goto out;
+    }
+
+    ret = syncop_dirfd(client, &dirloc, &fd, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret)
+        goto out;
+    ret = glfsh_crawl_directory(fs, top_subvol, rootloc, client, fd, &dirloc,
+                                xattr_req, num_entries, _gf_false);
+    if (fd)
+        fd_unref(fd);
+out:
+    loc_wipe(&dirloc);
+    return ret;
+}
+
+int
+glfsh_heal_from_brick(glfs_t *fs, xlator_t *top_subvol, loc_t *rootloc,
+                      char *hostname, char *brickpath, char *file)
+{
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+    xlator_t *client = NULL;
+    num_entries_t num_entries = {
+        0,
+    };
+    num_entries_t total = {
+        0,
+    };
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op",
+                         GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret)
+        goto out;
+    client = _brick_path_to_client_xlator(top_subvol, hostname, brickpath);
+    if (!client) {
+        printf("\"%s:%s\"- No such brick available in the volume.\n", hostname,
+               brickpath);
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(xattr_req, "child-name", client->name);
+    if (ret)
+        goto out;
+    if (file)
+        ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file,
+                                         xattr_req);
+    else {
+        ret = glfsh_heal_from_brick_type(fs, top_subvol, rootloc, hostname,
+                                         brickpath, client, xattr_req,
+                                         GF_XATTROP_INDEX_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        num_entries.num_entries = 0;
+        if (ret == -ENOTCONN)
+            goto out;
+
+        ret = glfsh_heal_from_brick_type(fs, top_subvol, rootloc, hostname,
+                                         brickpath, client, xattr_req,
+                                         GF_XATTROP_DIRTY_GFID, &num_entries);
+        total.num_entries += num_entries.num_entries;
+        if (ret < 0)
+            goto out;
+    }
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (!file)
+        glfsh_print_heal_op_status(ret, total.num_entries,
+                                   GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+
+    return ret;
+}
+
+int
+glfsh_heal_from_bigger_file_or_mtime(glfs_t *fs, xlator_t *top_subvol,
+                                     loc_t *rootloc, char *file,
+                                     gf_xl_afr_op_t heal_op)
+{
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto out;
+    ret = dict_set_int32(xattr_req, "heal-op", heal_op);
+    if (ret)
+        goto out;
+    ret = glfsh_heal_splitbrain_file(fs, top_subvol, rootloc, file, xattr_req);
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    return ret;
+}
+
+static void
+cleanup(glfs_t *fs)
+{
+    if (!fs)
+        return;
+#if 0
+        /* glfs fini path is still racy and crashing the program. Since
+         * this program any way has to die, we are not going to call fini
+         * in the released versions. i.e. final builds. For all
+         * internal testing lets enable this so that glfs_fini code
+         * path becomes stable. */
+        glfs_fini (fs);
+#endif
+}
+
+glfsh_info_t glfsh_human_readable = {
+    .init = glfsh_init,
+    .print_brick_from_xl = glfsh_print_brick_from_xl,
+    .print_heal_op_status = glfsh_print_hr_heal_op_status,
+    .print_heal_op_summary = glfsh_print_hr_heal_op_summary,
+    .print_heal_status = glfsh_print_hr_heal_status,
+    .print_spb_status = glfsh_print_hr_spb_status,
+    .end = glfsh_end};
+
+glfsh_info_t glfsh_no_print = {
+    .init = glfsh_init,
+    .print_brick_from_xl = glfsh_no_print_brick_from_xl,
+    .print_heal_op_status = glfsh_no_print_hr_heal_op_status,
+    .print_heal_status = glfsh_no_print_hr_status,
+    .print_spb_status = glfsh_no_print_hr_status,
+    .end = glfsh_end_op_granular_entry_heal};
+
+#if (HAVE_LIB_XML)
+glfsh_info_t glfsh_xml_output = {
+    .init = glfsh_xml_init,
+    .print_brick_from_xl = glfsh_print_xml_brick_from_xl,
+    .print_heal_op_status = glfsh_print_xml_heal_op_status,
+    .print_heal_op_summary = glfsh_print_xml_heal_op_summary,
+    .print_heal_status = glfsh_print_xml_file_status,
+    .print_spb_status = glfsh_print_xml_file_status,
+    .end = glfsh_xml_end};
+#endif
+
+static void
+parse_flags(int *argc, char **argv, int *flags)
+{
+    int i = 0;
+    char *opt = NULL;
+    int count = 0;
+
+    for (i = 0; i < *argc; i++) {
+        opt = strtail(argv[i], "--");
+        if (!opt)
+            continue;
+        if (strcmp(opt, "nolog") == 0) {
+            *flags |= MODE_NO_LOG;
+            count++;
+        } else if (strcmp(opt, "xml") == 0) {
+            *flags |= MODE_XML;
+            count++;
+        }
+    }
+    *argc = *argc - count;
+}
+
+int
+main(int argc, char **argv)
+{
+    glfs_t *fs = NULL;
+    int ret = 0;
+    char *volname = NULL;
+    xlator_t *top_subvol = NULL;
+    loc_t rootloc = {0};
+    char logfilepath[PATH_MAX] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *file = NULL;
+    char *op_errstr = NULL;
+    char *socket_filepath = NULL;
+    gf_xl_afr_op_t heal_op = -1;
+    gf_loglevel_t log_level = GF_LOG_INFO;
+    int flags = 0;
+
+    if (argc < 2) {
+        printf(USAGE_STR, argv[0]);
+        ret = -1;
+        goto out;
+    } else if (argc >= 4) {
+        if (!strcmp(argv[argc - 2], "glusterd-sock")) {
+            socket_filepath = argv[argc - 1];
+            argc = argc - 2;
+        }
+    }
+    volname = argv[1];
+
+    parse_flags(&argc, argv, &flags);
+    if (flags & MODE_NO_LOG)
+        log_level = GF_LOG_NONE;
+    if (flags & MODE_XML)
+        is_xml = 1;
+
+    switch (argc) {
+        case 2:
+            heal_op = GF_SHD_OP_INDEX_SUMMARY;
+            break;
+        case 3:
+            if (!strcmp(argv[2], "split-brain-info")) {
+                heal_op = GF_SHD_OP_SPLIT_BRAIN_FILES;
+            } else if (!strcmp(argv[2], "granular-entry-heal-op")) {
+                heal_op = GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE;
+            } else if (!strcmp(argv[2], "info-summary")) {
+                heal_op = GF_SHD_OP_HEAL_SUMMARY;
+            } else {
+                printf(USAGE_STR, argv[0]);
+                ret = -1;
+                goto out;
+            }
+            break;
+        case 4:
+            if (!strcmp(argv[2], "bigger-file")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE;
+                file = argv[3];
+            } else if (!strcmp(argv[2], "latest-mtime")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME;
+                file = argv[3];
+            } else if (!strcmp(argv[2], "source-brick")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
+                hostname = strtok(argv[3], ":");
+                path = strtok(NULL, ":");
+            } else {
+                printf(USAGE_STR, argv[0]);
+                ret = -1;
+                goto out;
+            }
+            break;
+        case 5:
+            if (!strcmp(argv[2], "source-brick")) {
+                heal_op = GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK;
+                hostname = strtok(argv[3], ":");
+                path = strtok(NULL, ":");
+                file = argv[4];
+            } else {
+                printf(USAGE_STR, argv[0]);
+                ret = -1;
+                goto out;
+            }
+            break;
+        default:
+            printf(USAGE_STR, argv[0]);
+            ret = -1;
+            goto out;
+    }
+
+    glfsh_output = &glfsh_human_readable;
+    if (is_xml) {
+#if (HAVE_LIB_XML)
+        if ((heal_op == GF_SHD_OP_INDEX_SUMMARY) ||
+            (heal_op == GF_SHD_OP_SPLIT_BRAIN_FILES) ||
+            (heal_op == GF_SHD_OP_HEAL_SUMMARY)) {
+            glfsh_output = &glfsh_xml_output;
+        } else {
+            printf(USAGE_STR, argv[0]);
+            ret = -1;
+            goto out;
+        }
+#else
+        /*No point doing anything, just fail the command*/
+        exit(EXIT_FAILURE);
+#endif
+    }
+
+    if (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+        glfsh_output = &glfsh_no_print;
+
+    ret = glfsh_output->init();
+    if (ret)
+        exit(EXIT_FAILURE);
+
+    fs = glfs_new(volname);
+    if (!fs) {
+        ret = -errno;
+        gf_asprintf(&op_errstr, "Not able to initialize volume '%s'", volname);
+        goto out;
+    }
+
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+    if (socket_filepath != NULL) {
+        ret = glfs_set_volfile_server(fs, "unix", socket_filepath, 0);
+    } else {
+        ret = glfs_set_volfile_server(fs, "unix", DEFAULT_GLUSTERD_SOCKFILE, 0);
+    }
+    if (ret) {
+        ret = -errno;
+        gf_asprintf(&op_errstr,
+                    "Setting the volfile server failed, "
+                    "%s",
+                    strerror(errno));
+        goto out;
+    }
+
+    ret = glfsh_set_heal_options(fs, heal_op);
+    if (ret) {
+        printf("Setting xlator heal options failed, %s\n", strerror(errno));
+        goto out;
+    }
+    snprintf(logfilepath, sizeof(logfilepath),
+             DEFAULT_HEAL_LOG_FILE_DIRECTORY "/glfsheal-%s.log", volname);
+    ret = glfs_set_logging(fs, logfilepath, log_level);
+    if (ret < 0) {
+        ret = -errno;
+        gf_asprintf(&op_errstr,
+                    "Failed to set the log file path, "
+                    "%s",
+                    strerror(errno));
+        goto out;
+    }
+
+    ret = glfs_setfspid(fs, GF_CLIENT_PID_GLFS_HEAL);
+    if (ret) {
+        printf("Setting client pid failed, %s\n", strerror(errno));
+        goto out;
+    }
+
+    ret = glfs_init(fs);
+    if (ret < 0) {
+        ret = -errno;
+        if (errno == ENOENT) {
+            gf_asprintf(&op_errstr, "Volume %s does not exist", volname);
+        } else {
+            gf_asprintf(&op_errstr,
+                        "%s: Not able to fetch "
+                        "volfile from glusterd",
+                        volname);
+        }
+        goto out;
+    }
+
+    top_subvol = glfs_active_subvol(fs);
+    if (!top_subvol) {
+        ret = -errno;
+        if (errno == ENOTCONN) {
+            gf_asprintf(&op_errstr,
+                        "Volume %s is not started "
+                        "(Or) All the bricks are not "
+                        "running.",
+                        volname);
+        } else {
+            gf_asprintf(&op_errstr,
+                        "%s: Not able to mount the "
+                        "volume, %s",
+                        volname, strerror(errno));
+        }
+        goto out;
+    }
+
+    char *var_str = (heal_op == GF_SHD_OP_INDEX_SUMMARY ||
+                     heal_op == GF_SHD_OP_HEAL_SUMMARY)
+                        ? "replicate/disperse"
+                        : "replicate";
+
+    ret = glfsh_validate_volume(top_subvol, heal_op);
+    if (ret < 0) {
+        ret = -EINVAL;
+        gf_asprintf(&op_errstr,
+                    "This command is supported "
+                    "for only volumes of %s type. Volume %s "
+                    "is not of type %s",
+                    var_str, volname, var_str);
+        goto out;
+    }
+    rootloc.inode = inode_ref(top_subvol->itable->root);
+    ret = glfs_loc_touchup(&rootloc);
+    if (ret < 0) {
+        ret = -errno;
+        goto out;
+    }
+
+    switch (heal_op) {
+        case GF_SHD_OP_INDEX_SUMMARY:
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_HEAL_SUMMARY:
+            ret = glfsh_gather_heal_info(fs, top_subvol, &rootloc, heal_op);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            ret = glfsh_heal_from_bigger_file_or_mtime(fs, top_subvol, &rootloc,
+                                                       file, heal_op);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = glfsh_heal_from_brick(fs, top_subvol, &rootloc, hostname,
+                                        path, file);
+            break;
+        default:
+            ret = -EINVAL;
+            break;
+    }
+
+    glfsh_output->end(ret, NULL);
+    if (ret < 0)
+        ret = -ret;
+    loc_wipe(&rootloc);
+    glfs_subvol_done(fs, top_subvol);
+    cleanup(fs);
+
+    return ret;
+out:
+    if (fs && top_subvol)
+        glfs_subvol_done(fs, top_subvol);
+    loc_wipe(&rootloc);
+    cleanup(fs);
+    if (glfsh_output)
+        glfsh_output->end(ret, op_errstr);
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    return ret;
+}
diff --git a/libgfchangelog.pc.in b/libgfchangelog.pc.in
new file mode 100644
index 00000000000..79eac2ad2d3
--- /dev/null
+++ b/libgfchangelog.pc.in
@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+
+Name: libgfchangelog
+Description: GlusterFS Changelog Consumer Library
+Version: @LIBGFCHANGELOG_VERSION@
+Requires: @PKGCONFIG_UUID@
+Libs: -L${libdir} -lgfchangelog -lglusterfs
+Cflags: -I${includedir} -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
diff --git a/scheduler/nufa/Makefile.am b/libglusterd/Makefile.am
index d471a3f9243..a985f42a877 100644
--- a/scheduler/nufa/Makefile.am
+++ b/libglusterd/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/libglusterd/src/Makefile.am b/libglusterd/src/Makefile.am
new file mode 100644
index 00000000000..684d2bac96b
--- /dev/null
+++ b/libglusterd/src/Makefile.am
@@ -0,0 +1,31 @@
+libglusterd_la_CFLAGS = $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+	-DDATADIR=\"$(localstatedir)\"
+
+libglusterd_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-DXLATORPARENTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)\" \
+	-DXXH_NAMESPACE=GF_ -D__USE_LARGEFILE64 \
+	-I$(CONTRIBDIR)/rbtree \
+	-I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \
+	-DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel \
+	-I$(CONTRIBDIR)/xxhash \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src/
+
+libglusterd_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS)
+libglusterd_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) $(GF_LDFLAGS) \
+	-export-symbols $(top_srcdir)/libglusterd/src/libglusterd.sym
+
+lib_LTLIBRARIES = libglusterd.la
+
+libglusterd_la_SOURCES = gd-common-utils.c
+
+libglusterd_la_HEADERS = gd-common-utils.h
+
+libglusterd_ladir = $(includedir)/glusterfs
+
+noinst_HEADERS = gd-common-utils.h
+
+EXTRA_DIST = libglusterd.sym
+
+CLEANFILES =
diff --git a/libglusterd/src/gd-common-utils.c b/libglusterd/src/gd-common-utils.c
new file mode 100644
index 00000000000..243fab215e6
--- /dev/null
+++ b/libglusterd/src/gd-common-utils.c
@@ -0,0 +1,78 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "gd-common-utils.h"
+#include "cli1-xdr.h"
+
+int
+get_vol_type(int type, int dist_count, int brick_count)
+{
+    if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
+        (dist_count < brick_count))
+        type = type + GF_CLUSTER_TYPE_MAX - 1;
+
+    return type;
+}
+
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val)
+{
+    switch (mem_num) {
+        case 0:
+            return (sts_val->node);
+        case 1:
+            return (sts_val->master);
+        case 2:
+            return (sts_val->brick);
+        case 3:
+            return (sts_val->slave_user);
+        case 4:
+            return (sts_val->slave);
+        case 5:
+            return (sts_val->slave_node);
+        case 6:
+            return (sts_val->worker_status);
+        case 7:
+            return (sts_val->crawl_status);
+        case 8:
+            return (sts_val->last_synced);
+        case 9:
+            return (sts_val->entry);
+        case 10:
+            return (sts_val->data);
+        case 11:
+            return (sts_val->meta);
+        case 12:
+            return (sts_val->failures);
+        case 13:
+            return (sts_val->checkpoint_time);
+        case 14:
+            return (sts_val->checkpoint_completed);
+        case 15:
+            return (sts_val->checkpoint_completion_time);
+        case 16:
+            return (sts_val->brick_host_uuid);
+        case 17:
+            return (sts_val->last_synced_utc);
+        case 18:
+            return (sts_val->checkpoint_time_utc);
+        case 19:
+            return (sts_val->checkpoint_completion_time_utc);
+        case 20:
+            return (sts_val->slavekey);
+        case 21:
+            return (sts_val->session_slave);
+        default:
+            goto out;
+    }
+
+out:
+    return NULL;
+}
diff --git a/libglusterd/src/gd-common-utils.h b/libglusterd/src/gd-common-utils.h
new file mode 100644
index 00000000000..b9bb4f956fe
--- /dev/null
+++ b/libglusterd/src/gd-common-utils.h
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GD_COMMON_UTILS_H
+#define _GD_COMMON_UTILS_H
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stddef.h>
+
+#include "protocol-common.h"
+#include "rpcsvc.h"
+
+int
+get_vol_type(int type, int dist_count, int brick_count);
+
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val);
+
+#endif /* _GD_COMMON_UTILS_H */
diff --git a/libglusterd/src/libglusterd.sym b/libglusterd/src/libglusterd.sym
new file mode 100644
index 00000000000..45969a87c12
--- /dev/null
+++ b/libglusterd/src/libglusterd.sym
@@ -0,0 +1,2 @@
+get_vol_type
+get_struct_variable
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 619ccf158a1..385e8ef4600 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -1,21 +1,116 @@
-libglusterfs_la_CFLAGS = -fPIC  -Wall -g -shared -nostartfiles $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
+noinst_PYTHON = generator.py gen-defaults.py $(top_srcdir)/events/eventskeygen.py
 
-libglusterfs_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" -D$(GF_HOST_OS) -I$(CONTRIBDIR)/rbtree  -DSCHEDULERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler\" -I$(CONTRIBDIR)/md5
+libglusterfs_la_CFLAGS = $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+	-DDATADIR=\"$(localstatedir)\"
 
-libglusterfs_la_LIBADD = @LEXLIB@
+libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-DXLATORPARENTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)\" \
+	-DXXH_NAMESPACE=GF_ -D__USE_LARGEFILE64 \
+	-I$(CONTRIBDIR)/rbtree \
+	-I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \
+	-DSBIN_DIR=\"$(sbindir)\" -I$(CONTRIBDIR)/timer-wheel \
+	-I$(CONTRIBDIR)/xxhash
+
+libglusterfs_la_LIBADD = $(ZLIB_LIBS) $(MATH_LIB) $(UUID_LIBS) $(LIB_DL) \
+	$(URCU_LIBS) $(URCU_CDS_LIBS)
+libglusterfs_la_LDFLAGS = -version-info $(LIBGLUSTERFS_LT_VERSION) $(GF_LDFLAGS) \
+	-export-symbols $(top_srcdir)/libglusterfs/src/libglusterfs.sym
 
 lib_LTLIBRARIES = libglusterfs.la
+libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
+
+CONTRIB_BUILDDIR = $(top_builddir)/contrib
+
+libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
+	hashfn.c common-utils.c timer.c inode.c call-stub.c \
+	compat.c fd.c compat-errno.c event.c mem-pool.c gf-dirent.c syscall.c \
+	iobuf.c globals.c statedump.c stack.c checksum.c daemon.c timespec.c \
+	$(CONTRIBDIR)/rbtree/rb.c rbthash.c store.c latency.c \
+	graph.c syncop.c graph-print.c trie.c run.c options.c fd-lk.c \
+	circ-buff.c event-history.c gidcache.c ctx.c client_t.c event-poll.c \
+	event-epoll.c syncop-utils.c cluster-syncop.c refcount.c \
+	$(CONTRIBDIR)/libgen/basename_r.c \
+	$(CONTRIBDIR)/libgen/dirname_r.c \
+	strfd.c parse-utils.c $(CONTRIBDIR)/mount/mntent.c \
+	$(CONTRIBDIR)/libexecinfo/execinfo.c quota-common-utils.c rot-buffs.c \
+	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
+	$(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \
+	$(CONTRIBDIR)/xxhash/xxhash.c \
+	throttle-tbf.c monitoring.c async.c
+
+nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c
+nodist_libglusterfs_la_HEADERS = y.tab.h
+
+BUILT_SOURCES = graph.lex.c defaults.c eventtypes.h
+
+libglusterfs_la_HEADERS = glusterfs/common-utils.h glusterfs/defaults.h \
+	glusterfs/default-args.h glusterfs/dict.h glusterfs/glusterfs.h \
+	glusterfs/hashfn.h glusterfs/timespec.h glusterfs/logging.h \
+	glusterfs/xlator.h glusterfs/stack.h glusterfs/timer.h glusterfs/list.h \
+	glusterfs/inode.h glusterfs/call-stub.h glusterfs/compat.h glusterfs/fd.h \
+	glusterfs/revision.h glusterfs/compat-errno.h glusterfs/gf-event.h \
+	glusterfs/mem-pool.h glusterfs/byte-order.h glusterfs/gf-dirent.h \
+	glusterfs/locking.h glusterfs/syscall.h glusterfs/iobuf.h \
+	glusterfs/globals.h glusterfs/statedump.h glusterfs/checksum.h \
+	glusterfs/daemon.h glusterfs/store.h glusterfs/rbthash.h glusterfs/iatt.h \
+	glusterfs/latency.h glusterfs/mem-types.h glusterfs/syncop.h \
+	glusterfs/cluster-syncop.h glusterfs/graph-utils.h glusterfs/trie.h \
+	glusterfs/refcount.h glusterfs/run.h glusterfs/options.h \
+	glusterfs/lkowner.h glusterfs/fd-lk.h glusterfs/circ-buff.h \
+	glusterfs/event-history.h glusterfs/gidcache.h glusterfs/client_t.h \
+	glusterfs/glusterfs-acl.h glusterfs/glfs-message-id.h \
+	glusterfs/template-component-messages.h glusterfs/strfd.h \
+	glusterfs/syncop-utils.h glusterfs/parse-utils.h \
+	glusterfs/libglusterfs-messages.h glusterfs/lvm-defaults.h \
+	glusterfs/quota-common-utils.h glusterfs/rot-buffs.h \
+	glusterfs/compat-uuid.h glusterfs/upcall-utils.h glusterfs/throttle-tbf.h \
+	glusterfs/events.h glusterfs/atomic.h glusterfs/monitoring.h \
+	glusterfs/async.h glusterfs/glusterfs-fops.h
 
-libglusterfs_la_SOURCES = dict.c graph.lex.c y.tab.c xlator.c logging.c  hashfn.c defaults.c common-utils.c timer.c inode.c call-stub.c compat.c fd.c compat-errno.c event.c mem-pool.c gf-dirent.c syscall.c iobuf.c globals.c statedump.c stack.c checksum.c $(CONTRIBDIR)/md5/md5.c $(CONTRIBDIR)/rbtree/rb.c rbthash.c latency.c graph.c $(CONTRIBDIR)/uuid/clear.c $(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c $(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/tst_uuid.c $(CONTRIBDIR)/uuid/parse.c $(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c $(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c $(CONTRIBDIR)/uuid/unpack.c syncop.c graph-print.c trie.c
+libglusterfs_ladir = $(includedir)/glusterfs
 
-noinst_HEADERS = common-utils.h defaults.h dict.h glusterfs.h hashfn.h logging.h  xlator.h  stack.h timer.h list.h inode.h call-stub.h compat.h fd.h revision.h compat-errno.h event.h mem-pool.h byte-order.h gf-dirent.h locking.h syscall.h iobuf.h globals.h statedump.h checksum.h $(CONTRIBDIR)/md5/md5.h $(CONTRIBDIR)/rbtree/rb.h rbthash.h iatt.h latency.h mem-types.h $(CONTRIBDIR)/uuid/uuidd.h $(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h $(CONTRIBDIR)/uuid/uuid_types.h syncop.h graph-utils.h graph-mem-types.h trie.h trie-mem-types.h
+noinst_HEADERS = unittest/unittest.h \
+	$(CONTRIBDIR)/rbtree/rb.h \
+	$(CONTRIBDIR)/mount/mntent_compat.h \
+	$(CONTRIBDIR)/libexecinfo/execinfo_compat.h \
+	$(CONTRIBDIR)/timer-wheel/timer-wheel.h \
+	$(CONTRIBDIR)/xxhash/xxhash.h \
+	$(CONTRIBDIR)/userspace-rcu/wfcqueue.h \
+	$(CONTRIBDIR)/userspace-rcu/wfstack.h \
+	$(CONTRIBDIR)/userspace-rcu/static-wfcqueue.h \
+	$(CONTRIBDIR)/userspace-rcu/static-wfstack.h
 
-EXTRA_DIST = graph.l graph.y
+eventtypes.h: $(top_srcdir)/events/eventskeygen.py
+	$(PYTHON) $(top_srcdir)/events/eventskeygen.py C_HEADER
+
+if BUILD_EVENTS
+libglusterfs_la_SOURCES += events.c
+endif
+
+libgfchangelog_HEADERS = changelog.h
+
+EXTRA_DIST = graph.l graph.y defaults-tmpl.c libglusterfs.sym
 
 graph.lex.c: graph.l y.tab.h
-	$(LEX) -t $(srcdir)/graph.l > $@
+	$(LEX) -Pgraphyy -t $(srcdir)/graph.l > $@
+
+y.tab.c: y.tab.h
+y.tab.h: graph.y
+	$(YACC) -p graphyy -d $(srcdir)/graph.y
+
+defaults.c: defaults-tmpl.c generator.py gen-defaults.py
+	$(PYTHON) $(srcdir)/gen-defaults.py $(srcdir)/defaults-tmpl.c > $@
+
+CLEANFILES = $(nodist_libglusterfs_la_SOURCES) \
+	$(nodist_libglusterfs_la_HEADERS) *.pyc
 
-y.tab.c y.tab.h: graph.y
-	$(YACC) -d $(srcdir)/graph.y
+if UNITTEST
+CLEANFILES += *.gcda *.gcno *_xunit.xml
+noinst_PROGRAMS =
+TESTS =
+endif
 
-CLEANFILES = graph.lex.c y.tab.c y.tab.h
+if BUILD_EVENTS
+CLEANFILES += eventtypes.h
+endif
diff --git a/libglusterfs/src/async.c b/libglusterfs/src/async.c
new file mode 100644
index 00000000000..1d6cfa374b6
--- /dev/null
+++ b/libglusterfs/src/async.c
@@ -0,0 +1,720 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc <https://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* To implement an efficient thread pool with minimum contention we have used
+ * the following ideas:
+ *
+ *    - The queue of jobs has been implemented using a Wait-Free queue provided
+ *      by the userspace-rcu library. This queue requires a mutex when multiple
+ *      consumers can be extracting items from it concurrently, but the locked
+ *      region is very small, which minimizes the chances of contention. To
+ *      further minimize contention, the number of active worker threads that
+ *      are accessing the queue is dynamically adjusted so that we always have
+ *      the minimum required amount of workers contending for the queue. Adding
+ *      new items can be done with a single atomic operation, without locks.
+ *
+ *    - All queue management operations, like creating more threads, enabling
+ *      sleeping ones, etc. are done by a single thread. This makes it possible
+ *      to manage all scaling related information and workers lists without
+ *      locks. This functionality is implemented as a role that can be assigned
+ *      to any of the worker threads, which avoids that some lengthy operations
+ *      could interfere with this task.
+ *
+ *    - Management is based on signals. We used signals for management tasks to
+ *      avoid multiple system calls for each request (with signals we can wait
+ *      for multiple events and get some additional data for each request in a
+ *      single call, instead of first polling and then reading).
+ *
+ * TODO: There are some other changes that can take advantage of this new
+ *       thread pool.
+ *
+ *          - Use this thread pool as the core threading model for synctasks. I
+ *            think this would improve synctask performance because I think we
+ *            currently have some contention there for some workloads.
+ *
+ *          - Implement a per thread timer that will allow adding and removing
+ *            timers without using mutexes.
+ *
+ *          - Integrate with userspace-rcu library in QSBR mode, allowing
+ *            other portions of code to be implemented using RCU-based
+ *            structures with a extremely fast read side without contention.
+ *
+ *          - Integrate I/O into the thread pool so that the thread pool is
+ *            able to efficiently manage all loads and scale dynamically. This
+ *            could make it possible to minimize context switching when serving
+ *            requests from fuse or network.
+ *
+ *          - Dynamically scale the number of workers based on system load.
+ *            This will make it possible to reduce contention when system is
+ *            heavily loaded, improving performance under these circumstances
+ *            (or minimizing performance loss). This will also make it possible
+ *            that gluster can coexist with other processes that also consume
+ *            CPU, with minimal interference from each other.
+ */
+
+#include <unistd.h>
+#include <pthread.h>
+#include <errno.h>
+
+#include "glusterfs/list.h"
+#include "glusterfs/mem-types.h"
+#include "glusterfs/async.h"
+
+/* These macros wrap a simple system/library call to check the returned error
+ * and log a message in case of failure. */
+#define GF_ASYNC_CHECK(_func, _args...)                                        \
+    ({                                                                         \
+        int32_t __async_error = -_func(_args);                                 \
+        if (caa_unlikely(__async_error != 0)) {                                \
+            gf_async_error(__async_error, #_func "() failed.");                \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+#define GF_ASYNC_CHECK_ERRNO(_func, _args...)                                  \
+    ({                                                                         \
+        int32_t __async_error = _func(_args);                                  \
+        if (caa_unlikely(__async_error < 0)) {                                 \
+            __async_error = -errno;                                            \
+            gf_async_error(__async_error, #_func "() failed.");                \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+/* These macros are used when, based on POSIX documentation, the function
+ * should never fail under the conditions we are using it. So any unexpected
+ * error will be handled as a fatal event. It probably means a critical bug
+ * or memory corruption. In both cases we consider that stopping the process
+ * is safer (otherwise it could cause more corruption with unknown effects
+ * that could be worse). */
+#define GF_ASYNC_CANTFAIL(_func, _args...)                                     \
+    do {                                                                       \
+        int32_t __async_error = -_func(_args);                                 \
+        if (caa_unlikely(__async_error != 0)) {                                \
+            gf_async_fatal(__async_error, #_func "() failed");                 \
+        }                                                                      \
+    } while (0)
+
+#define GF_ASYNC_CANTFAIL_ERRNO(_func, _args...)                               \
+    ({                                                                         \
+        int32_t __async_error = _func(_args);                                  \
+        if (caa_unlikely(__async_error < 0)) {                                 \
+            __async_error = -errno;                                            \
+            gf_async_fatal(__async_error, #_func "() failed");                 \
+        }                                                                      \
+        __async_error;                                                         \
+    })
+
+/* TODO: for now we allocate a static array of workers. There's an issue if we
+ *       try to use dynamic memory since these workers are initialized very
+ *       early in the process startup and it seems that sometimes not all is
+ *       ready to use dynamic memory. */
+static gf_async_worker_t gf_async_workers[GF_ASYNC_MAX_THREADS];
+
+/* This is the only global variable needed to manage the entire framework. */
+gf_async_control_t gf_async_ctrl = {};
+
+static __thread gf_async_worker_t *gf_async_current_worker = NULL;
+
+/* The main function of the worker threads. */
+static void *
+gf_async_worker(void *arg);
+
+static void
+gf_async_sync_init(void)
+{
+    GF_ASYNC_CANTFAIL(pthread_barrier_init, &gf_async_ctrl.sync, NULL, 2);
+}
+
+static void
+gf_async_sync_now(void)
+{
+    int32_t ret;
+
+    ret = pthread_barrier_wait(&gf_async_ctrl.sync);
+    if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
+        GF_ASYNC_CANTFAIL(pthread_barrier_destroy, &gf_async_ctrl.sync);
+        ret = 0;
+    }
+    if (caa_unlikely(ret != 0)) {
+        gf_async_fatal(-ret, "pthread_barrier_wait() failed");
+    }
+}
+
+static void
+gf_async_sigmask_empty(sigset_t *mask)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigemptyset, mask);
+}
+
+static void
+gf_async_sigmask_add(sigset_t *mask, int32_t signal)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigaddset, mask, signal);
+}
+
+static void
+gf_async_sigmask_set(int32_t mode, sigset_t *mask, sigset_t *old)
+{
+    GF_ASYNC_CANTFAIL(pthread_sigmask, mode, mask, old);
+}
+
+static void
+gf_async_sigaction(int32_t signum, const struct sigaction *action,
+                   struct sigaction *old)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(sigaction, signum, action, old);
+}
+
+static int32_t
+gf_async_sigwait(sigset_t *set)
+{
+    int32_t ret, signum;
+
+    do {
+        ret = sigwait(set, &signum);
+    } while (caa_unlikely((ret < 0) && (errno == EINTR)));
+
+    if (caa_unlikely(ret < 0)) {
+        ret = -errno;
+        gf_async_fatal(ret, "sigwait() failed");
+    }
+
+    return signum;
+}
+
+static int32_t
+gf_async_sigtimedwait(sigset_t *set, struct timespec *timeout)
+{
+    int32_t ret;
+
+    do {
+        ret = sigtimedwait(set, NULL, timeout);
+    } while (caa_unlikely((ret < 0) && (errno == EINTR)));
+    if (caa_unlikely(ret < 0)) {
+        ret = -errno;
+        /* EAGAIN means that the timeout has expired, so we allow this error.
+         * Any other error shouldn't happen. */
+        if (caa_unlikely(ret != -EAGAIN)) {
+            gf_async_fatal(ret, "sigtimedwait() failed");
+        }
+        ret = 0;
+    }
+
+    return ret;
+}
+
+static void
+gf_async_sigbroadcast(int32_t signum)
+{
+    GF_ASYNC_CANTFAIL_ERRNO(kill, gf_async_ctrl.pid, signum);
+}
+
+static void
+gf_async_signal_handler(int32_t signum)
+{
+    /* We should never handle a signal in this function. */
+    gf_async_fatal(-EBUSY,
+                   "Unexpected processing of signal %d through a handler.",
+                   signum);
+}
+
+static void
+gf_async_signal_setup(void)
+{
+    struct sigaction action;
+
+    /* We configure all related signals so that we can detect threads using an
+     * invalid signal mask that doesn't block our critical signal. */
+    memset(&action, 0, sizeof(action));
+    action.sa_handler = gf_async_signal_handler;
+
+    gf_async_sigaction(GF_ASYNC_SIGCTRL, &action, &gf_async_ctrl.handler_ctrl);
+
+    gf_async_sigaction(GF_ASYNC_SIGQUEUE, &action,
+                       &gf_async_ctrl.handler_queue);
+}
+
+static void
+gf_async_signal_restore(void)
+{
+    /* Handlers we have previously changed are restored back to their original
+     * value. */
+
+    if (gf_async_ctrl.handler_ctrl.sa_handler != gf_async_signal_handler) {
+        gf_async_sigaction(GF_ASYNC_SIGCTRL, &gf_async_ctrl.handler_ctrl, NULL);
+    }
+
+    if (gf_async_ctrl.handler_queue.sa_handler != gf_async_signal_handler) {
+        gf_async_sigaction(GF_ASYNC_SIGQUEUE, &gf_async_ctrl.handler_queue,
+                           NULL);
+    }
+}
+
+static void
+gf_async_signal_flush(void)
+{
+    struct timespec delay;
+
+    delay.tv_sec = 0;
+    delay.tv_nsec = 0;
+
+    /* We read all pending signals so that they don't trigger once the signal
+     * mask of some thread is changed. */
+    while (gf_async_sigtimedwait(&gf_async_ctrl.sigmask_ctrl, &delay) > 0) {
+    }
+    while (gf_async_sigtimedwait(&gf_async_ctrl.sigmask_queue, &delay) > 0) {
+    }
+}
+
+static int32_t
+gf_async_thread_create(pthread_t *thread, int32_t id, void *data)
+{
+    int32_t ret;
+
+    ret = gf_thread_create(thread, NULL, gf_async_worker, data,
+                           GF_ASYNC_THREAD_NAME "%u", id);
+    if (caa_unlikely(ret < 0)) {
+        /* TODO: gf_thread_create() should return a more specific error
+         *       code. */
+        return -ENOMEM;
+    }
+
+    return 0;
+}
+
+static void
+gf_async_thread_wait(pthread_t thread)
+{
+    /* TODO: this is a blocking call executed inside one of the workers of the
+     *       thread pool. This is bad, but this is only executed once we have
+     *       received a notification from the thread that it's terminating, so
+     *       this should return almost immediately. However, to be more robust
+     *       it would be better to use pthread_timedjoin_np() (or even a call
+     *       to pthread_tryjoin_np() followed by a delayed recheck if it
+     *       fails), but they are not portable. We should see how to do this
+     *       in other platforms. */
+    GF_ASYNC_CANTFAIL(pthread_join, thread, NULL);
+}
+
+static int32_t
+gf_async_worker_create(void)
+{
+    struct cds_wfs_node *node;
+    gf_async_worker_t *worker;
+    uint32_t counts, running, max;
+    int32_t ret;
+
+    node = __cds_wfs_pop_blocking(&gf_async_ctrl.available);
+    if (caa_unlikely(node == NULL)) {
+        /* There are no more available workers. We have all threads running. */
+        return 1;
+    }
+    cds_wfs_node_init(node);
+
+    ret = 1;
+
+    counts = uatomic_read(&gf_async_ctrl.counts);
+    max = uatomic_read(&gf_async_ctrl.max_threads);
+    running = GF_ASYNC_COUNT_RUNNING(counts);
+    if (running < max) {
+        uatomic_add(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(1, 0));
+
+        worker = caa_container_of(node, gf_async_worker_t, stack);
+
+        ret = gf_async_thread_create(&worker->thread, worker->id, worker);
+        if (caa_likely(ret >= 0)) {
+            return 0;
+        }
+
+        uatomic_add(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(-1, 0));
+    }
+
+    cds_wfs_push(&gf_async_ctrl.available, node);
+
+    return ret;
+}
+
+static void
+gf_async_worker_enable(void)
+{
+    /* This will wake one of the spare workers. If all workers are busy now,
+     * the signal will be queued so that the first one that completes its
+     * work will become the leader. */
+    gf_async_sigbroadcast(GF_ASYNC_SIGCTRL);
+
+    /* We have consumed a spare worker. We create another one for future
+     * needs. */
+    gf_async_worker_create();
+}
+
+static void
+gf_async_worker_wait(void)
+{
+    int32_t signum;
+
+    signum = gf_async_sigwait(&gf_async_ctrl.sigmask_ctrl);
+    if (caa_unlikely(signum != GF_ASYNC_SIGCTRL)) {
+        gf_async_fatal(-EINVAL, "Worker received an unexpected signal (%d)",
+                       signum);
+    }
+}
+
+static void
+gf_async_leader_wait(void)
+{
+    int32_t signum;
+
+    signum = gf_async_sigwait(&gf_async_ctrl.sigmask_queue);
+    if (caa_unlikely(signum != GF_ASYNC_SIGQUEUE)) {
+        gf_async_fatal(-EINVAL, "Leader received an unexpected signal (%d)",
+                       signum);
+    }
+}
+
+static void
+gf_async_run(struct cds_wfcq_node *node)
+{
+    gf_async_t *async;
+
+    /* We've just got work from the queue. Process it. */
+    async = caa_container_of(node, gf_async_t, queue);
+    /* TODO: remove dependency from THIS and xl. */
+    THIS = async->xl;
+    async->cbk(async->xl, async);
+}
+
+static void
+gf_async_worker_run(void)
+{
+    struct cds_wfcq_node *node;
+
+    do {
+        /* We keep executing jobs from the queue while it's not empty. Note
+         * that while we do this, we are ignoring any stop request. That's
+         * fine, since we need to process our own 'join' messages to fully
+         * terminate all threads. Note that normal jobs should have already
+         * completed once a stop request is received. */
+        node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                         &gf_async_ctrl.queue.tail);
+        if (node != NULL) {
+            gf_async_run(node);
+        }
+    } while (node != NULL);
+
+    /* TODO: I've tried to keep the worker looking at the queue for some small
+     *       amount of time in a busy loop to see if more jobs come soon. With
+     *       this I attempted to avoid the overhead of signal management if
+     *       jobs come fast enough. However experimental results seem to
+     *       indicate that doing this, CPU utilization grows and performance
+     *       is actually reduced. We need to see if that's because I used bad
+     *       parameters or it's really better to do it as it's done now. */
+}
+
+static void
+gf_async_leader_run(void)
+{
+    struct cds_wfcq_node *node;
+
+    node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                     &gf_async_ctrl.queue.tail);
+    while (caa_unlikely(node == NULL)) {
+        gf_async_leader_wait();
+
+        node = cds_wfcq_dequeue_blocking(&gf_async_ctrl.queue.head,
+                                         &gf_async_ctrl.queue.tail);
+    }
+
+    /* Activate the next available worker thread. It will become the new
+     * leader. */
+    gf_async_worker_enable();
+
+    gf_async_run(node);
+}
+
+static uint32_t
+gf_async_stop_check(gf_async_worker_t *worker)
+{
+    uint32_t counts, old, running, max;
+
+    /* First we check if we should stop without doing any costly atomic
+     * operation. */
+    old = uatomic_read(&gf_async_ctrl.counts);
+    max = uatomic_read(&gf_async_ctrl.max_threads);
+    running = GF_ASYNC_COUNT_RUNNING(old);
+    while (running > max) {
+        /* There are too many threads. We try to stop the current worker. */
+        counts = uatomic_cmpxchg(&gf_async_ctrl.counts, old,
+                                 old + GF_ASYNC_COUNTS(-1, 1));
+        if (old != counts) {
+            /* Another thread has just updated the counts. We need to retry. */
+            old = counts;
+            running = GF_ASYNC_COUNT_RUNNING(old);
+
+            continue;
+        }
+
+        running--;
+        worker->running = false;
+    }
+
+    return running;
+}
+
+static void
+gf_async_stop_all(xlator_t *xl, gf_async_t *async)
+{
+    if (gf_async_stop_check(gf_async_current_worker) > 0) {
+        /* There are more workers running. We propagate the stop request to
+         * them. */
+        gf_async(async, xl, gf_async_stop_all);
+    }
+}
+
+static void
+gf_async_join(xlator_t *xl, gf_async_t *async)
+{
+    gf_async_worker_t *worker;
+
+    worker = caa_container_of(async, gf_async_worker_t, async);
+
+    gf_async_thread_wait(worker->thread);
+
+    cds_wfs_push(&gf_async_ctrl.available, &worker->stack);
+}
+
+static void
+gf_async_terminate(gf_async_worker_t *worker)
+{
+    uint32_t counts;
+
+    counts = uatomic_add_return(&gf_async_ctrl.counts, GF_ASYNC_COUNTS(0, -1));
+    if (counts == 0) {
+        /* This is the termination of the last worker thread. We need to
+         * synchronize the main thread that is waiting for all workers to
+         * finish. */
+        gf_async_ctrl.sync_thread = worker->thread;
+
+        gf_async_sync_now();
+    } else {
+        /* Force someone else to join this thread to release resources. */
+        gf_async(&worker->async, THIS, gf_async_join);
+    }
+}
+
+static void *
+gf_async_worker(void *arg)
+{
+    gf_async_worker_t *worker;
+
+    worker = (gf_async_worker_t *)arg;
+    gf_async_current_worker = worker;
+
+    worker->running = true;
+    do {
+        /* This thread does nothing until someone enables it to become a
+         * leader. */
+        gf_async_worker_wait();
+
+        /* This thread is now a leader. It will process jobs from the queue
+         * and, if necessary, enable another worker and transfer leadership
+         * to it. */
+        gf_async_leader_run();
+
+        /* This thread is not a leader anymore. It will continue processing
+         * queued jobs until it becomes empty. */
+        gf_async_worker_run();
+
+        /* Stop the current thread if there are too many threads running. */
+        gf_async_stop_check(worker);
+    } while (worker->running);
+
+    gf_async_terminate(worker);
+
+    return NULL;
+}
+
+static void
+gf_async_cleanup(void)
+{
+    /* We do some basic initialization of the global variable 'gf_async_ctrl'
+     * so that it's put into a relatively consistent state. */
+
+    gf_async_ctrl.enabled = false;
+
+    gf_async_ctrl.pid = 0;
+    gf_async_sigmask_empty(&gf_async_ctrl.sigmask_ctrl);
+    gf_async_sigmask_empty(&gf_async_ctrl.sigmask_queue);
+
+    /* This is used to later detect if the handler of these signals have been
+     * changed or not. */
+    gf_async_ctrl.handler_ctrl.sa_handler = gf_async_signal_handler;
+    gf_async_ctrl.handler_queue.sa_handler = gf_async_signal_handler;
+
+    gf_async_ctrl.table = NULL;
+    gf_async_ctrl.max_threads = 0;
+    gf_async_ctrl.counts = 0;
+}
+
+void
+gf_async_fini(void)
+{
+    gf_async_t async;
+
+    if (uatomic_read(&gf_async_ctrl.counts) != 0) {
+        /* We ensure that all threads will quit on the next check. */
+        gf_async_ctrl.max_threads = 0;
+
+        /* Send the stop request to the thread pool. This will cause the
+         * execution of gf_async_stop_all() by one of the worker threads which,
+         * eventually, will terminate all worker threads. */
+        gf_async(&async, THIS, gf_async_stop_all);
+
+        /* We synchronize here with the last thread. */
+        gf_async_sync_now();
+
+        /* We have just synchronized with the latest thread. Now just wait for
+         * it to terminate. */
+        gf_async_thread_wait(gf_async_ctrl.sync_thread);
+
+        gf_async_signal_flush();
+    }
+
+    gf_async_signal_restore();
+
+    gf_async_cleanup();
+}
+
+void
+gf_async_adjust_threads(int32_t threads)
+{
+    if (threads == 0) {
+        /* By default we allow a maximum of 2 * #cores worker threads. This
+         * value is to try to accommodate threads that will do some I/O. Having
+         * more threads than cores we can keep CPU busy even if some threads
+         * are blocked for I/O. In the most efficient case, we can have #cores
+         * computing threads and #cores blocked threads on I/O. However this is
+         * hard to achieve because we can end with more than #cores computing
+         * threads, which won't provide a real benefit and will increase
+         * contention.
+         *
+         * TODO: implement a more intelligent dynamic maximum based on CPU
+         *       usage and/or system load. */
+        threads = sysconf(_SC_NPROCESSORS_ONLN) * 2;
+        if (threads < 0) {
+            /* If we can't get the current number of processors, we pick a
+             * random number. */
+            threads = 16;
+        }
+    }
+    if (threads > GF_ASYNC_MAX_THREADS) {
+        threads = GF_ASYNC_MAX_THREADS;
+    }
+    uatomic_set(&gf_async_ctrl.max_threads, threads);
+}
+
+int32_t
+gf_async_init(glusterfs_ctx_t *ctx)
+{
+    sigset_t set;
+    gf_async_worker_t *worker;
+    uint32_t i;
+    int32_t ret;
+    bool running;
+
+    gf_async_cleanup();
+
+    if (!ctx->cmd_args.global_threading ||
+        (ctx->process_mode == GF_GLUSTERD_PROCESS)) {
+        return 0;
+    }
+
+    /* At the init time, the maximum number of threads has not yet been
+     * configured. We use a small starting value that will be layer dynamically
+     * adjusted when ctx->config.max_threads is updated. */
+    gf_async_adjust_threads(GF_ASYNC_SPARE_THREADS + 1);
+
+    gf_async_ctrl.pid = getpid();
+
+    __cds_wfs_init(&gf_async_ctrl.available);
+    cds_wfcq_init(&gf_async_ctrl.queue.head, &gf_async_ctrl.queue.tail);
+
+    gf_async_sync_init();
+
+    /* TODO: it would be cleaner to use dynamic memory, but at this point some
+     *       memory management resources are not yet initialized. */
+    gf_async_ctrl.table = gf_async_workers;
+
+    /* We keep all workers in a stack. It will be used when a new thread needs
+     * to be created. */
+    for (i = GF_ASYNC_MAX_THREADS; i > 0; i--) {
+        worker = &gf_async_ctrl.table[i - 1];
+
+        worker->id = i - 1;
+        cds_wfs_node_init(&worker->stack);
+        cds_wfs_push(&gf_async_ctrl.available, &worker->stack);
+    }
+
+    /* Prepare the signal mask for regular workers and the leader. */
+    gf_async_sigmask_add(&gf_async_ctrl.sigmask_ctrl, GF_ASYNC_SIGCTRL);
+    gf_async_sigmask_add(&gf_async_ctrl.sigmask_queue, GF_ASYNC_SIGQUEUE);
+
+    /* TODO: this is needed to block our special signals in the current thread
+     *       and all children that it starts. It would be cleaner to do it when
+     *       signals are initialized, but there doesn't seem to be a unique
+     *       place to do that, so for now we do it here. */
+    gf_async_sigmask_empty(&set);
+    gf_async_sigmask_add(&set, GF_ASYNC_SIGCTRL);
+    gf_async_sigmask_add(&set, GF_ASYNC_SIGQUEUE);
+    gf_async_sigmask_set(SIG_BLOCK, &set, NULL);
+
+    /* Configure the signal handlers. This is mostly for safety, not really
+     * needed, but it doesn't hurt. Note that the caller must ensure that the
+     * signals we need to run are already blocked in any thread already
+     * started. Otherwise this won't work. */
+    gf_async_signal_setup();
+
+    running = false;
+
+    /* We start the spare workers + 1 for the leader. */
+    for (i = 0; i < GF_ASYNC_SPARE_THREADS; i++) {
+        ret = gf_async_worker_create();
+        if (caa_unlikely(ret < 0)) {
+            /* This is the initial start up so we enforce that the spare
+             * threads are created. If this fails at the beginning, it's very
+             * unlikely that the async workers could do its job, so we abort
+             * the initialization. */
+            goto out;
+        }
+
+        /* Once the first thread is started, we can enable it to become the
+         * initial leader. */
+        if ((ret == 0) && !running) {
+            running = true;
+            gf_async_worker_enable();
+        }
+    }
+
+    if (caa_unlikely(!running)) {
+        gf_async_fatal(-ENOMEM, "No worker thread has started");
+    }
+
+    gf_async_ctrl.enabled = true;
+
+    ret = 0;
+
+out:
+    if (ret < 0) {
+        gf_async_error(ret, "Unable to initialize the thread pool.");
+        gf_async_fini();
+    }
+
+    return ret;
+}
diff --git a/libglusterfs/src/byte-order.h b/libglusterfs/src/byte-order.h
deleted file mode 100644
index cabfcf4c63c..00000000000
--- a/libglusterfs/src/byte-order.h
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _BYTE_ORDER_H
-#define _BYTE_ORDER_H
-
-#include <inttypes.h>
-
-#define LS1 0x00ffU
-#define MS1 0xff00U
-#define LS2 0x0000ffffU
-#define MS2 0xffff0000U
-#define LS4 0x00000000ffffffffULL
-#define MS4 0xffffffff00000000ULL
-
-
-static uint16_t (*hton16) (uint16_t);
-static uint32_t (*hton32) (uint32_t);
-static uint64_t (*hton64) (uint64_t);
-
-#define ntoh16 hton16
-#define ntoh32 hton32
-#define ntoh64 hton64
-
-#define do_swap2(x) (((x&LS1) << 8)|(((x&MS1) >> 8)))
-#define do_swap4(x) ((do_swap2(x&LS2) << 16)|(do_swap2((x&MS2) >> 16)))
-#define do_swap8(x) ((do_swap4(x&LS4) << 32)|(do_swap4((x&MS4) >> 32)))
-
-
-static inline uint16_t
-__swap16 (uint16_t x)
-{
-	return do_swap2(x);
-}
-
-
-static inline uint32_t
-__swap32 (uint32_t x)
-{
-	return do_swap4(x);
-}
-
-
-static inline uint64_t
-__swap64 (uint64_t x)
-{
-	return do_swap8(x);
-}
-
-
-static inline uint16_t
-__noswap16 (uint16_t x)
-{
-	return x;
-}
-
-
-static inline uint32_t
-__noswap32 (uint32_t x)
-{
-	return x;
-}
-
-
-static inline uint64_t
-__noswap64 (uint64_t x)
-{
-	return x;
-}
-
-
-static inline uint16_t
-__byte_order_init16 (uint16_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton16 (i);
-}
-
-
-static inline uint32_t
-__byte_order_init32 (uint32_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton32 (i);
-}
-
-
-static inline uint64_t
-__byte_order_init64 (uint64_t i)
-{
-	uint32_t num = 1;
-
-	if (((char *)(&num))[0] == 1) {
-		hton16 = __swap16;
-		hton32 = __swap32;
-		hton64 = __swap64;
-	} else {
-		hton16 = __noswap16;
-		hton32 = __noswap32;
-		hton64 = __noswap64;
-	}
-
-	return hton64 (i);
-}
-
-
-static uint16_t (*hton16) (uint16_t) = __byte_order_init16;
-static uint32_t (*hton32) (uint32_t) = __byte_order_init32;
-static uint64_t (*hton64) (uint64_t) = __byte_order_init64;
-
-
-#endif /* _BYTE_ORDER_H */
diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c
index d3b29a39ad7..ee84f08acd4 100644
--- a/libglusterfs/src/call-stub.c
+++ b/libglusterfs/src/call-stub.c
@@ -1,3881 +1,2467 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
+#include <openssl/md5.h>
 #include <inttypes.h>
 
-#include "md5.h"
-#include "call-stub.h"
-#include "mem-types.h"
-
+#include "glusterfs/call-stub.h"
+#include "glusterfs/mem-types.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 static call_stub_t *
-stub_new (call_frame_t *frame,
-	  char wind,
-	  glusterfs_fop_t fop)
+stub_new(call_frame_t *frame, const char wind, const glusterfs_fop_t fop)
 {
-	call_stub_t *new = NULL;
+    call_stub_t *new = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", frame, out);
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    new = mem_get0(frame->this->ctx->stub_mem_pool);
+    GF_VALIDATE_OR_GOTO("call-stub", new, out);
 
-       new = mem_get0 (frame->this->ctx->stub_mem_pool);
-	GF_VALIDATE_OR_GOTO ("call-stub", new, out);
+    new->frame = frame;
+    new->wind = wind;
+    new->fop = fop;
+    new->stub_mem_pool = frame->this->ctx->stub_mem_pool;
+    INIT_LIST_HEAD(&new->list);
 
-	new->frame = frame;
-	new->wind = wind;
-	new->fop = fop;
-       new->stub_mem_pool = frame->this->ctx->stub_mem_pool;
-	INIT_LIST_HEAD (&new->list);
+    INIT_LIST_HEAD(&new->args_cbk.entries);
 out:
-	return new;
+    return new;
 }
 
-
 call_stub_t *
-fop_lookup_stub (call_frame_t *frame,
-		 fop_lookup_t fn,
-		 loc_t *loc,
-		 dict_t *xattr_req)
+fop_lookup_stub(call_frame_t *frame, fop_lookup_t fn, loc_t *loc, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_LOOKUP);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LOOKUP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.lookup.fn = fn;
+    stub->fn.lookup = fn;
+    args_lookup_store(&stub->args, loc, xdata);
+out:
+    return stub;
+}
+
+call_stub_t *
+fop_lookup_cbk_stub(call_frame_t *frame, fop_lookup_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode, struct iatt *buf,
+                    dict_t *xdata, struct iatt *postparent)
+{
+    call_stub_t *stub = NULL;
 
-	if (xattr_req)
-		stub->args.lookup.xattr_req = dict_ref (xattr_req);
+    stub = stub_new(frame, 0, GF_FOP_LOOKUP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	loc_copy (&stub->args.lookup.loc, loc);
+    stub->fn_cbk.lookup = fn;
+    args_lookup_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf, xdata,
+                          postparent);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame,
-		     fop_lookup_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
+fop_stat_stub(call_frame_t *frame, fop_stat_t fn, loc_t *loc, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_LOOKUP);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_STAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.lookup_cbk.fn = fn;
-	stub->args.lookup_cbk.op_ret = op_ret;
-	stub->args.lookup_cbk.op_errno = op_errno;
-	if (inode)
-		stub->args.lookup_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.lookup_cbk.buf = *buf;
-	if (dict)
-		stub->args.lookup_cbk.dict = dict_ref (dict);
-        if (postparent)
-                stub->args.lookup_cbk.postparent = *postparent;
+    stub->fn.stat = fn;
+    args_stat_store(&stub->args, loc, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
-
 call_stub_t *
-fop_stat_stub (call_frame_t *frame,
-	       fop_stat_t fn,
-	       loc_t *loc)
+fop_stat_cbk_stub(call_frame_t *frame, fop_stat_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-  
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_STAT);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_STAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.stat.fn = fn;
-	loc_copy (&stub->args.stat.loc, loc);
+    stub->fn_cbk.stat = fn;
+    args_stat_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame,
-		   fop_stat_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf)
+fop_fstat_stub(call_frame_t *frame, fop_fstat_t fn, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-	
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_STAT);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSTAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.stat_cbk.fn = fn;
-	stub->args.stat_cbk.op_ret = op_ret;
-	stub->args.stat_cbk.op_errno = op_errno;
-	if (op_ret == 0)
-		stub->args.stat_cbk.buf = *buf;
+    stub->fn.fstat = fn;
+    args_fstat_store(&stub->args, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fstat_stub (call_frame_t *frame,
-		fop_fstat_t fn,
-		fd_t *fd)
+fop_fstat_cbk_stub(call_frame_t *frame, fop_fstat_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-
-	stub = stub_new (frame, 1, GF_FOP_FSTAT);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    call_stub_t *stub = NULL;
 
-	stub->args.fstat.fn = fn;
+    stub = stub_new(frame, 0, GF_FOP_FSTAT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	if (fd)
-		stub->args.fstat.fd = fd_ref (fd);
+    stub->fn_cbk.fstat = fn;
+    args_fstat_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame,
-		    fop_fstat_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *buf)
+fop_truncate_stub(call_frame_t *frame, fop_truncate_t fn, loc_t *loc, off_t off,
+                  dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FSTAT);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_TRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fstat_cbk.fn = fn;
-	stub->args.fstat_cbk.op_ret = op_ret;
-	stub->args.fstat_cbk.op_errno = op_errno;
-	if (buf)
-		stub->args.fstat_cbk.buf = *buf;
+    stub->fn.truncate = fn;
+    args_truncate_store(&stub->args, loc, off, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_truncate_cbk_stub(call_frame_t *frame, fop_truncate_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-/* truncate */
+    stub = stub_new(frame, 0, GF_FOP_TRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.truncate = fn;
+    args_truncate_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_truncate_stub (call_frame_t *frame,
-		   fop_truncate_t fn,
-		   loc_t *loc,
-		   off_t off)
+fop_ftruncate_stub(call_frame_t *frame, fop_ftruncate_t fn, fd_t *fd, off_t off,
+                   dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);	
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 1, GF_FOP_FTRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_TRUNCATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.ftruncate = fn;
+    args_ftruncate_store(&stub->args, fd, off, xdata);
 
-	stub->args.truncate.fn = fn;
-	loc_copy (&stub->args.truncate.loc, loc);
-	stub->args.truncate.off = off;
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame,
-		       fop_truncate_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-                       struct iatt *postbuf)
+fop_ftruncate_cbk_stub(call_frame_t *frame, fop_ftruncate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FTRUNCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_TRUNCATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.ftruncate = fn;
+    args_ftruncate_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
-	stub->args.truncate_cbk.fn = fn;
-	stub->args.truncate_cbk.op_ret = op_ret;
-	stub->args.truncate_cbk.op_errno = op_errno;
-	if (prebuf)
-		stub->args.truncate_cbk.prebuf = *prebuf;
-        if (postbuf)
-                stub->args.truncate_cbk.postbuf = *postbuf;
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame,
-		    fop_ftruncate_t fn,
-		    fd_t *fd,
-		    off_t off)
+fop_access_stub(call_frame_t *frame, fop_access_t fn, loc_t *loc, int32_t mask,
+                dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_FTRUNCATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub->args.ftruncate.fn = fn;
-	if (fd)
-		stub->args.ftruncate.fd = fd_ref (fd);
+    stub = stub_new(frame, 1, GF_FOP_ACCESS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.ftruncate.off = off;
+    stub->fn.access = fn;
+    args_access_store(&stub->args, loc, mask, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame,
-			fop_ftruncate_cbk_t fn,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iatt *prebuf,
-                        struct iatt *postbuf)
+fop_access_cbk_stub(call_frame_t *frame, fop_access_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_ACCESS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FTRUNCATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.ftruncate_cbk.fn = fn;
-	stub->args.ftruncate_cbk.op_ret = op_ret;
-	stub->args.ftruncate_cbk.op_errno = op_errno;
-	if (prebuf)
-		stub->args.ftruncate_cbk.prebuf = *prebuf;
-	if (postbuf)
-		stub->args.ftruncate_cbk.postbuf = *postbuf;
+    stub->fn_cbk.access = fn;
+    args_access_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_access_stub (call_frame_t *frame,
-		 fop_access_t fn,
-		 loc_t *loc,
-		 int32_t mask)
+fop_readlink_stub(call_frame_t *frame, fop_readlink_t fn, loc_t *loc,
+                  size_t size, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_ACCESS);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_READLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.access.fn = fn;
-	loc_copy (&stub->args.access.loc, loc);
-	stub->args.access.mask = mask;
+    stub->fn.readlink = fn;
+    args_readlink_store(&stub->args, loc, size, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_readlink_cbk_stub(call_frame_t *frame, fop_readlink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, const char *path,
+                      struct iatt *stbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_READLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.readlink = fn;
+    args_readlink_cbk_store(&stub->args_cbk, op_ret, op_errno, path, stbuf,
+                            xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame,
-		     fop_access_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno)
+fop_mknod_stub(call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
+               dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_ACCESS);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_MKNOD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.access_cbk.fn = fn;
-	stub->args.access_cbk.op_ret = op_ret;
-	stub->args.access_cbk.op_errno = op_errno;
+    stub->fn.mknod = fn;
+    args_mknod_store(&stub->args, loc, mode, rdev, umask, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readlink_stub (call_frame_t *frame,
-		   fop_readlink_t fn,
-		   loc_t *loc,
-		   size_t size)
+fop_mknod_cbk_stub(call_frame_t *frame, fop_mknod_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
-	
-	stub = stub_new (frame, 1, GF_FOP_READLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_MKNOD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.readlink.fn = fn;
-	loc_copy (&stub->args.readlink.loc, loc);
-	stub->args.readlink.size = size;
+    stub->fn_cbk.mknod = fn;
+    args_mknod_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                         preparent, postparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame,
-		       fop_readlink_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       const char *path,
-                       struct iatt *sbuf)
+fop_mkdir_stub(call_frame_t *frame, fop_mkdir_t fn, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_READLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_MKDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.readlink_cbk.fn = fn;
-	stub->args.readlink_cbk.op_ret = op_ret;
-	stub->args.readlink_cbk.op_errno = op_errno;
-	if (path)
-		stub->args.readlink_cbk.buf = gf_strdup (path);
-        if (sbuf)
-                stub->args.readlink_cbk.sbuf = *sbuf;
+    stub->fn.mkdir = fn;
+    args_mkdir_store(&stub->args, loc, mode, umask, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn,
-		loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+fop_mkdir_cbk_stub(call_frame_t *frame, fop_mkdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 0, GF_FOP_MKDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_MKNOD);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.mknod.fn = fn;
-	loc_copy (&stub->args.mknod.loc, loc);
-	stub->args.mknod.mode = mode;
-	stub->args.mknod.rdev = rdev;
-        if (params)
-                stub->args.mknod.params = dict_ref (params);
+    stub->fn_cbk.mkdir = fn;
+    args_mkdir_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                         preparent, postparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame,
-		    fop_mknod_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_unlink_stub(call_frame_t *frame, fop_unlink_t fn, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_UNLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_MKNOD);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.unlink = fn;
+    args_unlink_store(&stub->args, loc, xflag, xdata);
 
-	stub->args.mknod_cbk.fn = fn;
-	stub->args.mknod_cbk.op_ret = op_ret;
-	stub->args.mknod_cbk.op_errno = op_errno;
-	if (inode)
-		stub->args.mknod_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.mknod_cbk.buf = *buf;
-        if (preparent)
-                stub->args.mknod_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.mknod_cbk.postparent = *postparent;
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
-		loc_t *loc, mode_t mode, dict_t *params)
+fop_unlink_cbk_stub(call_frame_t *frame, fop_unlink_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_MKDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_UNLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.mkdir.fn = fn;
-	loc_copy (&stub->args.mkdir.loc, loc);
-	stub->args.mkdir.mode = mode;
-        if (params)
-                stub->args.mkdir.params = dict_ref (params);
+    stub->fn_cbk.unlink = fn;
+    args_unlink_cbk_store(&stub->args_cbk, op_ret, op_errno, preparent,
+                          postparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame,
-		    fop_mkdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_rmdir_stub(call_frame_t *frame, fop_rmdir_t fn, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 1, GF_FOP_RMDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_MKDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn.rmdir = fn;
+    args_rmdir_store(&stub->args, loc, flags, xdata);
 
-	stub->args.mkdir_cbk.fn = fn;
-	stub->args.mkdir_cbk.op_ret = op_ret;
-	stub->args.mkdir_cbk.op_errno = op_errno;
-	if (inode)
-		stub->args.mkdir_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.mkdir_cbk.buf = *buf;
-        if (preparent)
-                stub->args.mkdir_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.mkdir_cbk.postparent = *postparent;
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_unlink_stub (call_frame_t *frame,
-		 fop_unlink_t fn,
-		 loc_t *loc)
+fop_rmdir_cbk_stub(call_frame_t *frame, fop_rmdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 0, GF_FOP_RMDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_UNLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.unlink.fn = fn;
-	loc_copy (&stub->args.unlink.loc, loc);
+    stub->fn_cbk.rmdir = fn;
+    args_rmdir_cbk_store(&stub->args_cbk, op_ret, op_errno, preparent,
+                         postparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame,
-		     fop_unlink_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
+fop_symlink_stub(call_frame_t *frame, fop_symlink_t fn, const char *linkname,
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", linkname, out);
 
-	stub = stub_new (frame, 0, GF_FOP_UNLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_SYMLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.unlink_cbk.fn = fn;
-	stub->args.unlink_cbk.op_ret = op_ret;
-	stub->args.unlink_cbk.op_errno = op_errno;
-        if (preparent)
-                stub->args.unlink_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.unlink_cbk.postparent = *postparent;
+    stub->fn.symlink = fn;
+    args_symlink_store(&stub->args, linkname, loc, umask, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_symlink_cbk_stub(call_frame_t *frame, fop_symlink_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_SYMLINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn_cbk.symlink = fn;
+    args_symlink_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-		loc_t *loc, int flags)
+fop_rename_stub(call_frame_t *frame, fop_rename_t fn, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", oldloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", newloc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_RMDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_RENAME);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.rmdir.fn = fn;
-	loc_copy (&stub->args.rmdir.loc, loc);
-        stub->args.rmdir.flags = flags;
+    stub->fn.rename = fn;
+    args_rename_store(&stub->args, oldloc, newloc, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame,
-		    fop_rmdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_rename_cbk_stub(call_frame_t *frame, fop_rename_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_RMDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_RENAME);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.rmdir_cbk.fn = fn;
-	stub->args.rmdir_cbk.op_ret = op_ret;
-	stub->args.rmdir_cbk.op_errno = op_errno;
-        if (preparent)
-                stub->args.rmdir_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.rmdir_cbk.postparent = *postparent;
+    stub->fn_cbk.rename = fn;
+    args_rename_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, preoldparent,
+                          postoldparent, prenewparent, postnewparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-		  const char *linkname, loc_t *loc, dict_t *params)
+fop_link_stub(call_frame_t *frame, fop_link_t fn, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", linkname, out);
+    GF_VALIDATE_OR_GOTO("call-stub", oldloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", newloc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_SYMLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.symlink.fn = fn;
-	stub->args.symlink.linkname = gf_strdup (linkname);
-	loc_copy (&stub->args.symlink.loc, loc);
-        if (params)
-                stub->args.symlink.params = dict_ref (params);
+    stub->fn.link = fn;
+    args_link_store(&stub->args, oldloc, newloc, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame,
-		      fop_symlink_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
+fop_link_cbk_stub(call_frame_t *frame, fop_link_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, inode_t *inode, struct iatt *buf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_LINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_SYMLINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.symlink_cbk.fn = fn;
-	stub->args.symlink_cbk.op_ret = op_ret;
-	stub->args.symlink_cbk.op_errno = op_errno;
-	if (inode)
-		stub->args.symlink_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.symlink_cbk.buf = *buf;
-        if (preparent)
-                stub->args.symlink_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.symlink_cbk.postparent = *postparent;
+    stub->fn_cbk.link = fn;
+    args_link_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                        preparent, postparent, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rename_stub (call_frame_t *frame,
-		 fop_rename_t fn,
-		 loc_t *oldloc,
-		 loc_t *newloc)
+fop_create_stub(call_frame_t *frame, fop_create_t fn, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", oldloc, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", newloc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_RENAME);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_CREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.rename.fn = fn;
-	loc_copy (&stub->args.rename.old, oldloc);
-	loc_copy (&stub->args.rename.new, newloc);
+    stub->fn.create = fn;
+    args_create_store(&stub->args, loc, flags, mode, umask, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_create_cbk_stub(call_frame_t *frame, fop_create_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_CREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.create = fn;
+    args_create_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, inode, buf,
+                          preparent, postparent, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame,
-		     fop_rename_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
+fop_open_stub(call_frame_t *frame, fop_open_t fn, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_RENAME);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_OPEN);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.rename_cbk.fn = fn;
-	stub->args.rename_cbk.op_ret = op_ret;
-	stub->args.rename_cbk.op_errno = op_errno;
-	if (buf)
-		stub->args.rename_cbk.buf = *buf;
-        if (preoldparent)
-                stub->args.rename_cbk.preoldparent = *preoldparent;
-        if (postoldparent)
-                stub->args.rename_cbk.postoldparent = *postoldparent;
-        if (prenewparent)
-                stub->args.rename_cbk.prenewparent = *prenewparent;
-        if (postnewparent)
-                stub->args.rename_cbk.postnewparent = *postnewparent;
+    stub->fn.open = fn;
+    args_open_store(&stub->args, loc, flags, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_link_stub (call_frame_t *frame,
-	       fop_link_t fn,
-	       loc_t *oldloc,
-	       loc_t *newloc)
+fop_open_cbk_stub(call_frame_t *frame, fop_open_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", oldloc, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", newloc, out);
+    stub = stub_new(frame, 0, GF_FOP_OPEN);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_LINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.open = fn;
+    args_open_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, xdata);
+out:
+    return stub;
+}
 
-	stub->args.link.fn = fn;
-	loc_copy (&stub->args.link.oldloc, oldloc);
-	loc_copy (&stub->args.link.newloc, newloc);
+call_stub_t *
+fop_readv_stub(call_frame_t *frame, fop_readv_t fn, fd_t *fd, size_t size,
+               off_t off, uint32_t flags, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
+    stub = stub_new(frame, 1, GF_FOP_READ);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.readv = fn;
+    args_readv_store(&stub->args, fd, size, off, flags, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame,
-		   fop_link_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
+fop_readv_cbk_stub(call_frame_t *frame, fop_readv_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iovec *vector, int32_t count,
+                   struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_READ);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_LINK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.link_cbk.fn = fn;
-	stub->args.link_cbk.op_ret = op_ret;
-	stub->args.link_cbk.op_errno = op_errno;
-	if (inode)
-		stub->args.link_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.link_cbk.buf = *buf;
-        if (preparent)
-                stub->args.link_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.link_cbk.postparent = *postparent;
+    stub->fn_cbk.readv = fn;
+    args_readv_cbk_store(&stub->args_cbk, op_ret, op_errno, vector, count,
+                         stbuf, iobref, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_create_stub (call_frame_t *frame, fop_create_t fn,
-		 loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params)
+fop_writev_stub(call_frame_t *frame, fop_writev_t fn, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                struct iobref *iobref, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", vector, out);
 
-	stub = stub_new (frame, 1, GF_FOP_CREATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_WRITE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.create.fn = fn;
-	loc_copy (&stub->args.create.loc, loc);
-	stub->args.create.flags = flags;
-	stub->args.create.mode = mode;
-	if (fd)
-		stub->args.create.fd = fd_ref (fd);
-        if (params)
-                stub->args.create.params = dict_ref (params);
+    stub->fn.writev = fn;
+    args_writev_store(&stub->args, fd, vector, count, off, flags, iobref,
+                      xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame,
-		     fop_create_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
+fop_writev_cbk_stub(call_frame_t *frame, fop_writev_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                    dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_CREATE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_WRITE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.create_cbk.fn = fn;
-	stub->args.create_cbk.op_ret = op_ret;
-	stub->args.create_cbk.op_errno = op_errno;
-	if (fd)
-		stub->args.create_cbk.fd = fd_ref (fd);
-	if (inode)
-		stub->args.create_cbk.inode = inode_ref (inode);
-	if (buf)
-		stub->args.create_cbk.buf = *buf;
-        if (preparent)
-                stub->args.create_cbk.preparent = *preparent;
-        if (postparent)
-                stub->args.create_cbk.postparent = *postparent;
+    stub->fn_cbk.writev = fn;
+    args_writev_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_open_stub (call_frame_t *frame,
-	       fop_open_t fn,
-	       loc_t *loc,
-	       int32_t flags, fd_t *fd,
-               int32_t wbflags)
+fop_flush_stub(call_frame_t *frame, fop_flush_t fn, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_OPEN);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FLUSH);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.open.fn = fn;
-	loc_copy (&stub->args.open.loc, loc);
-	stub->args.open.flags = flags;
-        stub->args.open.wbflags = wbflags;
-	if (fd)
-		stub->args.open.fd = fd_ref (fd);
+    stub->fn.flush = fn;
+    args_flush_store(&stub->args, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame,
-		   fop_open_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-
+fop_flush_cbk_stub(call_frame_t *frame, fop_flush_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_OPEN);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FLUSH);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.open_cbk.fn = fn;
-	stub->args.open_cbk.op_ret = op_ret;
-	stub->args.open_cbk.op_errno = op_errno;
-	if (fd)
-		stub->args.open_cbk.fd = fd_ref (fd);
+    stub->fn_cbk.flush = fn;
+    args_flush_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readv_stub (call_frame_t *frame,
-		fop_readv_t fn,
-		fd_t *fd,
-		size_t size,
-		off_t off)
+fop_fsync_stub(call_frame_t *frame, fop_fsync_t fn, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_READ);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSYNC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.readv.fn = fn;
-	if (fd)
-		stub->args.readv.fd = fd_ref (fd);
-	stub->args.readv.size = size;
-	stub->args.readv.off = off;
+    stub->fn.fsync = fn;
+    args_fsync_store(&stub->args, fd, datasync, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame,
-		    fop_readv_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iovec *vector,
-		    int32_t count,
-		    struct iatt *stbuf,
-                    struct iobref *iobref)
-
+fop_fsync_cbk_stub(call_frame_t *frame, fop_fsync_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                   dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_READ);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FSYNC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.readv_cbk.fn = fn;
-	stub->args.readv_cbk.op_ret = op_ret;
-	stub->args.readv_cbk.op_errno = op_errno;
-	if (op_ret >= 0) {
-		stub->args.readv_cbk.vector = iov_dup (vector, count);
-		stub->args.readv_cbk.count = count;
-		stub->args.readv_cbk.stbuf = *stbuf;
-		stub->args.readv_cbk.iobref = iobref_ref (iobref);
-	}
+    stub->fn_cbk.fsync = fn;
+    args_fsync_cbk_store(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_writev_stub (call_frame_t *frame,
-		 fop_writev_t fn,
-		 fd_t *fd,
-		 struct iovec *vector,
-		 int32_t count,
-		 off_t off,
-                 struct iobref *iobref)
+fop_opendir_stub(call_frame_t *frame, fop_opendir_t fn, loc_t *loc, fd_t *fd,
+                 dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", vector, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_WRITE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_OPENDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.writev.fn = fn;
-	if (fd)
-		stub->args.writev.fd = fd_ref (fd);
-	stub->args.writev.vector = iov_dup (vector, count);
-	stub->args.writev.count = count;
-	stub->args.writev.off = off;
-        stub->args.writev.iobref = iobref_ref (iobref);
+    stub->fn.opendir = fn;
+    args_opendir_store(&stub->args, loc, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame,
-		     fop_writev_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *prebuf,
-		     struct iatt *postbuf)
-
+fop_opendir_cbk_stub(call_frame_t *frame, fop_opendir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_OPENDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_WRITE);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.writev_cbk.fn = fn;
-	stub->args.writev_cbk.op_ret = op_ret;
-	stub->args.writev_cbk.op_errno = op_errno;
-	if (op_ret >= 0)
-		stub->args.writev_cbk.postbuf = *postbuf;
-        if (prebuf)
-                stub->args.writev_cbk.prebuf = *prebuf;
+    stub->fn_cbk.opendir = fn;
+    args_opendir_cbk_store(&stub->args_cbk, op_ret, op_errno, fd, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_fsyncdir_stub(call_frame_t *frame, fop_fsyncdir_t fn, fd_t *fd,
+                  int32_t datasync, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 1, GF_FOP_FSYNCDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn.fsyncdir = fn;
+    args_fsyncdir_store(&stub->args, fd, datasync, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_flush_stub (call_frame_t *frame,
-		fop_flush_t fn,
-		fd_t *fd)
+fop_fsyncdir_cbk_stub(call_frame_t *frame, fop_fsyncdir_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FSYNCDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_FLUSH);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.flush.fn = fn;
-	if (fd)
-		stub->args.flush.fd = fd_ref (fd);
+    stub->fn_cbk.fsyncdir = fn;
+    args_fsyncdir_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame,
-		    fop_flush_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno)
-
+fop_statfs_stub(call_frame_t *frame, fop_statfs_t fn, loc_t *loc, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FLUSH);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_STATFS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.flush_cbk.fn = fn;
-	stub->args.flush_cbk.op_ret = op_ret;
-	stub->args.flush_cbk.op_errno = op_errno;
+    stub->fn.statfs = fn;
+    args_statfs_store(&stub->args, loc, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_statfs_cbk_stub(call_frame_t *frame, fop_statfs_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
+    stub = stub_new(frame, 0, GF_FOP_STATFS);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn_cbk.statfs = fn;
+    args_statfs_cbk_store(&stub->args_cbk, op_ret, op_errno, buf, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_fsync_stub (call_frame_t *frame,
-		fop_fsync_t fn,
-		fd_t *fd,
-		int32_t datasync)
+fop_setxattr_stub(call_frame_t *frame, fop_setxattr_t fn, loc_t *loc,
+                  dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_FSYNC);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_SETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fsync.fn = fn;
-	if (fd)
-		stub->args.fsync.fd = fd_ref (fd);
-	stub->args.fsync.datasync = datasync;
+    stub->fn.setxattr = fn;
+    args_setxattr_store(&stub->args, loc, dict, flags, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame,
-		    fop_fsync_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *prebuf,
-                    struct iatt *postbuf)
+fop_setxattr_cbk_stub(call_frame_t *frame, fop_setxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_SETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FSYNC);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.fsync_cbk.fn = fn;
-	stub->args.fsync_cbk.op_ret = op_ret;
-	stub->args.fsync_cbk.op_errno = op_errno;
-        if (prebuf)
-                stub->args.fsync_cbk.prebuf = *prebuf;
-        if (postbuf)
-                stub->args.fsync_cbk.postbuf = *postbuf;
+    stub->fn_cbk.setxattr = fn;
+    args_setxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_opendir_stub (call_frame_t *frame,
-		  fop_opendir_t fn,
-		  loc_t *loc, fd_t *fd)
+fop_getxattr_stub(call_frame_t *frame, fop_getxattr_t fn, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
 
-	stub = stub_new (frame, 1, GF_FOP_OPENDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.opendir.fn = fn;
-	loc_copy (&stub->args.opendir.loc, loc);
-	if (fd)
-		stub->args.opendir.fd = fd_ref (fd);
+    stub->fn.getxattr = fn;
+    args_getxattr_store(&stub->args, loc, name, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame,
-		      fop_opendir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-
+fop_getxattr_cbk_stub(call_frame_t *frame, fop_getxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_OPENDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.opendir_cbk.fn = fn;
-	stub->args.opendir_cbk.op_ret = op_ret;
-	stub->args.opendir_cbk.op_errno = op_errno;
-
-	if (fd)
-		stub->args.opendir_cbk.fd = fd_ref (fd);
+    stub->fn_cbk.getxattr = fn;
+    args_getxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, dict, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame,
-		   fop_fsyncdir_t fn,
-		   fd_t *fd,
-		   int32_t datasync)
+fop_fsetxattr_stub(call_frame_t *frame, fop_fsetxattr_t fn, fd_t *fd,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-	stub = stub_new (frame, 1, GF_FOP_FSYNCDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FSETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fsyncdir.fn = fn;
-	if (fd)
-		stub->args.fsyncdir.fd = fd_ref (fd);
-	stub->args.fsyncdir.datasync = datasync;
+    stub->fn.fsetxattr = fn;
+    args_fsetxattr_store(&stub->args, fd, dict, flags, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame,
-		       fop_fsyncdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno)
-
+fop_fsetxattr_cbk_stub(call_frame_t *frame, fop_fsetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FSETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FSYNCDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.fsyncdir_cbk.fn = fn;
-	stub->args.fsyncdir_cbk.op_ret = op_ret;
-	stub->args.fsyncdir_cbk.op_errno = op_errno;
+    stub->fn_cbk.fsetxattr = fn;
+    args_fsetxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_statfs_stub (call_frame_t *frame,
-		 fop_statfs_t fn,
-		 loc_t *loc)
+fop_fgetxattr_stub(call_frame_t *frame, fop_fgetxattr_t fn, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out); 
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-	stub = stub_new (frame, 1, GF_FOP_STATFS);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FGETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.statfs.fn = fn;
-	loc_copy (&stub->args.statfs.loc, loc);
+    stub->fn.fgetxattr = fn;
+    args_fgetxattr_store(&stub->args, fd, name, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame,
-		     fop_statfs_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct statvfs *buf)
-
+fop_fgetxattr_cbk_stub(call_frame_t *frame, fop_fgetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_STATFS);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_GETXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.statfs_cbk.fn = fn;
-	stub->args.statfs_cbk.op_ret = op_ret;
-	stub->args.statfs_cbk.op_errno = op_errno;
-	if (op_ret == 0)
-		stub->args.statfs_cbk.buf = *buf;
+    stub->fn_cbk.fgetxattr = fn;
+    args_fgetxattr_cbk_store(&stub->args_cbk, op_ret, op_errno, dict, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_setxattr_stub (call_frame_t *frame,
-		   fop_setxattr_t fn,
-		   loc_t *loc,
-		   dict_t *dict,
-		   int32_t flags)
+fop_removexattr_stub(call_frame_t *frame, fop_removexattr_t fn, loc_t *loc,
+                     const char *name, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", loc, out);
+    GF_VALIDATE_OR_GOTO("call-stub", name, out);
 
-	stub = stub_new (frame, 1, GF_FOP_SETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_REMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.setxattr.fn = fn;
-	loc_copy (&stub->args.setxattr.loc, loc);
-	/* TODO */
-	if (dict)
-		stub->args.setxattr.dict = dict_ref (dict);
-	stub->args.setxattr.flags = flags;
+    stub->fn.removexattr = fn;
+    args_removexattr_store(&stub->args, loc, name, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_removexattr_cbk_stub(call_frame_t *frame, fop_removexattr_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_REMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.removexattr = fn;
+    args_removexattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_setxattr_cbk_stub (call_frame_t *frame,
-		       fop_setxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno)
+fop_fremovexattr_stub(call_frame_t *frame, fop_fremovexattr_t fn, fd_t *fd,
+                      const char *name, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", name, out);
 
-	stub = stub_new (frame, 0, GF_FOP_SETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FREMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.setxattr_cbk.fn = fn;
-	stub->args.setxattr_cbk.op_ret = op_ret;
-	stub->args.setxattr_cbk.op_errno = op_errno;
+    stub->fn.fremovexattr = fn;
+    args_fremovexattr_store(&stub->args, fd, name, xdata);
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_getxattr_stub (call_frame_t *frame,
-		   fop_getxattr_t fn,
-		   loc_t *loc,
-		   const char *name)
+fop_fremovexattr_cbk_stub(call_frame_t *frame, fop_fremovexattr_cbk_t fn,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
+    stub = stub_new(frame, 0, GF_FOP_FREMOVEXATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_GETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.fremovexattr = fn;
+    args_fremovexattr_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
+
+call_stub_t *
+fop_lk_stub(call_frame_t *frame, fop_lk_t fn, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-	stub->args.getxattr.fn = fn;
-	loc_copy (&stub->args.getxattr.loc, loc);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-	if (name)
-	        stub->args.getxattr.name = gf_strdup (name);
+    stub = stub_new(frame, 1, GF_FOP_LK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.lk = fn;
+    args_lk_store(&stub->args, fd, cmd, lock, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_lk_cbk_stub(call_frame_t *frame, fop_lk_cbk_t fn, int32_t op_ret,
+                int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_LK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.lk = fn;
+    args_lk_cbk_store(&stub->args_cbk, op_ret, op_errno, lock, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame,
-		       fop_getxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *dict)
+fop_inodelk_stub(call_frame_t *frame, fop_inodelk_t fn, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-	stub = stub_new (frame, 0, GF_FOP_GETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_INODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.getxattr_cbk.fn = fn;
-	stub->args.getxattr_cbk.op_ret = op_ret;
-	stub->args.getxattr_cbk.op_errno = op_errno;
-	/* TODO */
-	if (dict)
-		stub->args.getxattr_cbk.dict = dict_ref (dict);
+    stub->fn.inodelk = fn;
+    args_inodelk_store(&stub->args, volume, loc, cmd, lock, xdata);
 out:
-	return stub;
+    return stub;
 }
 
+call_stub_t *
+fop_inodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_INODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.inodelk = fn;
+    args_inodelk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_fsetxattr_stub (call_frame_t *frame,
-                    fop_fsetxattr_t fn,
-                    fd_t *fd,
-                    dict_t *dict,
-                    int32_t flags)
+fop_finodelk_stub(call_frame_t *frame, fop_finodelk_t fn, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lock, out);
 
-	stub = stub_new (frame, 1, GF_FOP_FSETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_FINODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fsetxattr.fn = fn;
-	stub->args.fsetxattr.fd = fd_ref (fd);
+    stub->fn.finodelk = fn;
+    args_finodelk_store(&stub->args, volume, fd, cmd, lock, xdata);
 
-	/* TODO */
-	if (dict)
-		stub->args.fsetxattr.dict = dict_ref (dict);
-	stub->args.fsetxattr.flags = flags;
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fsetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fsetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno)
+fop_finodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_FINODELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FSETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.fsetxattr_cbk.fn = fn;
-	stub->args.fsetxattr_cbk.op_ret = op_ret;
-	stub->args.fsetxattr_cbk.op_errno = op_errno;
+    stub->fn_cbk.finodelk = fn;
+    args_finodelk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fgetxattr_stub (call_frame_t *frame,
-                    fop_fgetxattr_t fn,
-                    fd_t *fd,
-                    const char *name)
+fop_entrylk_stub(call_frame_t *frame, fop_entrylk_t fn, const char *volume,
+                 loc_t *loc, const char *name, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_FGETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_ENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fgetxattr.fn = fn;
-	stub->args.fgetxattr.fd = fd_ref (fd);
+    stub->fn.entrylk = fn;
+    args_entrylk_store(&stub->args, volume, loc, name, cmd, type, xdata);
 
-	if (name)
-                stub->args.fgetxattr.name = gf_strdup (name);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fgetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fgetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        dict_t *dict)
+fop_entrylk_cbk_stub(call_frame_t *frame, fop_entrylk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_ENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_GETXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.entrylk = fn;
+    args_entrylk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
+out:
+    return stub;
+}
+
+call_stub_t *
+fop_fentrylk_stub(call_frame_t *frame, fop_fentrylk_t fn, const char *volume,
+                  fd_t *fd, const char *name, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-	stub->args.fgetxattr_cbk.fn = fn;
-	stub->args.fgetxattr_cbk.op_ret = op_ret;
-	stub->args.fgetxattr_cbk.op_errno = op_errno;
+    stub = stub_new(frame, 1, GF_FOP_FENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	/* TODO */
-	if (dict)
-		stub->args.fgetxattr_cbk.dict = dict_ref (dict);
+    stub->fn.fentrylk = fn;
+    args_fentrylk_store(&stub->args, volume, fd, name, cmd, type, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_removexattr_stub (call_frame_t *frame,
-		      fop_removexattr_t fn,
-		      loc_t *loc,
-		      const char *name)
+fop_fentrylk_cbk_stub(call_frame_t *frame, fop_fentrylk_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", loc, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", name, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 1, GF_FOP_REMOVEXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_FENTRYLK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.removexattr.fn = fn;
-	loc_copy (&stub->args.removexattr.loc, loc);
-        stub->args.removexattr.name = gf_strdup (name);
+    stub->fn_cbk.fentrylk = fn;
+    args_fentrylk_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame,
-			  fop_removexattr_cbk_t fn,
-			  int32_t op_ret,
-			  int32_t op_errno)
+fop_readdirp_cbk_stub(call_frame_t *frame, fop_readdirp_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_READDIRP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_REMOVEXATTR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	stub->args.removexattr_cbk.fn = fn;
-	stub->args.removexattr_cbk.op_ret = op_ret;
-	stub->args.removexattr_cbk.op_errno = op_errno;
+    stub->fn_cbk.readdirp = fn;
+    args_readdirp_cbk_store(&stub->args_cbk, op_ret, op_errno, entries, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lk_stub (call_frame_t *frame,
-	     fop_lk_t fn,
-	     fd_t *fd,
-	     int32_t cmd,
-	     struct gf_flock *lock)
+fop_readdir_cbk_stub(call_frame_t *frame, fop_readdir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", lock, out);
-	
-	stub = stub_new (frame, 1, GF_FOP_LK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_READDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.lk.fn = fn;
-	if (fd)
-		stub->args.lk.fd = fd_ref (fd);
-	stub->args.lk.cmd = cmd;
-	stub->args.lk.lock = *lock;
+    stub->fn_cbk.readdir = fn;
+    args_readdir_cbk_store(&stub->args_cbk, op_ret, op_errno, entries, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame,
-		 fop_lk_cbk_t fn,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock)
-
+fop_readdir_stub(call_frame_t *frame, fop_readdir_t fn, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_LK);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_READDIR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.lk_cbk.fn = fn;
-	stub->args.lk_cbk.op_ret = op_ret;
-	stub->args.lk_cbk.op_errno = op_errno;
-	if (op_ret == 0)
-		stub->args.lk_cbk.lock = *lock;
+    stub->fn.readdir = fn;
+    args_readdir_store(&stub->args, fd, size, off, xdata);
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
-		  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+fop_readdirp_stub(call_frame_t *frame, fop_readdirp_t fn, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame || !lock)
-    return NULL;
+    stub = stub_new(frame, 1, GF_FOP_READDIRP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub = stub_new (frame, 1, GF_FOP_INODELK);
-  if (!stub)
-    return NULL;
+    stub->fn.readdirp = fn;
+    args_readdirp_store(&stub->args, fd, size, off, xdata);
+out:
+    return stub;
+}
 
-  stub->args.inodelk.fn = fn;
+call_stub_t *
+fop_rchecksum_stub(call_frame_t *frame, fop_rchecksum_t fn, fd_t *fd,
+                   off_t offset, int32_t len, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-  if (volume)
-          stub->args.inodelk.volume = gf_strdup (volume);
+    GF_VALIDATE_OR_GOTO("call-stub", fd, out);
 
-  loc_copy (&stub->args.inodelk.loc, loc);
-  stub->args.inodelk.cmd  = cmd;
-  stub->args.inodelk.lock = *lock;
+    stub = stub_new(frame, 1, GF_FOP_RCHECKSUM);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn.rchecksum = fn;
+    args_rchecksum_store(&stub->args, fd, offset, len, xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno)
+fop_rchecksum_cbk_stub(call_frame_t *frame, fop_rchecksum_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame)
-    return NULL;
+    stub = stub_new(frame, 0, GF_FOP_RCHECKSUM);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.rchecksum = fn;
+    args_rchecksum_cbk_store(&stub->args_cbk, op_ret, op_errno, weak_checksum,
+                             strong_checksum, xdata);
+out:
+    return stub;
+}
 
-  stub = stub_new (frame, 0, GF_FOP_INODELK);
-  if (!stub)
-    return NULL;
+call_stub_t *
+fop_xattrop_cbk_stub(call_frame_t *frame, fop_xattrop_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-  stub->args.inodelk_cbk.fn       = fn;
-  stub->args.inodelk_cbk.op_ret   = op_ret;
-  stub->args.inodelk_cbk.op_errno = op_errno;
+    stub = stub_new(frame, 0, GF_FOP_XATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn_cbk.xattrop = fn;
+    args_xattrop_cbk_store(&stub->args_cbk, op_ret, op_errno, xattr, xdata);
+out:
+    return stub;
 }
 
+call_stub_t *
+fop_fxattrop_cbk_stub(call_frame_t *frame, fop_fxattrop_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_FXATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.fxattrop = fn;
+    args_xattrop_cbk_store(&stub->args_cbk, op_ret, op_errno, xattr, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
-		   const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+fop_xattrop_stub(call_frame_t *frame, fop_xattrop_t fn, loc_t *loc,
+                 gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame || !lock)
-    return NULL;
+    GF_VALIDATE_OR_GOTO("call-stub", xattr, out);
 
-  stub = stub_new (frame, 1, GF_FOP_FINODELK);
-  if (!stub)
-    return NULL;
+    stub = stub_new(frame, 1, GF_FOP_XATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub->args.finodelk.fn = fn;
+    stub->fn.xattrop = fn;
+    args_xattrop_store(&stub->args, loc, optype, xattr, xdata);
+out:
+    return stub;
+}
 
-  if (fd)
-	  stub->args.finodelk.fd   = fd_ref (fd);
+call_stub_t *
+fop_fxattrop_stub(call_frame_t *frame, fop_fxattrop_t fn, fd_t *fd,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-  if (volume)
-          stub->args.finodelk.volume = gf_strdup (volume);
+    GF_VALIDATE_OR_GOTO("call-stub", xattr, out);
 
-  stub->args.finodelk.cmd  = cmd;
-  stub->args.finodelk.lock = *lock;
+    stub = stub_new(frame, 1, GF_FOP_FXATTROP);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn.fxattrop = fn;
+    args_fxattrop_store(&stub->args, fd, optype, xattr, xdata);
+out:
+    return stub;
 }
 
-
 call_stub_t *
-fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno)
+fop_setattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame)
-    return NULL;
+    stub = stub_new(frame, 0, GF_FOP_SETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub = stub_new (frame, 0, GF_FOP_FINODELK);
-  if (!stub)
-    return NULL;
+    stub->fn_cbk.setattr = fn;
+    args_setattr_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre, statpost,
+                           xdata);
+out:
+    return stub;
+}
 
-  stub->args.finodelk_cbk.fn       = fn;
-  stub->args.finodelk_cbk.op_ret   = op_ret;
-  stub->args.finodelk_cbk.op_errno = op_errno;
+call_stub_t *
+fop_fsetattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-  return stub;
-}
+    stub = stub_new(frame, 0, GF_FOP_FSETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn_cbk.fsetattr = fn;
+    args_fsetattr_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                            statpost, xdata);
+out:
+    return stub;
+}
 
 call_stub_t *
-fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
-		  const char *volume, loc_t *loc, const char *name,
-		  entrylk_cmd cmd, entrylk_type type)
+fop_setattr_stub(call_frame_t *frame, fop_setattr_t fn, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame)
-    return NULL;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-  stub = stub_new (frame, 1, GF_FOP_ENTRYLK);
-  if (!stub)
-    return NULL;
+    stub = stub_new(frame, 1, GF_FOP_SETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub->args.entrylk.fn = fn;
+    stub->fn.setattr = fn;
+    args_setattr_store(&stub->args, loc, stbuf, valid, xdata);
+out:
+    return stub;
+}
 
-  if (volume)
-          stub->args.entrylk.volume = gf_strdup (volume);
+call_stub_t *
+fop_fsetattr_stub(call_frame_t *frame, fop_fsetattr_t fn, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-  loc_copy (&stub->args.entrylk.loc, loc);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-  stub->args.entrylk.cmd = cmd;
-  stub->args.entrylk.type = type;
-  if (name)
-          stub->args.entrylk.name = gf_strdup (name);
+    stub = stub_new(frame, 1, GF_FOP_FSETATTR);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn.fsetattr = fn;
+    args_fsetattr_store(&stub->args, fd, stbuf, valid, xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno)
+fop_fallocate_cbk_stub(call_frame_t *frame, fop_fallocate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
-
-  if (!frame)
-    return NULL;
+    call_stub_t *stub = NULL;
 
-  stub = stub_new (frame, 0, GF_FOP_ENTRYLK);
-  if (!stub)
-    return NULL;
+    stub = stub_new(frame, 0, GF_FOP_FALLOCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub->args.entrylk_cbk.fn       = fn;
-  stub->args.entrylk_cbk.op_ret   = op_ret;
-  stub->args.entrylk_cbk.op_errno = op_errno;
+    stub->fn_cbk.fallocate = fn;
 
-  return stub;
+    args_fallocate_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                             statpost, xdata);
+out:
+    return stub;
 }
 
-
 call_stub_t *
-fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
-		   const char *volume, fd_t *fd, const char *name,
-		   entrylk_cmd cmd, entrylk_type type)
+fop_fallocate_stub(call_frame_t *frame, fop_fallocate_t fn, fd_t *fd,
+                   int32_t mode, off_t offset, size_t len, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame)
-    return NULL;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-  stub = stub_new (frame, 1, GF_FOP_FENTRYLK);
-  if (!stub)
-    return NULL;
+    stub = stub_new(frame, 1, GF_FOP_FALLOCATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub->args.fentrylk.fn = fn;
+    stub->fn.fallocate = fn;
+    args_fallocate_store(&stub->args, fd, mode, offset, len, xdata);
+out:
+    return stub;
+}
 
-  if (volume)
-          stub->args.fentrylk.volume = gf_strdup (volume);
+call_stub_t *
+fop_discard_cbk_stub(call_frame_t *frame, fop_discard_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_DISCARD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  if (fd)
-	  stub->args.fentrylk.fd = fd_ref (fd);
-  stub->args.fentrylk.cmd = cmd;
-  stub->args.fentrylk.type = type;
-  if (name)
-          stub->args.fentrylk.name = gf_strdup (name);
+    stub->fn_cbk.discard = fn;
 
-  return stub;
+    args_discard_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre, statpost,
+                           xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_fentrylk_cbk_stub (call_frame_t *frame, fop_fentrylk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno)
+fop_discard_stub(call_frame_t *frame, fop_discard_t fn, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  if (!frame)
-    return NULL;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-  stub = stub_new (frame, 0, GF_FOP_FENTRYLK);
-  if (!stub)
-    return NULL;
+    stub = stub_new(frame, 1, GF_FOP_DISCARD);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  stub->args.fentrylk_cbk.fn       = fn;
-  stub->args.fentrylk_cbk.op_ret   = op_ret;
-  stub->args.fentrylk_cbk.op_errno = op_errno;
-
-  return stub;
+    stub->fn.discard = fn;
+    args_discard_store(&stub->args, fd, offset, len, xdata);
+out:
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdirp_cbk_stub (call_frame_t *frame,
-		       fop_readdirp_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       gf_dirent_t *entries)
+fop_zerofill_cbk_stub(call_frame_t *frame, fop_zerofill_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-	gf_dirent_t *stub_entry = NULL, *entry = NULL;
-
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_READDIRP);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 0, GF_FOP_ZEROFILL);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.readdirp_cbk.fn = fn;
-	stub->args.readdirp_cbk.op_ret = op_ret;
-	stub->args.readdirp_cbk.op_errno = op_errno;
-	INIT_LIST_HEAD (&stub->args.readdirp_cbk.entries.list);
+    stub->fn_cbk.zerofill = fn;
 
-        /* This check must come after the init of head above
-         * so we're sure the list is empty for list_empty.
-         */
-        if (!entries)
-                goto out;
-
-	if (op_ret > 0) {
-		list_for_each_entry (entry, &entries->list, list) {
-			stub_entry = gf_dirent_for_name (entry->d_name);
-                        if (!stub_entry)
-                                goto out;
-			stub_entry->d_off = entry->d_off;
-			stub_entry->d_ino = entry->d_ino;
-			stub_entry->d_stat = entry->d_stat;
-			list_add_tail (&stub_entry->list,
-				       &stub->args.readdirp_cbk.entries.list);
-		}
-	}
+    args_zerofill_cbk_store(&stub->args_cbk, op_ret, op_errno, statpre,
+                            statpost, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame,
-		      fop_readdir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      gf_dirent_t *entries)
+fop_zerofill_stub(call_frame_t *frame, fop_zerofill_t fn, fd_t *fd,
+                  off_t offset, off_t len, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-	gf_dirent_t *stub_entry = NULL, *entry = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-	stub = stub_new (frame, 0, GF_FOP_READDIR);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-	
-	stub->args.readdir_cbk.fn = fn;
-	stub->args.readdir_cbk.op_ret = op_ret;
-	stub->args.readdir_cbk.op_errno = op_errno;
-	INIT_LIST_HEAD (&stub->args.readdir_cbk.entries.list);
+    stub = stub_new(frame, 1, GF_FOP_ZEROFILL);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        /* This check must come after the init of head above
-         * so we're sure the list is empty for list_empty.
-         */
-        if (!entries)
-                goto out;
+    stub->fn.zerofill = fn;
+    args_zerofill_store(&stub->args, fd, offset, len, xdata);
+out:
+    return stub;
+}
+
+call_stub_t *
+fop_ipc_cbk_stub(call_frame_t *frame, fop_ipc_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-	if (op_ret > 0) {
-		list_for_each_entry (entry, &entries->list, list) {
-			stub_entry = gf_dirent_for_name (entry->d_name);
-                        if (!stub_entry)
-                                goto out;
-			stub_entry->d_off = entry->d_off;
-			stub_entry->d_ino = entry->d_ino;
+    stub = stub_new(frame, 0, GF_FOP_IPC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-			list_add_tail (&stub_entry->list,
-				       &stub->args.readdir_cbk.entries.list);
-		}
-	}
+    stub->fn_cbk.ipc = fn;
+
+    args_ipc_cbk_store(&stub->args_cbk, op_ret, op_errno, xdata);
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_readdir_stub (call_frame_t *frame,
-		  fop_readdir_t fn,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off)
+fop_ipc_stub(call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-  stub = stub_new (frame, 1, GF_FOP_READDIR);
-  stub->args.readdir.fn = fn;
-  stub->args.readdir.fd = fd_ref (fd);
-  stub->args.readdir.size = size;
-  stub->args.readdir.off = off;
+    stub = stub_new(frame, 1, GF_FOP_IPC);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn.ipc = fn;
+    args_ipc_store(&stub->args, op, xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_readdirp_stub (call_frame_t *frame,
-		   fop_readdirp_t fn,
-		   fd_t *fd,
-		   size_t size,
-		   off_t off)
+fop_lease_cbk_stub(call_frame_t *frame, fop_lease_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-  call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-  stub = stub_new (frame, 1, GF_FOP_READDIRP);
-  stub->args.readdirp.fn = fn;
-  stub->args.readdirp.fd = fd_ref (fd);
-  stub->args.readdirp.size = size;
-  stub->args.readdirp.off = off;
+    stub = stub_new(frame, 0, GF_FOP_LEASE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-  return stub;
+    stub->fn_cbk.lease = fn;
+    args_lease_cbk_store(&stub->args_cbk, op_ret, op_errno, lease, xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_rchecksum_stub (call_frame_t *frame,
-                    fop_rchecksum_t fn,
-                    fd_t *fd, off_t offset,
-                    int32_t len)
+fop_lease_stub(call_frame_t *frame, fop_lease_t fn, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	GF_VALIDATE_OR_GOTO ("call-stub", fd, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
+    GF_VALIDATE_OR_GOTO("call-stub", lease, out);
 
-	stub = stub_new (frame, 1, GF_FOP_RCHECKSUM);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub = stub_new(frame, 1, GF_FOP_LEASE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.rchecksum.fn = fn;
-        stub->args.rchecksum.fd = fd_ref (fd);
-	stub->args.rchecksum.offset = offset;
-	stub->args.rchecksum.len    = len;
+    stub->fn.lease = fn;
+    args_lease_store(&stub->args, loc, lease, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_rchecksum_cbk_stub (call_frame_t *frame,
-                        fop_rchecksum_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        uint32_t weak_checksum,
-                        uint8_t *strong_checksum)
+fop_seek_cbk_stub(call_frame_t *frame, fop_seek_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, off_t offset, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    stub = stub_new(frame, 0, GF_FOP_SEEK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_RCHECKSUM);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    stub->fn_cbk.seek = fn;
 
-	stub->args.rchecksum_cbk.fn = fn;
-	stub->args.rchecksum_cbk.op_ret = op_ret;
-	stub->args.rchecksum_cbk.op_errno = op_errno;
-
-	if (op_ret >= 0)
-	{
-		stub->args.rchecksum_cbk.weak_checksum =
-                        weak_checksum;
-
-		stub->args.rchecksum_cbk.strong_checksum = 
-			memdup (strong_checksum, MD5_DIGEST_LEN);
-	}
+    args_seek_cbk_store(&stub->args_cbk, op_ret, op_errno, offset, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_xattrop_cbk_stub (call_frame_t *frame,
-		      fop_xattrop_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno)
+fop_seek_stub(call_frame_t *frame, fop_seek_t fn, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
-	
-	stub = stub_new (frame, 0, GF_FOP_XATTROP);
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-	stub->args.xattrop_cbk.fn       = fn;
-	stub->args.xattrop_cbk.op_ret   = op_ret;
-	stub->args.xattrop_cbk.op_errno = op_errno;
+    stub = stub_new(frame, 1, GF_FOP_SEEK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn.seek = fn;
+    args_seek_store(&stub->args, fd, offset, what, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_fxattrop_cbk_stub (call_frame_t *frame,
-		       fop_fxattrop_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *xattr)
+fop_getactivelk_cbk_stub(call_frame_t *frame, fop_getactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *lmi, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-	GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_GETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FXATTROP);
-	stub->args.fxattrop_cbk.fn = fn;
-	stub->args.fxattrop_cbk.op_ret = op_ret;
-	stub->args.fxattrop_cbk.op_errno = op_errno;
-	if (xattr) 
-		stub->args.fxattrop_cbk.xattr = dict_ref (xattr);
+    stub->fn_cbk.getactivelk = fn;
 
+    args_getactivelk_cbk_store(&stub->args_cbk, op_ret, op_errno, lmi, xdata);
 out:
-	return stub;
+    return stub;
 }
 
-
 call_stub_t *
-fop_xattrop_stub (call_frame_t *frame,
-		  fop_xattrop_t fn,
-		  loc_t *loc,
-		  gf_xattrop_flags_t optype,
-		  dict_t *xattr)
+fop_getactivelk_stub(call_frame_t *frame, fop_getactivelk_t fn, loc_t *loc,
+                     dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	if (!frame || !xattr)
-		return NULL;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-	stub = stub_new (frame, 1, GF_FOP_XATTROP);
-	if (!stub)
-		return NULL;
+    stub = stub_new(frame, 1, GF_FOP_GETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.xattrop.fn = fn;
-	
-	loc_copy (&stub->args.xattrop.loc, loc);
+    stub->fn.getactivelk = fn;
 
-	stub->args.xattrop.optype = optype;
-	stub->args.xattrop.xattr = dict_ref (xattr);
+    loc_copy(&stub->args.loc, loc);
 
-	return stub;
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
+out:
+    return stub;
 }
 
 call_stub_t *
-fop_fxattrop_stub (call_frame_t *frame,
-		   fop_fxattrop_t fn,
-		   fd_t *fd,
-		   gf_xattrop_flags_t optype,
-		   dict_t *xattr)
+fop_setactivelk_cbk_stub(call_frame_t *frame, fop_setactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-	if (!frame || !xattr)
-		return NULL;
+    stub = stub_new(frame, 0, GF_FOP_SETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_FXATTROP);
-	if (!stub)
-		return NULL;
+    stub->fn_cbk.setactivelk = fn;
+    stub->args_cbk.op_ret = op_ret;
+    stub->args_cbk.op_errno = op_errno;
 
-	stub->args.fxattrop.fn = fn;
-	
-	stub->args.fxattrop.fd = fd_ref (fd);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 
-	stub->args.fxattrop.optype = optype;
-	stub->args.fxattrop.xattr = dict_ref (xattr);
-
-	return stub;
+out:
+    return stub;
 }
 
-
 call_stub_t *
-fop_setattr_cbk_stub (call_frame_t *frame,
-                      fop_setattr_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      struct iatt *statpre,
-                      struct iatt *statpost)
+fop_setactivelk_stub(call_frame_t *frame, fop_setactivelk_t fn, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
-
-        if (frame == NULL)
-                goto out;
+    call_stub_t *stub = NULL;
 
-	stub = stub_new (frame, 0, GF_FOP_SETATTR);
-	if (stub == NULL)
-                goto out;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-	stub->args.setattr_cbk.fn = fn;
+    stub = stub_new(frame, 1, GF_FOP_SETACTIVELK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        stub->args.setattr_cbk.op_ret = op_ret;
-        stub->args.setattr_cbk.op_errno = op_errno;
+    stub->fn.setactivelk = fn;
 
-        if (statpre)
-                stub->args.setattr_cbk.statpre = *statpre;
-        if (statpost)
-                stub->args.setattr_cbk.statpost = *statpost;
+    args_setactivelk_store(&stub->args, loc, locklist, xdata);
 
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_fsetattr_cbk_stub (call_frame_t *frame,
-                       fop_setattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *statpre,
-                       struct iatt *statpost)
+fop_copy_file_range_stub(call_frame_t *frame, fop_copy_file_range_t fn,
+                         fd_t *fd_in, off64_t off_in, fd_t *fd_out,
+                         off64_t off_out, size_t len, uint32_t flags,
+                         dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (frame == NULL)
-                goto out;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-	stub = stub_new (frame, 0, GF_FOP_FSETATTR);
-	if (stub == NULL)
-                goto out;
+    stub = stub_new(frame, 1, GF_FOP_COPY_FILE_RANGE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub->args.fsetattr_cbk.fn = fn;
+    stub->fn.copy_file_range = fn;
 
-        stub->args.fsetattr_cbk.op_ret = op_ret;
-        stub->args.fsetattr_cbk.op_errno = op_errno;
+    args_copy_file_range_store(&stub->args, fd_in, off_in, fd_out, off_out, len,
+                               flags, xdata);
 
-        if (statpre)
-                stub->args.setattr_cbk.statpre = *statpre;
-        if (statpost)
-                stub->args.fsetattr_cbk.statpost = *statpost;
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_setattr_stub (call_frame_t *frame,
-                  fop_setattr_t fn,
-                  loc_t *loc,
-                  struct iatt *stbuf,
-                  int32_t valid)
+fop_copy_file_range_cbk_stub(call_frame_t *frame, fop_copy_file_range_cbk_t fn,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (frame == NULL)
-                goto out;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        if (fn == NULL)
-                goto out;
+    stub = stub_new(frame, 0, GF_FOP_COPY_FILE_RANGE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	stub = stub_new (frame, 1, GF_FOP_SETATTR);
-	if (stub == NULL)
-                goto out;
+    stub->fn_cbk.copy_file_range = fn;
+    args_copy_file_range_cbk_store(&stub->args_cbk, op_ret, op_errno, stbuf,
+                                   prebuf_dst, postbuf_dst, xdata);
 
-	stub->args.setattr.fn = fn;
+out:
+    return stub;
+}
 
-	loc_copy (&stub->args.setattr.loc, loc);
+call_stub_t *
+fop_put_stub(call_frame_t *frame, fop_put_t fn, loc_t *loc, mode_t mode,
+             mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+             off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-        if (stbuf)
-                stub->args.setattr.stbuf = *stbuf;
+    GF_VALIDATE_OR_GOTO("call-stub", vector, out);
 
-        stub->args.setattr.valid = valid;
+    stub = stub_new(frame, 1, GF_FOP_PUT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
+    stub->fn.put = fn;
+    args_put_store(&stub->args, loc, mode, umask, flags, vector, count, offset,
+                   iobref, xattr, xdata);
 out:
-	return stub;
+    return stub;
 }
 
 call_stub_t *
-fop_fsetattr_stub (call_frame_t *frame,
-                   fop_fsetattr_t fn,
-                   fd_t *fd,
-                   struct iatt *stbuf,
-                   int32_t valid)
+fop_put_cbk_stub(call_frame_t *frame, fop_put_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (frame == NULL)
-                goto out;
+    stub = stub_new(frame, 0, GF_FOP_PUT);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        if (fn == NULL)
-                goto out;
+    stub->fn_cbk.put = fn;
+    args_put_cbk_store(&stub->args_cbk, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+out:
+    return stub;
+}
 
-	stub = stub_new (frame, 1, GF_FOP_FSETATTR);
-	if (stub == NULL)
-                goto out;
+call_stub_t *
+fop_icreate_stub(call_frame_t *frame, fop_icreate_t fn, loc_t *loc, mode_t mode,
+                 dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-	stub->args.fsetattr.fn = fn;
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
 
-        if (fd)
-                stub->args.fsetattr.fd = fd_ref (fd);
+    stub = stub_new(frame, 1, GF_FOP_ICREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        if (stbuf)
-                stub->args.fsetattr.stbuf = *stbuf;
+    stub->fn.icreate = fn;
 
-        stub->args.fsetattr.valid = valid;
+    stub->args.mode = mode;
+    if (loc)
+        loc_copy(&stub->args.loc, loc);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
 
 out:
-	return stub;
+    return stub;
 }
 
 static void
-call_resume_wind (call_stub_t *stub)
-{
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	switch (stub->fop) {
-	case GF_FOP_OPEN:
-	{
-		stub->args.open.fn (stub->frame, 
-				    stub->frame->this,
-				    &stub->args.open.loc, 
-				    stub->args.open.flags, stub->args.open.fd,
-                                    stub->args.open.wbflags);
-		break;
-	}
-	case GF_FOP_CREATE:
-	{
-		stub->args.create.fn (stub->frame,
-				      stub->frame->this,
-				      &stub->args.create.loc,
-				      stub->args.create.flags,
-				      stub->args.create.mode,
-				      stub->args.create.fd,
-                                      stub->args.create.params);
-		break;
-	}
-	case GF_FOP_STAT:
-	{
-		stub->args.stat.fn (stub->frame,
-				    stub->frame->this,
-				    &stub->args.stat.loc);
-		break;
-	}
-	case GF_FOP_READLINK:
-	{
-		stub->args.readlink.fn (stub->frame,
-					stub->frame->this,
-					&stub->args.readlink.loc,
-					stub->args.readlink.size);
-		break;
-	}
-  
-	case GF_FOP_MKNOD:
-	{
-		stub->args.mknod.fn (stub->frame, stub->frame->this,
-				     &stub->args.mknod.loc,
-				     stub->args.mknod.mode,
-				     stub->args.mknod.rdev,
-                                     stub->args.mknod.params);
-	}
-	break;
-  
-	case GF_FOP_MKDIR:
-	{
-		stub->args.mkdir.fn (stub->frame, stub->frame->this,
-				     &stub->args.mkdir.loc,
-				     stub->args.mkdir.mode,
-                                     stub->args.mkdir.params);
-	}
-	break;
-  
-	case GF_FOP_UNLINK:
-	{
-		stub->args.unlink.fn (stub->frame,
-				      stub->frame->this,
-				      &stub->args.unlink.loc);
-	}
-	break;
-
-	case GF_FOP_RMDIR:
-	{
-		stub->args.rmdir.fn (stub->frame, stub->frame->this,
-				     &stub->args.rmdir.loc,
-                                     stub->args.rmdir.flags);
-	}
-	break;
-      
-	case GF_FOP_SYMLINK:
-	{
-		stub->args.symlink.fn (stub->frame,
-				       stub->frame->this,
-				       stub->args.symlink.linkname,
-				       &stub->args.symlink.loc,
-                                       stub->args.symlink.params);
-	}
-	break;
-  
-	case GF_FOP_RENAME:
-	{
-		stub->args.rename.fn (stub->frame,
-				      stub->frame->this,
-				      &stub->args.rename.old,
-				      &stub->args.rename.new);
-	}
-	break;
-
-	case GF_FOP_LINK:
-	{
-		stub->args.link.fn (stub->frame,
-				    stub->frame->this,
-				    &stub->args.link.oldloc,
-				    &stub->args.link.newloc);
-	}
-	break;
-  
-	case GF_FOP_TRUNCATE:
-	{
-		stub->args.truncate.fn (stub->frame,
-					stub->frame->this,
-					&stub->args.truncate.loc,
-					stub->args.truncate.off);
-		break;
-	}
-      
-	case GF_FOP_READ:
-	{
-		stub->args.readv.fn (stub->frame,
-				     stub->frame->this,
-				     stub->args.readv.fd,
-				     stub->args.readv.size,
-				     stub->args.readv.off);
-		break;
-	}
-  
-	case GF_FOP_WRITE:
-	{
-		stub->args.writev.fn (stub->frame,
-				      stub->frame->this,
-				      stub->args.writev.fd,
-				      stub->args.writev.vector,
-				      stub->args.writev.count,
-				      stub->args.writev.off,
-                                      stub->args.writev.iobref);
-		break;
-	}
-  
-	case GF_FOP_STATFS:
-	{
-		stub->args.statfs.fn (stub->frame,
-				      stub->frame->this,
-				      &stub->args.statfs.loc);
-		break;
-	}
-	case GF_FOP_FLUSH:
-	{
-		stub->args.flush.fn (stub->frame,
-				     stub->frame->this,
-				     stub->args.flush.fd);
-		break;
-	}
-  
-	case GF_FOP_FSYNC:
-	{
-		stub->args.fsync.fn (stub->frame,
-				     stub->frame->this,
-				     stub->args.fsync.fd,
-				     stub->args.fsync.datasync);
-		break;
-	}
-
-	case GF_FOP_SETXATTR:
-	{
-		stub->args.setxattr.fn (stub->frame,
-					stub->frame->this,
-					&stub->args.setxattr.loc,
-					stub->args.setxattr.dict,
-					stub->args.setxattr.flags);
-		break;
-	}
-  
-	case GF_FOP_GETXATTR:
-	{
-		stub->args.getxattr.fn (stub->frame,
-					stub->frame->this,
-					&stub->args.getxattr.loc,
-					stub->args.getxattr.name);
-		break;
-	}
-
-	case GF_FOP_FSETXATTR:
-	{
-		stub->args.fsetxattr.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.fsetxattr.fd,
-                                         stub->args.fsetxattr.dict,
-                                         stub->args.fsetxattr.flags);
-		break;
-	}
-
-	case GF_FOP_FGETXATTR:
-	{
-		stub->args.fgetxattr.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.fgetxattr.fd,
-                                         stub->args.fgetxattr.name);
-		break;
-	}
-
-	case GF_FOP_REMOVEXATTR:
-	{
-		stub->args.removexattr.fn (stub->frame,
-					   stub->frame->this,
-					   &stub->args.removexattr.loc,
-					   stub->args.removexattr.name);
-		break;
-	}
-  
-	case GF_FOP_OPENDIR:
-	{
-		stub->args.opendir.fn (stub->frame,
-				       stub->frame->this,
-				       &stub->args.opendir.loc,
-				       stub->args.opendir.fd);
-		break;
-	}
-
-	case GF_FOP_FSYNCDIR:
-	{
-		stub->args.fsyncdir.fn (stub->frame,
-					stub->frame->this,
-					stub->args.fsyncdir.fd,
-					stub->args.fsyncdir.datasync);
-		break;
-	}
-  
-	case GF_FOP_ACCESS:
-	{
-		stub->args.access.fn (stub->frame,
-				      stub->frame->this,
-				      &stub->args.access.loc,
-				      stub->args.access.mask);
-		break;
-	}
-  
-	case GF_FOP_FTRUNCATE:
-	{
-		stub->args.ftruncate.fn (stub->frame,
-					 stub->frame->this,
-					 stub->args.ftruncate.fd,
-					 stub->args.ftruncate.off);
-		break;
-	}
-  
-	case GF_FOP_FSTAT:
-	{
-		stub->args.fstat.fn (stub->frame,
-				     stub->frame->this,
-				     stub->args.fstat.fd);
-		break;
-	}
-  
-	case GF_FOP_LK:
-	{
-		stub->args.lk.fn (stub->frame,
-				  stub->frame->this,
-				  stub->args.lk.fd,
-				  stub->args.lk.cmd,
-				  &stub->args.lk.lock);
-		break;
-	}
-
-	case GF_FOP_INODELK:
-	{
-		stub->args.inodelk.fn (stub->frame,
-				       stub->frame->this,
-                                       stub->args.inodelk.volume,
-				       &stub->args.inodelk.loc,
-				       stub->args.inodelk.cmd,
-				       &stub->args.inodelk.lock);
-		break;
-	}
-
-	case GF_FOP_FINODELK:
-	{
-		stub->args.finodelk.fn (stub->frame,
-					stub->frame->this,
-                                        stub->args.finodelk.volume,
-					stub->args.finodelk.fd,
-					stub->args.finodelk.cmd,
-					&stub->args.finodelk.lock);
-		break;
-	}
-
-	case GF_FOP_ENTRYLK:
-	{
-		stub->args.entrylk.fn (stub->frame,
-				       stub->frame->this,
-                                       stub->args.entrylk.volume,
-				       &stub->args.entrylk.loc,
-				       stub->args.entrylk.name,
-				       stub->args.entrylk.cmd,
-				       stub->args.entrylk.type);
-		break;
-	}
-
-	case GF_FOP_FENTRYLK:
-	{
-		stub->args.fentrylk.fn (stub->frame,
-					stub->frame->this,
-                                        stub->args.fentrylk.volume,
-					stub->args.fentrylk.fd,
-					stub->args.fentrylk.name,
-					stub->args.fentrylk.cmd,
-					stub->args.fentrylk.type);
-		break;
-	}
-  
-	break;
-  
-	case GF_FOP_LOOKUP:
-	{
-		stub->args.lookup.fn (stub->frame, 
-				      stub->frame->this,
-				      &stub->args.lookup.loc,
-				      stub->args.lookup.xattr_req);
-		break;
-	}
-
-	case GF_FOP_RCHECKSUM:
-	{
-		stub->args.rchecksum.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.rchecksum.fd,
-                                         stub->args.rchecksum.offset,
-                                         stub->args.rchecksum.len);
-		break;
-	}
-
-	case GF_FOP_READDIR:
-	{
-		stub->args.readdir.fn (stub->frame,
-				       stub->frame->this,
-				       stub->args.readdir.fd,
-				       stub->args.readdir.size,
-				       stub->args.readdir.off);
-		break;
-	}
+args_icreate_store_cbk(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, inode_t *inode, struct iatt *buf,
+                       dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+}
 
-        case GF_FOP_READDIRP:
-	{
-		stub->args.readdirp.fn (stub->frame,
-				        stub->frame->this,
-				        stub->args.readdirp.fd,
-				        stub->args.readdirp.size,
-				        stub->args.readdirp.off);
-		break;
-	}
-
-	case GF_FOP_XATTROP:
-	{
-		stub->args.xattrop.fn (stub->frame,
-				       stub->frame->this,
-				       &stub->args.xattrop.loc,
-				       stub->args.xattrop.optype,
-				       stub->args.xattrop.xattr);
-
-		break;
-	}
-	case GF_FOP_FXATTROP:
-	{
-		stub->args.fxattrop.fn (stub->frame,
-					stub->frame->this,
-					stub->args.fxattrop.fd,
-					stub->args.fxattrop.optype,
-					stub->args.fxattrop.xattr);
-
-		break;
-	}
-        case GF_FOP_SETATTR:
-        {
-                stub->args.setattr.fn (stub->frame,
-                                       stub->frame->this,
-                                       &stub->args.setattr.loc,
-                                       &stub->args.setattr.stbuf,
-                                       stub->args.setattr.valid);
-                break;
-        }
-        case GF_FOP_FSETATTR:
-        {
-                stub->args.fsetattr.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.fsetattr.fd,
-                                        &stub->args.fsetattr.stbuf,
-                                        stub->args.fsetattr.valid);
-                break;
-        }
-	default:
-	{
-		gf_log ("call-stub", GF_LOG_ERROR, "Invalid value of FOP (%d)",
-                        stub->fop);
-                break;
-	}
+call_stub_t *
+fop_icreate_cbk_stub(call_frame_t *frame, fop_icreate_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_ICREATE);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.icreate = fn;
+    args_icreate_store_cbk(&stub->args_cbk, op_ret, op_errno, inode, buf,
+                           xdata);
 
-	}
 out:
-	return;
+    return stub;
 }
 
+call_stub_t *
+fop_namelink_stub(call_frame_t *frame, fop_namelink_t fn, loc_t *loc,
+                  dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("call-stub", fn, out);
+
+    stub = stub_new(frame, 1, GF_FOP_NAMELINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn.namelink = fn;
 
+    if (loc)
+        loc_copy(&stub->args.loc, loc);
+    if (xdata)
+        stub->args.xdata = dict_ref(xdata);
+
+out:
+    return stub;
+}
 
 static void
-call_resume_unwind (call_stub_t *stub)
-{
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
-
-	switch (stub->fop) {
-	case GF_FOP_OPEN:
-	{
-		if (!stub->args.open_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.open_cbk.op_ret,
-				      stub->args.open_cbk.op_errno,
-				      stub->args.open_cbk.fd);
-		else
-			stub->args.open_cbk.fn (stub->frame, 
-						stub->frame->cookie,
-						stub->frame->this,
-						stub->args.open_cbk.op_ret, 
-						stub->args.open_cbk.op_errno,
-						stub->args.open_cbk.fd);
-		break;
-	}
-
-	case GF_FOP_CREATE:
-	{
-		if (!stub->args.create_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.create_cbk.op_ret,
-				      stub->args.create_cbk.op_errno,
-				      stub->args.create_cbk.fd,
-				      stub->args.create_cbk.inode,
-				      &stub->args.create_cbk.buf,
-                                      &stub->args.create_cbk.preparent,
-                                      &stub->args.create_cbk.postparent);
-		else
-			stub->args.create_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.create_cbk.op_ret,
-						  stub->args.create_cbk.op_errno,
-						  stub->args.create_cbk.fd,
-						  stub->args.create_cbk.inode,
-						  &stub->args.create_cbk.buf,
-                                                  &stub->args.create_cbk.preparent,
-                                                  &stub->args.create_cbk.postparent);
-      
-		break;
-	}
-
-	case GF_FOP_STAT:
-	{
-		if (!stub->args.stat_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.stat_cbk.op_ret,
-				      stub->args.stat_cbk.op_errno,
-				      &stub->args.stat_cbk.buf);
-		else
-			stub->args.stat_cbk.fn (stub->frame,
-						stub->frame->cookie,
-						stub->frame->this,
-						stub->args.stat_cbk.op_ret,
-						stub->args.stat_cbk.op_errno,
-						&stub->args.stat_cbk.buf);
-
-		break;
-	}
-
-	case GF_FOP_READLINK:
-	{
-		if (!stub->args.readlink_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.readlink_cbk.op_ret,
-				      stub->args.readlink_cbk.op_errno,
-				      stub->args.readlink_cbk.buf,
-                                      &stub->args.readlink_cbk.sbuf);
-		else
-			stub->args.readlink_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.readlink_cbk.op_ret,
-						    stub->args.readlink_cbk.op_errno,
-						    stub->args.readlink_cbk.buf,
-                                                    &stub->args.readlink_cbk.sbuf);
-
-		break;
-	}
-  
-	case GF_FOP_MKNOD:
-	{
-		if (!stub->args.mknod_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.mknod_cbk.op_ret,
-				      stub->args.mknod_cbk.op_errno,
-				      stub->args.mknod_cbk.inode,
-                                      &stub->args.mknod_cbk.buf,
-                                      &stub->args.mknod_cbk.preparent,
-                                      &stub->args.mknod_cbk.postparent);
-		else
-			stub->args.mknod_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.mknod_cbk.op_ret,
-						 stub->args.mknod_cbk.op_errno,
-						 stub->args.mknod_cbk.inode,
-                                                 &stub->args.mknod_cbk.buf,
-                                                 &stub->args.mknod_cbk.preparent,
-                                                 &stub->args.mknod_cbk.postparent);
-		break;
-	}
-
-	case GF_FOP_MKDIR:
-	{
-		if (!stub->args.mkdir_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.mkdir_cbk.op_ret,
-				      stub->args.mkdir_cbk.op_errno,
-				      stub->args.mkdir_cbk.inode,
-                                      &stub->args.mkdir_cbk.buf,
-                                      &stub->args.mkdir_cbk.preparent,
-                                      &stub->args.mkdir_cbk.postparent);
-		else
-			stub->args.mkdir_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.mkdir_cbk.op_ret,
-						 stub->args.mkdir_cbk.op_errno,
-						 stub->args.mkdir_cbk.inode,
-                                                 &stub->args.mkdir_cbk.buf,
-                                                 &stub->args.mkdir_cbk.preparent,
-                                                 &stub->args.mkdir_cbk.postparent);
-
-		if (stub->args.mkdir_cbk.inode)
-			inode_unref (stub->args.mkdir_cbk.inode);
-
-		break;
-	}
-  
-	case GF_FOP_UNLINK:
-	{
-		if (!stub->args.unlink_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.unlink_cbk.op_ret,
-				      stub->args.unlink_cbk.op_errno,
-                                      &stub->args.unlink_cbk.preparent,
-                                      &stub->args.unlink_cbk.postparent);
-		else
-			stub->args.unlink_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.unlink_cbk.op_ret,
-						  stub->args.unlink_cbk.op_errno,
-                                                  &stub->args.unlink_cbk.preparent,
-                                                  &stub->args.unlink_cbk.postparent);
-		break;
-	}
-  
-	case GF_FOP_RMDIR:
-	{
-		if (!stub->args.rmdir_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.rmdir_cbk.op_ret,
-				      stub->args.rmdir_cbk.op_errno,
-                                      &stub->args.rmdir_cbk.preparent,
-                                      &stub->args.rmdir_cbk.postparent);
-		else
-			stub->args.rmdir_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.rmdir_cbk.op_ret,
-                                                 stub->args.rmdir_cbk.op_errno,
-                                                 &stub->args.rmdir_cbk.preparent,
-                                                 &stub->args.rmdir_cbk.postparent);
-		break;
-	}
-
-	case GF_FOP_SYMLINK:
-	{
-		if (!stub->args.symlink_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.symlink_cbk.op_ret,
-				      stub->args.symlink_cbk.op_errno,
-				      stub->args.symlink_cbk.inode,
-                                      &stub->args.symlink_cbk.buf,
-                                      &stub->args.symlink_cbk.preparent,
-                                      &stub->args.symlink_cbk.postparent);
-		else
-			stub->args.symlink_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.symlink_cbk.op_ret,
-						   stub->args.symlink_cbk.op_errno,
-						   stub->args.symlink_cbk.inode,
-                                                   &stub->args.symlink_cbk.buf,
-                                                   &stub->args.symlink_cbk.preparent,
-                                                   &stub->args.symlink_cbk.postparent);
-	}
-	break;
-  
-	case GF_FOP_RENAME:
-	{
-#if 0
-		if (!stub->args.rename_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.rename_cbk.op_ret,
-				      stub->args.rename_cbk.op_errno,
-				      &stub->args.rename_cbk.buf,
-                                      &stub->args.rename_cbk.preoldparent,
-                                      &stub->args.rename_cbk.postoldparent,
-                                      &stub->args.rename_cbk.prenewparent,
-                                      &stub->args.rename_cbk.postnewparent);
-		else
-			stub->args.rename_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.rename_cbk.op_ret,
-						  stub->args.rename_cbk.op_errno,
-						  &stub->args.rename_cbk.buf,
-                                                  &stub->args.rename_cbk.preoldparent,
-                                                  &stub->args.rename_cbk.postoldparent,
-                                                  &stub->args.rename_cbk.prenewparent,
-                                                  &stub->args.rename_cbk.postnewparent);
-#endif
-		break;
-	}
-  
-	case GF_FOP_LINK:
-	{
-		if (!stub->args.link_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.link_cbk.op_ret,
-				      stub->args.link_cbk.op_errno,
-				      stub->args.link_cbk.inode,
-				      &stub->args.link_cbk.buf);
-		else
-			stub->args.link_cbk.fn (stub->frame,
-						stub->frame->cookie,
-						stub->frame->this,
-						stub->args.link_cbk.op_ret,
-						stub->args.link_cbk.op_errno,
-						stub->args.link_cbk.inode,
-                                                &stub->args.link_cbk.buf,
-                                                &stub->args.link_cbk.preparent,
-                                                &stub->args.link_cbk.postparent);
-		break;
-	}
-  
-	case GF_FOP_TRUNCATE:
-	{
-		if (!stub->args.truncate_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.truncate_cbk.op_ret,
-				      stub->args.truncate_cbk.op_errno,
-				      &stub->args.truncate_cbk.prebuf,
-                                      &stub->args.truncate_cbk.postbuf);
-		else
-			stub->args.truncate_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.truncate_cbk.op_ret,
-						    stub->args.truncate_cbk.op_errno,
-						    &stub->args.truncate_cbk.prebuf,
-                                                    &stub->args.truncate_cbk.postbuf);
-		break;
-	}
-      
-	case GF_FOP_READ:
-	{
-		if (!stub->args.readv_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.readv_cbk.op_ret,
-				      stub->args.readv_cbk.op_errno,
-				      stub->args.readv_cbk.vector,
-				      stub->args.readv_cbk.count,
-				      &stub->args.readv_cbk.stbuf,
-                                      stub->args.readv_cbk.iobref);
-		else
-			stub->args.readv_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.readv_cbk.op_ret,
-						 stub->args.readv_cbk.op_errno,
-						 stub->args.readv_cbk.vector,
-						 stub->args.readv_cbk.count,
-						 &stub->args.readv_cbk.stbuf,
-                                                 stub->args.readv_cbk.iobref);
-	}
-	break;
-  
-	case GF_FOP_WRITE:
-	{
-		if (!stub->args.writev_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.writev_cbk.op_ret,
-				      stub->args.writev_cbk.op_errno,
-                                      &stub->args.writev_cbk.prebuf,
-				      &stub->args.writev_cbk.postbuf);
-		else
-			stub->args.writev_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.writev_cbk.op_ret,
-						  stub->args.writev_cbk.op_errno,
-                                                  &stub->args.writev_cbk.prebuf,
-						  &stub->args.writev_cbk.postbuf);
-		break;
-	}
-  
-	case GF_FOP_STATFS:
-	{
-		if (!stub->args.statfs_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.statfs_cbk.op_ret,
-				      stub->args.statfs_cbk.op_errno,
-				      &(stub->args.statfs_cbk.buf));
-		else
-			stub->args.statfs_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.statfs_cbk.op_ret,
-						  stub->args.statfs_cbk.op_errno,
-						  &(stub->args.statfs_cbk.buf));
-	}
-	break;
-  
-	case GF_FOP_FLUSH:
-	{
-		if (!stub->args.flush_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.flush_cbk.op_ret,
-				      stub->args.flush_cbk.op_errno);
-		else
-			stub->args.flush_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.flush_cbk.op_ret,
-						 stub->args.flush_cbk.op_errno);
-      
-		break;
-	}
-  
-	case GF_FOP_FSYNC:
-	{
-		if (!stub->args.fsync_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fsync_cbk.op_ret,
-				      stub->args.fsync_cbk.op_errno,
-                                      &stub->args.fsync_cbk.prebuf,
-                                      &stub->args.fsync_cbk.postbuf);
-		else
-			stub->args.fsync_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.fsync_cbk.op_ret,
-						 stub->args.fsync_cbk.op_errno,
-                                                 &stub->args.fsync_cbk.prebuf,
-                                                 &stub->args.fsync_cbk.postbuf);
-		break;
-	}
-  
-	case GF_FOP_SETXATTR:
-	{
-		if (!stub->args.setxattr_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.setxattr_cbk.op_ret,
-				      stub->args.setxattr_cbk.op_errno);
-
-		else
-			stub->args.setxattr_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.setxattr_cbk.op_ret,
-						    stub->args.setxattr_cbk.op_errno);
-
-		break;
-	}
-  
-	case GF_FOP_GETXATTR:
-	{
-		if (!stub->args.getxattr_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.getxattr_cbk.op_ret,
-				      stub->args.getxattr_cbk.op_errno,
-				      stub->args.getxattr_cbk.dict);
-		else
-			stub->args.getxattr_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.getxattr_cbk.op_ret,
-						    stub->args.getxattr_cbk.op_errno,
-						    stub->args.getxattr_cbk.dict);
-		break;
-	}
-
-	case GF_FOP_FSETXATTR:
-	{
-		if (!stub->args.fsetxattr_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fsetxattr_cbk.op_ret,
-				      stub->args.fsetxattr_cbk.op_errno);
-
-		else
-			stub->args.fsetxattr_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.fsetxattr_cbk.op_ret,
-                                                     stub->args.fsetxattr_cbk.op_errno);
-
-		break;
-	}
-
-	case GF_FOP_FGETXATTR:
-	{
-		if (!stub->args.fgetxattr_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fgetxattr_cbk.op_ret,
-				      stub->args.fgetxattr_cbk.op_errno,
-				      stub->args.fgetxattr_cbk.dict);
-		else
-			stub->args.fgetxattr_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.fgetxattr_cbk.op_ret,
-                                                     stub->args.fgetxattr_cbk.op_errno,
-                                                     stub->args.fgetxattr_cbk.dict);
-		break;
-	}
-
-	case GF_FOP_REMOVEXATTR:
-	{
-		if (!stub->args.removexattr_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.removexattr_cbk.op_ret,
-				      stub->args.removexattr_cbk.op_errno);
-		else
-			stub->args.removexattr_cbk.fn (stub->frame,
-						       stub->frame->cookie,
-						       stub->frame->this,
-						       stub->args.removexattr_cbk.op_ret,
-						       stub->args.removexattr_cbk.op_errno);
-
-		break;
-	}
-  
-	case GF_FOP_OPENDIR:
-	{
-		if (!stub->args.opendir_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.opendir_cbk.op_ret,
-				      stub->args.opendir_cbk.op_errno,
-				      stub->args.opendir_cbk.fd);
-		else
-			stub->args.opendir_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.opendir_cbk.op_ret,
-						   stub->args.opendir_cbk.op_errno,
-						   stub->args.opendir_cbk.fd);
-		break;
-	}
-  
-	case GF_FOP_FSYNCDIR:
-	{
-		if (!stub->args.fsyncdir_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fsyncdir_cbk.op_ret,
-				      stub->args.fsyncdir_cbk.op_errno);
-		else
-			stub->args.fsyncdir_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.fsyncdir_cbk.op_ret,
-						    stub->args.fsyncdir_cbk.op_errno);
-		break;
-	}
-  
-	case GF_FOP_ACCESS:
-	{
-		if (!stub->args.access_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.access_cbk.op_ret,
-				      stub->args.access_cbk.op_errno);
-		else
-			stub->args.access_cbk.fn (stub->frame,
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.access_cbk.op_ret,
-						  stub->args.access_cbk.op_errno);
-
-		break;
-	}
-  
-	case GF_FOP_FTRUNCATE:
-	{
-		if (!stub->args.ftruncate_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.ftruncate_cbk.op_ret,
-				      stub->args.ftruncate_cbk.op_errno,
-				      &stub->args.ftruncate_cbk.prebuf,
-				      &stub->args.ftruncate_cbk.postbuf);
-		else
-			stub->args.ftruncate_cbk.fn (stub->frame,
-						     stub->frame->cookie,
-						     stub->frame->this,
-						     stub->args.ftruncate_cbk.op_ret,
-						     stub->args.ftruncate_cbk.op_errno,
-						     &stub->args.ftruncate_cbk.prebuf,
-						     &stub->args.ftruncate_cbk.postbuf);
-		break;
-	}
-  
-	case GF_FOP_FSTAT:
-	{
-		if (!stub->args.fstat_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fstat_cbk.op_ret,
-				      stub->args.fstat_cbk.op_errno,
-				      &stub->args.fstat_cbk.buf);
-		else
-			stub->args.fstat_cbk.fn (stub->frame,
-						 stub->frame->cookie,
-						 stub->frame->this,
-						 stub->args.fstat_cbk.op_ret,
-						 stub->args.fstat_cbk.op_errno,
-						 &stub->args.fstat_cbk.buf);
-      
-		break;
-	}
-  
-	case GF_FOP_LK:
-	{
-		if (!stub->args.lk_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.lk_cbk.op_ret,
-				      stub->args.lk_cbk.op_errno,
-				      &stub->args.lk_cbk.lock);
-		else
-			stub->args.lk_cbk.fn (stub->frame,
-					      stub->frame->cookie,
-					      stub->frame->this,
-					      stub->args.lk_cbk.op_ret,
-					      stub->args.lk_cbk.op_errno,
-					      &stub->args.lk_cbk.lock);
-		break;
-	}
-
-	case GF_FOP_INODELK:
-	{
-		if (!stub->args.inodelk_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.inodelk_cbk.op_ret,
-				      stub->args.inodelk_cbk.op_errno);
-
-		else
-			stub->args.inodelk_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.inodelk_cbk.op_ret,
-						   stub->args.inodelk_cbk.op_errno);
-		break;
-	}
-
-	case GF_FOP_FINODELK:
-	{
-		if (!stub->args.finodelk_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.finodelk_cbk.op_ret,
-				      stub->args.finodelk_cbk.op_errno);
-
-		else
-			stub->args.finodelk_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.finodelk_cbk.op_ret,
-						    stub->args.finodelk_cbk.op_errno);
-		break;
-	}
-
-	case GF_FOP_ENTRYLK:
-	{
-		if (!stub->args.entrylk_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.entrylk_cbk.op_ret,
-				      stub->args.entrylk_cbk.op_errno);
-
-		else
-			stub->args.entrylk_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.entrylk_cbk.op_ret,
-						   stub->args.entrylk_cbk.op_errno);
-		break;
-	}
-
-	case GF_FOP_FENTRYLK:
-	{
-		if (!stub->args.fentrylk_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fentrylk_cbk.op_ret,
-				      stub->args.fentrylk_cbk.op_errno);
-
-		else
-			stub->args.fentrylk_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.fentrylk_cbk.op_ret,
-						    stub->args.fentrylk_cbk.op_errno);
-		break;
-	}
-  
-	case GF_FOP_LOOKUP:
-	{
-		if (!stub->args.lookup_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.lookup_cbk.op_ret,
-				      stub->args.lookup_cbk.op_errno,
-				      stub->args.lookup_cbk.inode,
-				      &stub->args.lookup_cbk.buf,
-                                      stub->args.lookup_cbk.dict,
-                                      &stub->args.lookup_cbk.postparent);
-		else
-			stub->args.lookup_cbk.fn (stub->frame, 
-						  stub->frame->cookie,
-						  stub->frame->this,
-						  stub->args.lookup_cbk.op_ret,
-						  stub->args.lookup_cbk.op_errno,
-						  stub->args.lookup_cbk.inode,
-                                                  &stub->args.lookup_cbk.buf,
-                                                  stub->args.lookup_cbk.dict,
-                                                  &stub->args.lookup_cbk.postparent);
-		/* FIXME NULL should not be passed */
-
-		if (stub->args.lookup_cbk.dict)
-			dict_unref (stub->args.lookup_cbk.dict);
-		if (stub->args.lookup_cbk.inode)
-			inode_unref (stub->args.lookup_cbk.inode);
-
-		break;
-	}
-
-	case GF_FOP_RCHECKSUM:
-	{
-		if (!stub->args.rchecksum_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.rchecksum_cbk.op_ret,
-				      stub->args.rchecksum_cbk.op_errno,
-				      stub->args.rchecksum_cbk.weak_checksum,
-				      stub->args.rchecksum_cbk.strong_checksum);
-		else
-			stub->args.rchecksum_cbk.fn (stub->frame, 
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.rchecksum_cbk.op_ret, 
-                                                     stub->args.rchecksum_cbk.op_errno,
-                                                     stub->args.rchecksum_cbk.weak_checksum,
-                                                     stub->args.rchecksum_cbk.strong_checksum);
-		if (stub->args.rchecksum_cbk.op_ret >= 0)
-		{
-			GF_FREE (stub->args.rchecksum_cbk.strong_checksum);
-		}
-
-		break;
-	}
-
-	case GF_FOP_READDIR:
-	{
-		if (!stub->args.readdir_cbk.fn) 
-			STACK_UNWIND (stub->frame,
-				      stub->args.readdir_cbk.op_ret,
-				      stub->args.readdir_cbk.op_errno,
-				      &stub->args.readdir_cbk.entries);
-		else 
-			stub->args.readdir_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.readdir_cbk.op_ret,
-						   stub->args.readdir_cbk.op_errno,
-						   &stub->args.readdir_cbk.entries);
-		
-		if (stub->args.readdir_cbk.op_ret > 0) 
-			gf_dirent_free (&stub->args.readdir_cbk.entries);
-
-		break;
-	}
+args_namelink_store_cbk(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+}
+
+call_stub_t *
+fop_namelink_cbk_stub(call_frame_t *frame, fop_namelink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    stub = stub_new(frame, 0, GF_FOP_NAMELINK);
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    stub->fn_cbk.namelink = fn;
+    args_namelink_store_cbk(&stub->args_cbk, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+
+out:
+    return stub;
+}
 
+void
+call_resume_wind(call_stub_t *stub)
+{
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    switch (stub->fop) {
+        case GF_FOP_OPEN:
+            stub->fn.open(stub->frame, stub->frame->this, &stub->args.loc,
+                          stub->args.flags, stub->args.fd, stub->args.xdata);
+            break;
+        case GF_FOP_CREATE:
+            stub->fn.create(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.flags, stub->args.mode, stub->args.umask,
+                            stub->args.fd, stub->args.xdata);
+            break;
+        case GF_FOP_STAT:
+            stub->fn.stat(stub->frame, stub->frame->this, &stub->args.loc,
+                          stub->args.xdata);
+            break;
+        case GF_FOP_READLINK:
+            stub->fn.readlink(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.size, stub->args.xdata);
+            break;
+        case GF_FOP_MKNOD:
+            stub->fn.mknod(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.mode, stub->args.rdev, stub->args.umask,
+                           stub->args.xdata);
+            break;
+        case GF_FOP_MKDIR:
+            stub->fn.mkdir(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.mode, stub->args.umask, stub->args.xdata);
+            break;
+        case GF_FOP_UNLINK:
+            stub->fn.unlink(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xflag, stub->args.xdata);
+            break;
+        case GF_FOP_RMDIR:
+            stub->fn.rmdir(stub->frame, stub->frame->this, &stub->args.loc,
+                           stub->args.flags, stub->args.xdata);
+            break;
+        case GF_FOP_SYMLINK:
+            stub->fn.symlink(stub->frame, stub->frame->this,
+                             stub->args.linkname, &stub->args.loc,
+                             stub->args.umask, stub->args.xdata);
+            break;
+        case GF_FOP_RENAME:
+            stub->fn.rename(stub->frame, stub->frame->this, &stub->args.loc,
+                            &stub->args.loc2, stub->args.xdata);
+            break;
+        case GF_FOP_LINK:
+            stub->fn.link(stub->frame, stub->frame->this, &stub->args.loc,
+                          &stub->args.loc2, stub->args.xdata);
+            break;
+        case GF_FOP_TRUNCATE:
+            stub->fn.truncate(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.offset, stub->args.xdata);
+            break;
+        case GF_FOP_READ:
+            stub->fn.readv(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.size, stub->args.offset, stub->args.flags,
+                           stub->args.xdata);
+            break;
+        case GF_FOP_WRITE:
+            stub->fn.writev(stub->frame, stub->frame->this, stub->args.fd,
+                            stub->args.vector, stub->args.count,
+                            stub->args.offset, stub->args.flags,
+                            stub->args.iobref, stub->args.xdata);
+            break;
+        case GF_FOP_STATFS:
+            stub->fn.statfs(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xdata);
+            break;
+        case GF_FOP_FLUSH:
+            stub->fn.flush(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.xdata);
+            break;
+        case GF_FOP_FSYNC:
+            stub->fn.fsync(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.datasync, stub->args.xdata);
+            break;
+        case GF_FOP_SETXATTR:
+            stub->fn.setxattr(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.xattr, stub->args.flags,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_GETXATTR:
+            stub->fn.getxattr(stub->frame, stub->frame->this, &stub->args.loc,
+                              stub->args.name, stub->args.xdata);
+            break;
+        case GF_FOP_FSETXATTR:
+            stub->fn.fsetxattr(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.xattr, stub->args.flags,
+                               stub->args.xdata);
+            break;
+        case GF_FOP_FGETXATTR:
+            stub->fn.fgetxattr(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.name, stub->args.xdata);
+            break;
+        case GF_FOP_REMOVEXATTR:
+            stub->fn.removexattr(stub->frame, stub->frame->this,
+                                 &stub->args.loc, stub->args.name,
+                                 stub->args.xdata);
+            break;
+        case GF_FOP_FREMOVEXATTR:
+            stub->fn.fremovexattr(stub->frame, stub->frame->this, stub->args.fd,
+                                  stub->args.name, stub->args.xdata);
+            break;
+        case GF_FOP_OPENDIR:
+            stub->fn.opendir(stub->frame, stub->frame->this, &stub->args.loc,
+                             stub->args.fd, stub->args.xdata);
+            break;
+        case GF_FOP_FSYNCDIR:
+            stub->fn.fsyncdir(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.datasync, stub->args.xdata);
+            break;
+        case GF_FOP_ACCESS:
+            stub->fn.access(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.mask, stub->args.xdata);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            stub->fn.ftruncate(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.offset, stub->args.xdata);
+            break;
+        case GF_FOP_FSTAT:
+            stub->fn.fstat(stub->frame, stub->frame->this, stub->args.fd,
+                           stub->args.xdata);
+            break;
+        case GF_FOP_LK:
+            stub->fn.lk(stub->frame, stub->frame->this, stub->args.fd,
+                        stub->args.cmd, &stub->args.lock, stub->args.xdata);
+            break;
+        case GF_FOP_INODELK:
+            stub->fn.inodelk(stub->frame, stub->frame->this, stub->args.volume,
+                             &stub->args.loc, stub->args.cmd, &stub->args.lock,
+                             stub->args.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            stub->fn.finodelk(stub->frame, stub->frame->this, stub->args.volume,
+                              stub->args.fd, stub->args.cmd, &stub->args.lock,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            stub->fn.entrylk(stub->frame, stub->frame->this, stub->args.volume,
+                             &stub->args.loc, stub->args.name,
+                             stub->args.entrylkcmd, stub->args.entrylktype,
+                             stub->args.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            stub->fn.fentrylk(stub->frame, stub->frame->this, stub->args.volume,
+                              stub->args.fd, stub->args.name,
+                              stub->args.entrylkcmd, stub->args.entrylktype,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_LOOKUP:
+            stub->fn.lookup(stub->frame, stub->frame->this, &stub->args.loc,
+                            stub->args.xdata);
+            break;
+        case GF_FOP_RCHECKSUM:
+            stub->fn.rchecksum(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.offset, stub->args.size,
+                               stub->args.xdata);
+            break;
+        case GF_FOP_READDIR:
+            stub->fn.readdir(stub->frame, stub->frame->this, stub->args.fd,
+                             stub->args.size, stub->args.offset,
+                             stub->args.xdata);
+            break;
         case GF_FOP_READDIRP:
-	{
-		if (!stub->args.readdirp_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.readdirp_cbk.op_ret,
-				      stub->args.readdirp_cbk.op_errno,
-				      &stub->args.readdirp_cbk.entries);
-		else
-			stub->args.readdirp_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.readdirp_cbk.op_ret,
-						    stub->args.readdirp_cbk.op_errno,
-						    &stub->args.readdirp_cbk.entries);
-
-		if (stub->args.readdirp_cbk.op_ret > 0)
-			gf_dirent_free (&stub->args.readdirp_cbk.entries);
-
-		break;
-	}
-
-	case GF_FOP_XATTROP:
-	{
-		if (!stub->args.xattrop_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.xattrop_cbk.op_ret,
-				      stub->args.xattrop_cbk.op_errno);
-		else
-			stub->args.xattrop_cbk.fn (stub->frame,
-						   stub->frame->cookie,
-						   stub->frame->this,
-						   stub->args.xattrop_cbk.op_ret,
-						   stub->args.xattrop_cbk.op_errno,
-						   stub->args.xattrop_cbk.xattr);
-
-		if (stub->args.xattrop_cbk.xattr)
-			dict_unref (stub->args.xattrop_cbk.xattr);
-
-		break;
-	}
-	case GF_FOP_FXATTROP:
-	{
-		if (!stub->args.fxattrop_cbk.fn)
-			STACK_UNWIND (stub->frame,
-				      stub->args.fxattrop_cbk.op_ret,
-				      stub->args.fxattrop_cbk.op_errno);
-		else
-			stub->args.fxattrop_cbk.fn (stub->frame,
-						    stub->frame->cookie,
-						    stub->frame->this,
-						    stub->args.fxattrop_cbk.op_ret,
-						    stub->args.fxattrop_cbk.op_errno,
-						    stub->args.fxattrop_cbk.xattr);
-
-		if (stub->args.fxattrop_cbk.xattr)
-			dict_unref (stub->args.fxattrop_cbk.xattr);
-
-		break;
-	}
+            stub->fn.readdirp(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.size, stub->args.offset,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_XATTROP:
+            stub->fn.xattrop(stub->frame, stub->frame->this, &stub->args.loc,
+                             stub->args.optype, stub->args.xattr,
+                             stub->args.xdata);
+            break;
+        case GF_FOP_FXATTROP:
+            stub->fn.fxattrop(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.optype, stub->args.xattr,
+                              stub->args.xdata);
+            break;
         case GF_FOP_SETATTR:
-        {
-                if (!stub->args.setattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.setattr_cbk.op_ret,
-                                      stub->args.setattr_cbk.op_errno,
-                                      &stub->args.setattr_cbk.statpre,
-                                      &stub->args.setattr_cbk.statpost);
-                else
-                        stub->args.setattr_cbk.fn (
-                                stub->frame,
-                                stub->frame->cookie,
-                                stub->frame->this,
-                                stub->args.setattr_cbk.op_ret,
-                                stub->args.setattr_cbk.op_errno,
-                                &stub->args.setattr_cbk.statpre,
-                                &stub->args.setattr_cbk.statpost);
-                break;
-        }
+            stub->fn.setattr(stub->frame, stub->frame->this, &stub->args.loc,
+                             &stub->args.stat, stub->args.valid,
+                             stub->args.xdata);
+            break;
         case GF_FOP_FSETATTR:
-        {
-                if (!stub->args.fsetattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fsetattr_cbk.op_ret,
-                                      stub->args.fsetattr_cbk.op_errno,
-                                      &stub->args.fsetattr_cbk.statpre,
-                                      &stub->args.fsetattr_cbk.statpost);
-                else
-                        stub->args.fsetattr_cbk.fn (
-                                stub->frame,
-                                stub->frame->cookie,
-                                stub->frame->this,
-                                stub->args.fsetattr_cbk.op_ret,
-                                stub->args.fsetattr_cbk.op_errno,
-                                &stub->args.fsetattr_cbk.statpre,
-                                &stub->args.fsetattr_cbk.statpost);
-                break;
-        }
-        default:
-	{
-		gf_log ("call-stub", GF_LOG_ERROR, "Invalid value of FOP (%d)",
-                        stub->fop);
-                break;
-	}
-	}
-out:
-	return;
-}
+            stub->fn.fsetattr(stub->frame, stub->frame->this, stub->args.fd,
+                              &stub->args.stat, stub->args.valid,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_FALLOCATE:
+            stub->fn.fallocate(stub->frame, stub->frame->this, stub->args.fd,
+                               stub->args.flags, stub->args.offset,
+                               stub->args.size, stub->args.xdata);
+            break;
+        case GF_FOP_DISCARD:
+            stub->fn.discard(stub->frame, stub->frame->this, stub->args.fd,
+                             stub->args.offset, stub->args.size,
+                             stub->args.xdata);
+            break;
+        case GF_FOP_ZEROFILL:
+            stub->fn.zerofill(stub->frame, stub->frame->this, stub->args.fd,
+                              stub->args.offset, stub->args.size,
+                              stub->args.xdata);
+            break;
+        case GF_FOP_IPC:
+            stub->fn.ipc(stub->frame, stub->frame->this, stub->args.cmd,
+                         stub->args.xdata);
+            break;
+        case GF_FOP_SEEK:
+            stub->fn.seek(stub->frame, stub->frame->this, stub->args.fd,
+                          stub->args.offset, stub->args.what, stub->args.xdata);
+            break;
+        case GF_FOP_LEASE:
+            stub->fn.lease(stub->frame, stub->frame->this, &stub->args.loc,
+                           &stub->args.lease, stub->args.xdata);
+            break;
+
+        case GF_FOP_GETACTIVELK:
+            stub->fn.getactivelk(stub->frame, stub->frame->this,
+                                 &stub->args.loc, stub->args.xdata);
+            break;
+
+        case GF_FOP_SETACTIVELK:
+            stub->fn.setactivelk(stub->frame, stub->frame->this,
+                                 &stub->args.loc, &stub->args.locklist,
+                                 stub->args.xdata);
+            break;
+
+        case GF_FOP_PUT:
+            stub->fn.put(stub->frame, stub->frame->this, &stub->args.loc,
+                         stub->args.mode, stub->args.umask, stub->args.flags,
+                         stub->args.vector, stub->args.count, stub->args.offset,
+                         stub->args.iobref, stub->args.xattr, stub->args.xdata);
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            stub->fn.copy_file_range(
+                stub->frame, stub->frame->this, stub->args.fd,
+                stub->args.off_in, stub->args.fd_dst, stub->args.off_out,
+                stub->args.size, stub->args.flags, stub->args.xdata);
+            break;
 
+        default:
+            gf_msg_callingfn("call-stub", GF_LOG_ERROR, EINVAL,
+                             LG_MSG_INVALID_ENTRY,
+                             "Invalid value of FOP"
+                             " (%d)",
+                             stub->fop);
+            break;
+    }
+out:
+    return;
+}
+
+#define STUB_UNWIND(stb, fop, args...)                                         \
+    do {                                                                       \
+        if (stb->fn_cbk.fop)                                                   \
+            stb->fn_cbk.fop(stb->frame, stb->frame->cookie, stb->frame->this,  \
+                            stb->args_cbk.op_ret, stb->args_cbk.op_errno,      \
+                            args);                                             \
+        else                                                                   \
+            STACK_UNWIND_STRICT(fop, stb->frame, stb->args_cbk.op_ret,         \
+                                stb->args_cbk.op_errno, args);                 \
+    } while (0)
 
 static void
-call_stub_destroy_wind (call_stub_t *stub)
-{
-	switch (stub->fop) {
-	case GF_FOP_OPEN:
-	{
-		loc_wipe (&stub->args.open.loc);
-		if (stub->args.open.fd)
-			fd_unref (stub->args.open.fd);
-		break;
-	}
-	case GF_FOP_CREATE:
-	{
-		loc_wipe (&stub->args.create.loc);
-		if (stub->args.create.fd)
-			fd_unref (stub->args.create.fd);
-                if (stub->args.create.params)
-                        dict_unref (stub->args.create.params);
-		break;
-	}
-	case GF_FOP_STAT:
-	{
-		loc_wipe (&stub->args.stat.loc);
-		break;
-	}
-	case GF_FOP_READLINK:
-	{
-		loc_wipe (&stub->args.readlink.loc);
-		break;
-	}
-  
-	case GF_FOP_MKNOD:
-	{
-		loc_wipe (&stub->args.mknod.loc);
-                if (stub->args.mknod.params)
-                        dict_unref (stub->args.mknod.params);
-	}
-	break;
-  
-	case GF_FOP_MKDIR:
-	{
-		loc_wipe (&stub->args.mkdir.loc);
-                if (stub->args.mkdir.params)
-                        dict_unref (stub->args.mkdir.params);
-	}
-	break;
-  
-	case GF_FOP_UNLINK:
-	{
-		loc_wipe (&stub->args.unlink.loc);
-	}
-	break;
-
-	case GF_FOP_RMDIR:
-	{
-		loc_wipe (&stub->args.rmdir.loc);
-	}
-	break;
-      
-	case GF_FOP_SYMLINK:
-	{
-		GF_FREE ((char *)stub->args.symlink.linkname);
-		loc_wipe (&stub->args.symlink.loc);
-                if (stub->args.symlink.params)
-                        dict_unref (stub->args.symlink.params);
-	}
-	break;
-  
-	case GF_FOP_RENAME:
-	{
-		loc_wipe (&stub->args.rename.old);
-		loc_wipe (&stub->args.rename.new);
-	}
-	break;
-
-	case GF_FOP_LINK:
-	{
-		loc_wipe (&stub->args.link.oldloc);
-		loc_wipe (&stub->args.link.newloc);
-	}
-	break;
-  
-	case GF_FOP_TRUNCATE:
-	{
-		loc_wipe (&stub->args.truncate.loc);
-		break;
-	}
-      
-	case GF_FOP_READ:
-	{
-		if (stub->args.readv.fd)
-			fd_unref (stub->args.readv.fd);
-		break;
-	}
-  
-	case GF_FOP_WRITE:
-	{
-		struct iobref *iobref = stub->args.writev.iobref;
-		if (stub->args.writev.fd)
-			fd_unref (stub->args.writev.fd);
-		GF_FREE (stub->args.writev.vector);
-		if (iobref)
-			iobref_unref (iobref);
-		break;
-	}
-  
-	case GF_FOP_STATFS:
-	{
-		loc_wipe (&stub->args.statfs.loc);
-		break;
-	}
-	case GF_FOP_FLUSH:
-	{
-		if (stub->args.flush.fd)
-			fd_unref (stub->args.flush.fd);      
-		break;
-	}
-  
-	case GF_FOP_FSYNC:
-	{
-		if (stub->args.fsync.fd)
-			fd_unref (stub->args.fsync.fd);
-		break;
-	}
-
-	case GF_FOP_SETXATTR:
-	{
-		loc_wipe (&stub->args.setxattr.loc);
-		if (stub->args.setxattr.dict)
-			dict_unref (stub->args.setxattr.dict);
-		break;
-	}
-  
-	case GF_FOP_GETXATTR:
-	{
-		if (stub->args.getxattr.name)
-			GF_FREE ((char *)stub->args.getxattr.name);
-		loc_wipe (&stub->args.getxattr.loc);
-		break;
-	}
-
-	case GF_FOP_FSETXATTR:
-	{
-		fd_unref (stub->args.fsetxattr.fd);
-		if (stub->args.fsetxattr.dict)
-			dict_unref (stub->args.fsetxattr.dict);
-		break;
-	}
-
-	case GF_FOP_FGETXATTR:
-	{
-		if (stub->args.fgetxattr.name)
-			GF_FREE ((char *)stub->args.fgetxattr.name);
-		fd_unref (stub->args.fgetxattr.fd);
-		break;
-	}
-
-	case GF_FOP_REMOVEXATTR:
-	{
-		loc_wipe (&stub->args.removexattr.loc);
-		GF_FREE ((char *)stub->args.removexattr.name);
-		break;
-	}
-
-	case GF_FOP_OPENDIR:
-	{
-		loc_wipe (&stub->args.opendir.loc);
-		if (stub->args.opendir.fd)
-			fd_unref (stub->args.opendir.fd);
-		break;
-	}
-
-	case GF_FOP_FSYNCDIR:
-	{
-		if (stub->args.fsyncdir.fd)
-			fd_unref (stub->args.fsyncdir.fd);
-		break;
-	}
-  
-	case GF_FOP_ACCESS:
-	{
-		loc_wipe (&stub->args.access.loc);
-		break;
-	}
-  
-	case GF_FOP_FTRUNCATE:
-	{
-		if (stub->args.ftruncate.fd)
-			fd_unref (stub->args.ftruncate.fd);
-		break;
-	}
-  
-	case GF_FOP_FSTAT:
-	{
-		if (stub->args.fstat.fd)
-			fd_unref (stub->args.fstat.fd);
-		break;
-	}
-  
-	case GF_FOP_LK:
-	{
-		if (stub->args.lk.fd)
-			fd_unref (stub->args.lk.fd);
-		break;
-	}
-
-	case GF_FOP_INODELK:
-	{
-                if (stub->args.inodelk.volume)
-                        GF_FREE ((char *)stub->args.inodelk.volume);
-
-		loc_wipe (&stub->args.inodelk.loc);
-		break;
-	}
-	case GF_FOP_FINODELK:
-	{
-                if (stub->args.finodelk.volume)
-                        GF_FREE ((char *)stub->args.finodelk.volume);
-
-		if (stub->args.finodelk.fd)
-			fd_unref (stub->args.finodelk.fd);
-		break;
-	}
-	case GF_FOP_ENTRYLK:
-	{
-                if (stub->args.entrylk.volume)
-                        GF_FREE ((char *)stub->args.entrylk.volume);
-
-		if (stub->args.entrylk.name)
-			GF_FREE ((char *)stub->args.entrylk.name);
-		loc_wipe (&stub->args.entrylk.loc);
-		break;
-	}
-	case GF_FOP_FENTRYLK:
-	{
-                if (stub->args.fentrylk.volume)
-                        GF_FREE ((char *)stub->args.fentrylk.volume);
-
-		if (stub->args.fentrylk.name)
-			GF_FREE ((char *)stub->args.fentrylk.name);
-
- 		if (stub->args.fentrylk.fd)
-			fd_unref (stub->args.fentrylk.fd);
-		break;
-	}
-  
-	case GF_FOP_LOOKUP:
-	{
-		loc_wipe (&stub->args.lookup.loc);
-		if (stub->args.lookup.xattr_req)
-			dict_unref (stub->args.lookup.xattr_req);
-		break;
-	}
-
-	case GF_FOP_RCHECKSUM:
-	{
-                if (stub->args.rchecksum.fd)
-                        fd_unref (stub->args.rchecksum.fd);
-		break;
-	}
-
-	case GF_FOP_READDIR:
-	{
-		if (stub->args.readdir.fd)
-			fd_unref (stub->args.readdir.fd);
-		break;
-	}
-
+call_resume_unwind(call_stub_t *stub)
+{
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    switch (stub->fop) {
+        case GF_FOP_OPEN:
+            STUB_UNWIND(stub, open, stub->args_cbk.fd, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_CREATE:
+            STUB_UNWIND(stub, create, stub->args_cbk.fd, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_STAT:
+            STUB_UNWIND(stub, stat, &stub->args_cbk.stat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_READLINK:
+            STUB_UNWIND(stub, readlink, stub->args_cbk.buf,
+                        &stub->args_cbk.stat, stub->args.xdata);
+            break;
+        case GF_FOP_MKNOD:
+            STUB_UNWIND(stub, mknod, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_MKDIR:
+            STUB_UNWIND(stub, mkdir, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_UNLINK:
+            STUB_UNWIND(stub, unlink, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_RMDIR:
+            STUB_UNWIND(stub, rmdir, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_SYMLINK:
+            STUB_UNWIND(stub, symlink, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, &stub->args_cbk.preparent,
+                        &stub->args_cbk.postparent, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_RENAME:
+            STUB_UNWIND(stub, rename, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        &stub->args_cbk.preparent2, &stub->args_cbk.postparent2,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_LINK:
+            STUB_UNWIND(stub, link, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_TRUNCATE:
+            STUB_UNWIND(stub, truncate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_READ:
+            STUB_UNWIND(stub, readv, stub->args_cbk.vector,
+                        stub->args_cbk.count, &stub->args_cbk.stat,
+                        stub->args_cbk.iobref, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_WRITE:
+            STUB_UNWIND(stub, writev, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_STATFS:
+            STUB_UNWIND(stub, statfs, &stub->args_cbk.statvfs,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FLUSH:
+            STUB_UNWIND(stub, flush, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FSYNC:
+            STUB_UNWIND(stub, fsync, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_SETXATTR:
+            STUB_UNWIND(stub, setxattr, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_GETXATTR:
+            STUB_UNWIND(stub, getxattr, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FSETXATTR:
+            STUB_UNWIND(stub, fsetxattr, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FGETXATTR:
+            STUB_UNWIND(stub, fgetxattr, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_REMOVEXATTR:
+            STUB_UNWIND(stub, removexattr, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FREMOVEXATTR:
+            STUB_UNWIND(stub, fremovexattr, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_OPENDIR:
+            STUB_UNWIND(stub, opendir, stub->args_cbk.fd, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FSYNCDIR:
+            STUB_UNWIND(stub, fsyncdir, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_ACCESS:
+            STUB_UNWIND(stub, access, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FTRUNCATE:
+            STUB_UNWIND(stub, ftruncate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FSTAT:
+            STUB_UNWIND(stub, fstat, &stub->args_cbk.stat,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_LK:
+            STUB_UNWIND(stub, lk, &stub->args_cbk.lock, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_INODELK:
+            STUB_UNWIND(stub, inodelk, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            STUB_UNWIND(stub, finodelk, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            STUB_UNWIND(stub, entrylk, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            STUB_UNWIND(stub, fentrylk, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_LOOKUP:
+            STUB_UNWIND(stub, lookup, stub->args_cbk.inode,
+                        &stub->args_cbk.stat, stub->args_cbk.xdata,
+                        &stub->args_cbk.postparent);
+            break;
+        case GF_FOP_RCHECKSUM:
+            STUB_UNWIND(stub, rchecksum, stub->args_cbk.weak_checksum,
+                        stub->args_cbk.strong_checksum, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_READDIR:
+            STUB_UNWIND(stub, readdir, &stub->args_cbk.entries,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_READDIRP:
-	{
-		if (stub->args.readdirp.fd)
-			fd_unref (stub->args.readdirp.fd);
-		break;
-	}
-
-	case GF_FOP_XATTROP:
-	{
-		loc_wipe (&stub->args.xattrop.loc);
-		dict_unref (stub->args.xattrop.xattr);
-		break;
-	}
-	case GF_FOP_FXATTROP:
-	{
-		if (stub->args.fxattrop.fd)
-			fd_unref (stub->args.fxattrop.fd);
-		dict_unref (stub->args.fxattrop.xattr);
-		break;
-	}
+            STUB_UNWIND(stub, readdir, &stub->args_cbk.entries,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_XATTROP:
+            STUB_UNWIND(stub, xattrop, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FXATTROP:
+            STUB_UNWIND(stub, fxattrop, stub->args_cbk.xattr,
+                        stub->args_cbk.xdata);
+            break;
         case GF_FOP_SETATTR:
-        {
-                loc_wipe (&stub->args.setattr.loc);
-                break;
-        }
+            STUB_UNWIND(stub, setattr, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
         case GF_FOP_FSETATTR:
-        {
-                if (stub->args.fsetattr.fd)
-                        fd_unref (stub->args.fsetattr.fd);
-                break;
-        }
+            STUB_UNWIND(stub, fsetattr, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_FALLOCATE:
+            STUB_UNWIND(stub, fallocate, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_DISCARD:
+            STUB_UNWIND(stub, discard, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_ZEROFILL:
+            STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat,
+                        &stub->args_cbk.poststat, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_IPC:
+            STUB_UNWIND(stub, ipc, stub->args_cbk.xdata);
+            break;
+        case GF_FOP_SEEK:
+            STUB_UNWIND(stub, seek, stub->args_cbk.offset,
+                        stub->args_cbk.xdata);
+            break;
+        case GF_FOP_LEASE:
+            STUB_UNWIND(stub, lease, &stub->args_cbk.lease,
+                        stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_GETACTIVELK:
+            STUB_UNWIND(stub, getactivelk, &stub->args_cbk.locklist,
+                        stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_SETACTIVELK:
+            STUB_UNWIND(stub, setactivelk, stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_PUT:
+            STUB_UNWIND(stub, put, stub->args_cbk.inode, &stub->args_cbk.stat,
+                        &stub->args_cbk.preparent, &stub->args_cbk.postparent,
+                        stub->args_cbk.xdata);
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            STUB_UNWIND(stub, copy_file_range, &stub->args_cbk.stat,
+                        &stub->args_cbk.prestat, &stub->args_cbk.poststat,
+                        stub->args_cbk.xdata);
+            break;
+
         default:
-	{
-		gf_log ("call-stub", GF_LOG_ERROR, "Invalid value of FOP (%d)",
-                        stub->fop);
-                break;
-	}
-	}
+            gf_msg_callingfn("call-stub", GF_LOG_ERROR, EINVAL,
+                             LG_MSG_INVALID_ENTRY,
+                             "Invalid value of FOP"
+                             " (%d)",
+                             stub->fop);
+            break;
+    }
+out:
+    return;
 }
 
+static void
+call_stub_wipe_args(call_stub_t *stub)
+{
+    args_wipe(&stub->args);
+}
 
 static void
-call_stub_destroy_unwind (call_stub_t *stub)
-{
-	switch (stub->fop) {
-	case GF_FOP_OPEN:
-	{
-		if (stub->args.open_cbk.fd) 
-			fd_unref (stub->args.open_cbk.fd);
-	}
-	break;
-
-	case GF_FOP_CREATE:
-	{
-		if (stub->args.create_cbk.fd) 
-			fd_unref (stub->args.create_cbk.fd);
-
-		if (stub->args.create_cbk.inode)
-			inode_unref (stub->args.create_cbk.inode);
-	}
-	break;
-
-	case GF_FOP_STAT:
-		break;
-
-	case GF_FOP_READLINK:
-	{
-		if (stub->args.readlink_cbk.buf) 
-			GF_FREE ((char *)stub->args.readlink_cbk.buf);
-	}
-	break;
-  
-	case GF_FOP_MKNOD:
-	{
-		if (stub->args.mknod_cbk.inode)
-			inode_unref (stub->args.mknod_cbk.inode);
-	}
-	break;
-  
-	case GF_FOP_MKDIR:
-	{
-		if (stub->args.mkdir_cbk.inode)
-			inode_unref (stub->args.mkdir_cbk.inode);
-	}
-	break;
-  
-	case GF_FOP_UNLINK:
-		break;
-
-	case GF_FOP_RMDIR:
-		break;
-      
-	case GF_FOP_SYMLINK:
-	{
-		if (stub->args.symlink_cbk.inode) 
-			inode_unref (stub->args.symlink_cbk.inode);
-	}
-	break;
-  
-	case GF_FOP_RENAME:
-		break;
-
-	case GF_FOP_LINK:
-	{
-		if (stub->args.link_cbk.inode)
-			inode_unref (stub->args.link_cbk.inode);
-	}
-	break;
-  
-	case GF_FOP_TRUNCATE:
-		break;
-
-	case GF_FOP_READ:
-	{
-		if (stub->args.readv_cbk.op_ret >= 0) {
-			struct iobref *iobref = stub->args.readv_cbk.iobref;
-			GF_FREE (stub->args.readv_cbk.vector);
-			
-			if (iobref) {
-				iobref_unref (iobref);
-			}
-		}
-	}
-	break;
-
-	case GF_FOP_WRITE:
-		break;
-  
-	case GF_FOP_STATFS:
-		break;
-
-	case GF_FOP_FLUSH:
-		break;
-  
-	case GF_FOP_FSYNC:
-		break;
-
-	case GF_FOP_SETXATTR:
-		break;
-  
-	case GF_FOP_GETXATTR:
-	{
-		if (stub->args.getxattr_cbk.dict)
-			dict_unref (stub->args.getxattr_cbk.dict);
-	}
-	break;
-
-	case GF_FOP_FSETXATTR:
-		break;
-
-	case GF_FOP_FGETXATTR:
-	{
-		if (stub->args.fgetxattr_cbk.dict)
-			dict_unref (stub->args.fgetxattr_cbk.dict);
-	}
-	break;
-
-	case GF_FOP_REMOVEXATTR:
-		break;
-
-	case GF_FOP_OPENDIR:
-	{
-		if (stub->args.opendir_cbk.fd)
-			fd_unref (stub->args.opendir_cbk.fd);
-	}
-	break;
-
-	case GF_FOP_FSYNCDIR:
-		break;
-  
-	case GF_FOP_ACCESS:
-		break;
-  
-	case GF_FOP_FTRUNCATE:
-		break;
-  
-	case GF_FOP_FSTAT:
-		break;
-  
-	case GF_FOP_LK:
-		break;
-
-	case GF_FOP_INODELK:
-		break;
-
-	case GF_FOP_FINODELK:
-		break;
-
-	case GF_FOP_ENTRYLK:
-		break;
-
-	case GF_FOP_FENTRYLK:
-		break;
-
-	case GF_FOP_LOOKUP:
-	{
-		if (stub->args.lookup_cbk.inode)
-			inode_unref (stub->args.lookup_cbk.inode);
-
-		if (stub->args.lookup_cbk.dict)
-			dict_unref (stub->args.lookup_cbk.dict);
-	}
-	break;
-
-	case GF_FOP_RCHECKSUM:
-	{
-		if (stub->args.rchecksum_cbk.op_ret >= 0) {
-			GF_FREE (stub->args.rchecksum_cbk.strong_checksum);
-		}
-	}
-  	break;
-
-	case GF_FOP_READDIR:
-	{
-		if (stub->args.readdir_cbk.op_ret > 0) {
-			gf_dirent_free (&stub->args.readdir_cbk.entries);
-		}
-	}
-	break;
+call_stub_wipe_args_cbk(call_stub_t *stub)
+{
+    args_cbk_wipe(&stub->args_cbk);
+}
 
-        case GF_FOP_READDIRP:
-	{
-		if (stub->args.readdirp_cbk.op_ret > 0) {
-			gf_dirent_free (&stub->args.readdirp_cbk.entries);
-		}
-	}
-	break;
-
-	case GF_FOP_XATTROP:
-	{
-		if (stub->args.xattrop_cbk.xattr)
-			dict_unref (stub->args.xattrop_cbk.xattr);
-	}
-	break;
-
-	case GF_FOP_FXATTROP:
-	{
-		if (stub->args.fxattrop_cbk.xattr) 
-			dict_unref (stub->args.fxattrop_cbk.xattr);
-	}
-	break;
-        
-        case GF_FOP_SETATTR:
-        {
-                break;
-        }
+void
+call_stub_destroy(call_stub_t *stub)
+{
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-        case GF_FOP_FSETATTR:
-        {
-                break;
-        }
+    if (stub->wind)
+        call_stub_wipe_args(stub);
+    else
+        call_stub_wipe_args_cbk(stub);
 
-        default:
-	{
-		gf_log ("call-stub", GF_LOG_ERROR, "Invalid value of FOP (%d)",
-                        stub->fop);
-                break;
-	}
-	}
+    stub->stub_mem_pool = NULL;
+
+    mem_put(stub);
+out:
+    return;
 }
 
- 
 void
-call_stub_destroy (call_stub_t *stub)
+call_resume(call_stub_t *stub)
 {
-       struct mem_pool *tmp_pool = NULL;
+    xlator_t *old_THIS = NULL;
+
+    errno = EINVAL;
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
 
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    list_del_init(&stub->list);
 
-       tmp_pool = stub->stub_mem_pool;
-	
-	if (stub->wind) {
-		call_stub_destroy_wind (stub);
-	} else {
-		call_stub_destroy_unwind (stub);
-	}
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        if (stub->wind)
+            call_resume_wind(stub);
+        else
+            call_resume_unwind(stub);
+    }
+    THIS = old_THIS;
 
-       stub->stub_mem_pool = NULL;
-       mem_put (tmp_pool, stub);
+    call_stub_destroy(stub);
 out:
-       tmp_pool = NULL;
+    return;
+}
+
+void
+call_unwind_error(call_stub_t *stub, int op_ret, int op_errno)
+{
+    xlator_t *old_THIS = NULL;
+
+    list_del_init(&stub->list);
+
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        call_resume_unwind(stub);
+    }
+    THIS = old_THIS;
+
+    call_stub_destroy(stub);
 
-	return;
+    return;
 }
 
 void
-call_resume (call_stub_t *stub)
+call_unwind_error_keep_stub(call_stub_t *stub, int op_ret, int op_errno)
 {
-        xlator_t *old_THIS = NULL;
+    xlator_t *old_THIS = NULL;
 
-	errno = EINVAL;
-	GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+    list_del_init(&stub->list);
 
-	list_del_init (&stub->list);
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        call_resume_unwind(stub);
+    }
 
-        old_THIS = THIS;
-        THIS = stub->frame->this;
-        {
-                if (stub->wind)
-                        call_resume_wind (stub);
-                else
-                        call_resume_unwind (stub);
-        }
-        THIS = old_THIS;
+    THIS = old_THIS;
 
-	call_stub_destroy (stub);
-out:
-	return;
+    return;
 }
 
+void
+call_resume_keep_stub(call_stub_t *stub)
+{
+    xlator_t *old_THIS = NULL;
+
+    errno = EINVAL;
+    GF_VALIDATE_OR_GOTO("call-stub", stub, out);
+
+    list_del_init(&stub->list);
 
+    old_THIS = THIS;
+    THIS = stub->frame->this;
+    {
+        if (stub->wind)
+            call_resume_wind(stub);
+        else
+            call_resume_unwind(stub);
+    }
+
+    THIS = old_THIS;
+
+out:
+    return;
+}
diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h
deleted file mode 100644
index 0d34b3d2e07..00000000000
--- a/libglusterfs/src/call-stub.h
+++ /dev/null
@@ -1,1116 +0,0 @@
-/*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CALL_STUB_H_
-#define _CALL_STUB_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "stack.h"
-#include "list.h"
-
-typedef struct {
-	struct list_head list;
-	char wind;
-	call_frame_t *frame;
-	glusterfs_fop_t fop;
-       struct mem_pool *stub_mem_pool;    /* pointer to stub mempool in glusterfs ctx */
-
-	union {
-		/* lookup */
-		struct {
-			fop_lookup_t fn;
-			loc_t loc;
-			dict_t *xattr_req;
-		} lookup;
-		struct {
-			fop_lookup_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-			dict_t *dict;
-                        struct iatt postparent;
-		} lookup_cbk;
-
-		/* stat */
-		struct {
-			fop_stat_t fn;
-			loc_t loc;
-		} stat;
-		struct {
-			fop_stat_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-		} stat_cbk;
-
-		/* fstat */
-		struct {
-			fop_fstat_t fn;
-			fd_t *fd;
-		} fstat;
-		struct {
-			fop_fstat_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-		} fstat_cbk;
-
-		/* truncate */
-		struct {
-			fop_truncate_t fn;
-			loc_t loc;
-			off_t off;
-		} truncate;
-		struct {
-			fop_truncate_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} truncate_cbk;
-
-		/* ftruncate */
-		struct {
-			fop_ftruncate_t fn;
-			fd_t *fd;
-			off_t off;
-		} ftruncate;
-		struct {
-			fop_ftruncate_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} ftruncate_cbk;
-
-		/* access */
-		struct {
-			fop_access_t fn;
-			loc_t loc;
-			int32_t mask;
-		} access;
-		struct {
-			fop_access_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} access_cbk;
-
-		/* readlink */
-		struct {
-			fop_readlink_t fn;
-			loc_t loc;
-			size_t size;
-		} readlink;
-		struct {
-			fop_readlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-			const char *buf;
-                        struct iatt sbuf;
-		} readlink_cbk;
-
-		/* mknod */
-		struct {
-			fop_mknod_t fn;
-			loc_t loc;
-			mode_t mode;
-			dev_t rdev;
-                        dict_t *params;
-		} mknod;
-		struct {
-			fop_mknod_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} mknod_cbk;
-
-		/* mkdir */
-		struct {
-			fop_mkdir_t fn;
-			loc_t loc;
-			mode_t mode;
-                        dict_t *params;
-		} mkdir;
-		struct {
-			fop_mkdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} mkdir_cbk;
-
-		/* unlink */
-		struct {
-			fop_unlink_t fn;
-			loc_t loc;
-		} unlink;
-		struct {
-			fop_unlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} unlink_cbk;
-
-		/* rmdir */
-		struct {
-			fop_rmdir_t fn;
-			loc_t loc;
-                        int   flags;
-		} rmdir;
-		struct {
-			fop_rmdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} rmdir_cbk;
-
-		/* symlink */
-		struct {
-			fop_symlink_t fn;
-			const char *linkname;
-			loc_t loc;
-                        dict_t *params;
-		} symlink;
-		struct {
-			fop_symlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} symlink_cbk;
-
-		/* rename */
-		struct {
-			fop_rename_t fn;
-			loc_t old;
-			loc_t new;
-		} rename;
-		struct {
-			fop_rename_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-                        struct iatt preoldparent;
-                        struct iatt postoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postnewparent;
-		} rename_cbk;
-
-		/* link */
-		struct {
-			fop_link_t fn;
-			loc_t oldloc;
-			loc_t newloc;
-		} link;
-		struct {
-			fop_link_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} link_cbk;
-
-		/* create */
-		struct {
-			fop_create_t fn;
-			loc_t loc;
-			int32_t flags;
-			mode_t mode;
-			fd_t *fd;
-                        dict_t *params;
-		} create;
-		struct {
-			fop_create_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} create_cbk;
-
-		/* open */
-		struct {
-			fop_open_t fn;
-			loc_t loc;
-			int32_t flags;
-			fd_t *fd;
-                        int32_t wbflags;
-		} open;
-		struct {
-			fop_open_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-		} open_cbk;
-
-		/* readv */
-		struct {
-			fop_readv_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-		} readv;
-		struct {
-			fop_readv_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			struct iovec *vector;
-			int32_t count;
-			struct iatt stbuf;
-			struct iobref *iobref;
-		} readv_cbk;
-
-		/* writev */
-		struct {
-			fop_writev_t fn;
-			fd_t *fd;
-			struct iovec *vector;
-			int32_t count;
-			off_t off;
-			struct iobref *iobref;
-		} writev;
-		struct {
-			fop_writev_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt prebuf;
-			struct iatt postbuf;
-		} writev_cbk;
-
-		/* flush */
-		struct {
-			fop_flush_t fn;
-			fd_t *fd;
-		} flush;
-		struct {
-			fop_flush_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} flush_cbk;
-
-		/* fsync */
-		struct {
-			fop_fsync_t fn;
-			fd_t *fd;
-			int32_t datasync;
-		} fsync;
-		struct {
-			fop_fsync_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-		} fsync_cbk;
-
-		/* opendir */
-		struct {
-			fop_opendir_t fn;
-			loc_t loc;
-			fd_t *fd;
-		} opendir;
-		struct {
-			fop_opendir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-		} opendir_cbk;
-
-
-		/* fsyncdir */
-		struct {
-			fop_fsyncdir_t fn;
-			fd_t *fd;
-			int32_t datasync;
-		} fsyncdir;
-		struct {
-			fop_fsyncdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fsyncdir_cbk;
-
-		/* statfs */
-		struct {
-			fop_statfs_t fn;
-			loc_t loc;
-		} statfs;
-		struct {
-			fop_statfs_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct statvfs buf;
-		} statfs_cbk;
-
-		/* setxattr */
-		struct {
-			fop_setxattr_t fn;
-			loc_t loc;
-			dict_t *dict;
-			int32_t flags;
-		} setxattr;
-		struct {
-			fop_setxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} setxattr_cbk;
-
-		/* getxattr */
-		struct {
-			fop_getxattr_t fn;
-			loc_t loc;
-			const char *name;
-		} getxattr;
-		struct {
-			fop_getxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-			dict_t *dict;
-		} getxattr_cbk;
-
-		/* fsetxattr */
-		struct {
-			fop_fsetxattr_t fn;
-			fd_t *fd;
-			dict_t *dict;
-			int32_t flags;
-		} fsetxattr;
-		struct {
-			fop_fsetxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fsetxattr_cbk;
-
-		/* fgetxattr */
-		struct {
-			fop_fgetxattr_t fn;
-			fd_t *fd;
-			const char *name;
-		} fgetxattr;
-		struct {
-			fop_fgetxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-			dict_t *dict;
-		} fgetxattr_cbk;
-
-		/* removexattr */
-		struct {
-			fop_removexattr_t fn;
-			loc_t loc;
-			const char *name;
-		} removexattr;
-		struct {
-			fop_removexattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} removexattr_cbk;
-
-		/* lk */
-		struct {
-			fop_lk_t fn;
-			fd_t *fd;
-			int32_t cmd;
-			struct gf_flock lock;
-		} lk;
-		struct {
-			fop_lk_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct gf_flock lock;
-		} lk_cbk;
-
-		/* inodelk */
-		struct {
-			fop_inodelk_t fn;
-                        const char *volume;
-			loc_t loc;
-			int32_t cmd;
-			struct gf_flock lock;
-		} inodelk;
-
-		struct {
-			fop_inodelk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} inodelk_cbk;
-
-		/* finodelk */
-		struct {
-			fop_finodelk_t fn;
-                        const char *volume;
-			fd_t *fd;
-			int32_t cmd;
-			struct gf_flock lock;
-		} finodelk;
-
-		struct {
-			fop_finodelk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} finodelk_cbk;
-
-		/* entrylk */
-		struct {
-			fop_entrylk_t fn;
-			loc_t loc;
-                        const char *volume;
-			const char *name;
-			entrylk_cmd cmd;
-			entrylk_type type;
-		} entrylk;
-
-		struct {
-			fop_entrylk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} entrylk_cbk;
-
-		/* fentrylk */
-		struct {
-			fop_fentrylk_t fn;
-			fd_t *fd;
-                        const char *volume;
-			const char *name;
-			entrylk_cmd cmd;
-			entrylk_type type;
-		} fentrylk;
-
-		struct {
-			fop_fentrylk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fentrylk_cbk;
-
-		/* readdir */
-		struct {
-			fop_readdir_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-		} readdir;
-		struct {
-			fop_readdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			gf_dirent_t entries;
-		} readdir_cbk;
-
-                /* readdirp */
-		struct {
-			fop_readdirp_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-		} readdirp;
-		struct {
-			fop_readdirp_cbk_t fn;
-			int32_t op_ret, op_errno;
-			gf_dirent_t entries;
-		} readdirp_cbk;
-
-		/* rchecksum */
-		struct {
-			fop_rchecksum_t fn;
-			fd_t *fd;
-                        off_t offset;
-			int32_t len;
-		} rchecksum;
-		struct {
-			fop_rchecksum_cbk_t fn;
-			int32_t op_ret, op_errno;
-			uint32_t weak_checksum;
-			uint8_t *strong_checksum;
-		} rchecksum_cbk;
-
-		/* xattrop */
-		struct {
-			fop_xattrop_t fn;
-			loc_t loc;
-			gf_xattrop_flags_t optype;
-			dict_t *xattr;
-		} xattrop;
-		struct {
-			fop_xattrop_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			dict_t *xattr;
-		} xattrop_cbk;
-
-		/* fxattrop */
-		struct {
-			fop_fxattrop_t fn;
-			fd_t *fd;
-			gf_xattrop_flags_t optype;
-			dict_t *xattr;
-		} fxattrop;
-		struct {
-			fop_fxattrop_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			dict_t *xattr;
-		} fxattrop_cbk;
-
-                /* setattr */
-                struct {
-                        fop_setattr_t fn;
-                        loc_t loc;
-                        struct iatt stbuf;
-                        int32_t valid;
-                } setattr;
-                struct {
-                        fop_setattr_cbk_t fn;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt statpre;
-                        struct iatt statpost;
-                } setattr_cbk;
-
-                /* fsetattr */
-                struct {
-                        fop_fsetattr_t fn;
-                        fd_t *fd;
-                        struct iatt stbuf;
-                        int32_t valid;
-                } fsetattr;
-                struct {
-                        fop_fsetattr_cbk_t fn;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt statpre;
-                        struct iatt statpost;
-                } fsetattr_cbk;
-
-	} args;
-} call_stub_t;
-
-call_stub_t *
-fop_lookup_stub (call_frame_t *frame,
-		 fop_lookup_t fn,
-		 loc_t *loc,
-		 dict_t *xattr_req);
-
-call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame,
-		     fop_lookup_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent);
-call_stub_t *
-fop_stat_stub (call_frame_t *frame,
-	       fop_stat_t fn,
-	       loc_t *loc);
-call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame,
-		   fop_stat_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf);
-call_stub_t *
-fop_fstat_stub (call_frame_t *frame,
-		fop_fstat_t fn,
-		fd_t *fd);
-call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame,
-		    fop_fstat_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *buf);
-
-call_stub_t *
-fop_truncate_stub (call_frame_t *frame,
-		   fop_truncate_t fn,
-		   loc_t *loc,
-		   off_t off);
-
-call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame,
-		       fop_truncate_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-                       struct iatt *postbuf);
-
-call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame,
-		    fop_ftruncate_t fn,
-		    fd_t *fd,
-		    off_t off);
-
-call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame,
-			fop_ftruncate_cbk_t fn,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iatt *prebuf,
-                        struct iatt *postbuf);
-
-call_stub_t *
-fop_access_stub (call_frame_t *frame,
-		 fop_access_t fn,
-		 loc_t *loc,
-		 int32_t mask);
-
-call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame,
-		     fop_access_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno);
-
-call_stub_t *
-fop_readlink_stub (call_frame_t *frame,
-		   fop_readlink_t fn,
-		   loc_t *loc,
-		   size_t size);
-
-call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame,
-		       fop_readlink_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       const char *path,
-                       struct iatt *buf);
-
-call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn,
-		loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
-
-call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame,
-		    fop_mknod_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent);
-
-call_stub_t *
-fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
-		loc_t *loc, mode_t mode, dict_t *params);
-
-call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame,
-		    fop_mkdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent);
-
-call_stub_t *
-fop_unlink_stub (call_frame_t *frame,
-		 fop_unlink_t fn,
-		 loc_t *loc);
-
-call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame,
-		     fop_unlink_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *preparent,
-                     struct iatt *postparent);
-
-call_stub_t *
-fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-		loc_t *loc, int flags);
-
-call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame,
-		    fop_rmdir_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent);
-
-call_stub_t *
-fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-		  const char *linkname, loc_t *loc, dict_t *params);
-
-call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame,
-		      fop_symlink_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent);
-
-call_stub_t *
-fop_rename_stub (call_frame_t *frame,
-		 fop_rename_t fn,
-		 loc_t *oldloc,
-		 loc_t *newloc);
-
-call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame,
-		     fop_rename_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent);
-
-call_stub_t *
-fop_link_stub (call_frame_t *frame,
-	       fop_link_t fn,
-	       loc_t *oldloc,
-	       loc_t *newloc);
-
-call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame,
-		   fop_link_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent);
-
-call_stub_t *
-fop_create_stub (call_frame_t *frame, fop_create_t fn,
-		 loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params);
-
-call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame,
-		     fop_create_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent);
-
-call_stub_t *
-fop_open_stub (call_frame_t *frame,
-	       fop_open_t fn,
-	       loc_t *loc,
-	       int32_t flags,
-	       fd_t *fd,
-               int32_t wbflags);
-
-call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame,
-		   fop_open_cbk_t fn,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd);
-
-call_stub_t *
-fop_readv_stub (call_frame_t *frame,
-		fop_readv_t fn,
-		fd_t *fd,
-		size_t size,
-		off_t off);
-
-call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame,
-		    fop_readv_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iovec *vector,
-		    int32_t count,
-		    struct iatt *stbuf,
-                    struct iobref *iobref);
-
-call_stub_t *
-fop_writev_stub (call_frame_t *frame,
-		 fop_writev_t fn,
-		 fd_t *fd,
-		 struct iovec *vector,
-		 int32_t count,
-		 off_t off,
-                 struct iobref *iobref);
-
-call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame,
-		     fop_writev_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-                     struct iatt *prebuf,
-                     struct iatt *postbuf);
-
-call_stub_t *
-fop_flush_stub (call_frame_t *frame,
-		fop_flush_t fn,
-		fd_t *fd);
-
-call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame,
-		    fop_flush_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-call_stub_t *
-fop_fsync_stub (call_frame_t *frame,
-		fop_fsync_t fn,
-		fd_t *fd,
-		int32_t datasync);
-
-call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame,
-		    fop_fsync_cbk_t fn,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *prebuf,
-                    struct iatt *postbuf);
-
-call_stub_t *
-fop_opendir_stub (call_frame_t *frame,
-		  fop_opendir_t fn,
-		  loc_t *loc, fd_t *fd);
-
-call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame,
-		      fop_opendir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd);
-
-call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame,
-		   fop_fsyncdir_t fn,
-		   fd_t *fd,
-		   int32_t datasync);
-
-call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame,
-		       fop_fsyncdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno);
-
-call_stub_t *
-fop_statfs_stub (call_frame_t *frame,
-		 fop_statfs_t fn,
-		 loc_t *loc);
-
-call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame,
-		     fop_statfs_cbk_t fn,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct statvfs *buf);
-
-call_stub_t *
-fop_setxattr_stub (call_frame_t *frame,
-		   fop_setxattr_t fn,
-		   loc_t *loc,
-		   dict_t *dict,
-		   int32_t flags);
-
-call_stub_t *
-fop_setxattr_cbk_stub (call_frame_t *frame,
-		       fop_setxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno);
-
-call_stub_t *
-fop_getxattr_stub (call_frame_t *frame,
-		   fop_getxattr_t fn,
-		   loc_t *loc,
-		   const char *name);
-
-call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame,
-		       fop_getxattr_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *value);
-
-call_stub_t *
-fop_fsetxattr_stub (call_frame_t *frame,
-                    fop_fsetxattr_t fn,
-                    fd_t *fd,
-                    dict_t *dict,
-                    int32_t flags);
-
-call_stub_t *
-fop_fsetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fsetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno);
-
-call_stub_t *
-fop_fgetxattr_stub (call_frame_t *frame,
-                    fop_fgetxattr_t fn,
-                    fd_t *fd,
-                    const char *name);
-
-call_stub_t *
-fop_fgetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fgetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        dict_t *value);
-
-call_stub_t *
-fop_removexattr_stub (call_frame_t *frame,
-		      fop_removexattr_t fn,
-		      loc_t *loc,
-		      const char *name);
-
-call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame,
-			  fop_removexattr_cbk_t fn,
-			  int32_t op_ret,
-			  int32_t op_errno);
-call_stub_t *
-fop_lk_stub (call_frame_t *frame,
-	     fop_lk_t fn,
-	     fd_t *fd,
-	     int32_t cmd,
-	     struct gf_flock *lock);
-
-call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame,
-		 fop_lk_cbk_t fn,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock);
-
-call_stub_t *
-fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
-		  const char *volume, loc_t *loc, int32_t cmd, 
-                  struct gf_flock *lock);
-
-call_stub_t *
-fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
-		   const char *volume, fd_t *fd, int32_t cmd, 
-                   struct gf_flock *lock);
-
-call_stub_t *
-fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
-		  const char *volume, loc_t *loc, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type);
-
-call_stub_t *
-fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
-		   const char *volume, fd_t *fd, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type);
-
-call_stub_t *
-fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno);
-
-call_stub_t *
-fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno);
-
-call_stub_t *
-fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno);
-
-call_stub_t *
-fop_fentrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno);
-
-call_stub_t *
-fop_readdir_stub (call_frame_t *frame,
-		  fop_readdir_t fn,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off);
-
-call_stub_t *
-fop_readdirp_stub (call_frame_t *frame,
-		   fop_readdir_t fn,
-		   fd_t *fd,
-		   size_t size,
-		   off_t off);
-
-call_stub_t *
-fop_readdirp_cbk_stub (call_frame_t *frame,
-		       fop_readdir_cbk_t fn,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       gf_dirent_t *entries);
-
-call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame,
-		      fop_readdir_cbk_t fn,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      gf_dirent_t *entries);
-
-call_stub_t *
-fop_rchecksum_stub (call_frame_t *frame,
-                    fop_rchecksum_t fn,
-                    fd_t *fd, off_t offset,
-                    int32_t len);
-
-call_stub_t *
-fop_rchecksum_cbk_stub (call_frame_t *frame,
-                        fop_rchecksum_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        uint32_t weak_checksum,
-                        uint8_t *strong_checksum);
-
-call_stub_t *
-fop_xattrop_stub (call_frame_t *frame,
-		  fop_xattrop_t fn,
-		  loc_t *loc,
-		  gf_xattrop_flags_t optype,
-		  dict_t *xattr);
-
-call_stub_t *
-fop_xattrop_stub_cbk_stub (call_frame_t *frame,
-			   fop_xattrop_cbk_t fn,
-			   int32_t op_ret,
-			   int32_t op_errno);
-
-call_stub_t *
-fop_fxattrop_stub (call_frame_t *frame,
-		   fop_fxattrop_t fn,
-		   fd_t *fd,
-		   gf_xattrop_flags_t optype,
-		   dict_t *xattr);
-
-call_stub_t *
-fop_fxattrop_stub_cbk_stub (call_frame_t *frame,
-			    fop_xattrop_cbk_t fn,
-			    int32_t op_ret,
-			    int32_t op_errno);
-
-call_stub_t *
-fop_setattr_stub (call_frame_t *frame,
-                  fop_setattr_t fn,
-                  loc_t *loc,
-                  struct iatt *stbuf,
-                  int32_t valid);
-
-call_stub_t *
-fop_setattr_cbk_stub (call_frame_t *frame,
-                      fop_setattr_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      struct iatt *statpre,
-                      struct iatt *statpost);
-
-call_stub_t *
-fop_fsetattr_stub (call_frame_t *frame,
-                   fop_fsetattr_t fn,
-                   fd_t *fd,
-                   struct iatt *stbuf,
-                   int32_t valid);
-
-call_stub_t *
-fop_fsetattr_cbk_stub (call_frame_t *frame,
-                       fop_setattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *statpre,
-                       struct iatt *statpost);
-
-void call_resume (call_stub_t *stub);
-void call_stub_destroy (call_stub_t *stub);
-#endif
diff --git a/libglusterfs/src/changelog.h b/libglusterfs/src/changelog.h
new file mode 100644
index 00000000000..a09d9f25287
--- /dev/null
+++ b/libglusterfs/src/changelog.h
@@ -0,0 +1,115 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+struct gf_brick_spec;
+
+/**
+ * Max bit shiter for event selection
+ */
+#define CHANGELOG_EV_SELECTION_RANGE 5
+
+#define CHANGELOG_OP_TYPE_JOURNAL (1 << 0)
+#define CHANGELOG_OP_TYPE_OPEN (1 << 1)
+#define CHANGELOG_OP_TYPE_CREATE (1 << 2)
+#define CHANGELOG_OP_TYPE_RELEASE (1 << 3)
+#define CHANGELOG_OP_TYPE_BR_RELEASE                                           \
+    (1 << 4) /* logical release (last close()),                                \
+                sent by bitrot stub */
+#define CHANGELOG_OP_TYPE_MAX (1 << CHANGELOG_EV_SELECTION_RANGE)
+
+struct ev_open {
+    unsigned char gfid[16];
+    int32_t flags;
+};
+
+struct ev_creat {
+    unsigned char gfid[16];
+    int32_t flags;
+};
+
+struct ev_release {
+    unsigned char gfid[16];
+};
+
+struct ev_release_br {
+    unsigned long version;
+    unsigned char gfid[16];
+    int32_t sign_info;
+};
+
+struct ev_changelog {
+    char path[PATH_MAX];
+};
+
+typedef struct changelog_event {
+    unsigned int ev_type;
+
+    union {
+        struct ev_open open;
+        struct ev_creat create;
+        struct ev_release release;
+        struct ev_changelog journal;
+        struct ev_release_br releasebr;
+    } u;
+} changelog_event_t;
+
+#define CHANGELOG_EV_SIZE (sizeof(changelog_event_t))
+
+/**
+ * event callback, connected & disconnection defs
+ */
+typedef void(CALLBACK)(void *, char *, void *, changelog_event_t *);
+typedef void *(INIT)(void *, struct gf_brick_spec *);
+typedef void(FINI)(void *, char *, void *);
+typedef void(CONNECT)(void *, char *, void *);
+typedef void(DISCONNECT)(void *, char *, void *);
+
+struct gf_brick_spec {
+    char *brick_path;
+    unsigned int filter;
+
+    INIT *init;
+    FINI *fini;
+    CALLBACK *callback;
+    CONNECT *connected;
+    DISCONNECT *disconnected;
+
+    void *ptr;
+};
+
+/* API set */
+
+int
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+                      int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan();
+
+int
+gf_changelog_start_fresh();
+
+ssize_t
+gf_changelog_next_change(char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done(char *file);
+
+/* newer flexible API */
+int
+gf_changelog_init(void *xl);
+
+int
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+                              int ordered, char *logfile, int lvl, void *xl);
+
+#endif
diff --git a/libglusterfs/src/checksum.c b/libglusterfs/src/checksum.c
index de86adc9a37..acdaed04ae2 100644
--- a/libglusterfs/src/checksum.c
+++ b/libglusterfs/src/checksum.c
@@ -1,78 +1,44 @@
 /*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <inttypes.h>
-
-#include "glusterfs.h"
-#include "md5.h"
-#include "checksum.h"
-
+#include <openssl/md5.h>
+#include <openssl/sha.h>
+#include <zlib.h>
+#include <stdint.h>
+#include <string.h>
 
 /*
- * The "weak" checksum required for the rsync algorithm,
- * adapted from the rsync source code. The following comment
- * appears there:
+ * The "weak" checksum required for the rsync algorithm.
  *
- * "a simple 32 bit checksum that can be upadted from either end
- *  (inspired by Mark Adler's Adler-32 checksum)"
+ * Note: these functions are only called to compute checksums on
+ * pathnames; they don't need to handle arbitrarily long strings of
+ * data. Thus int32_t and uint32_t are sufficient
  */
-
 uint32_t
-gf_rsync_weak_checksum (char *buf1, int32_t len)
+gf_rsync_weak_checksum(unsigned char *buf, size_t len)
 {
-        int32_t i;
-        uint32_t s1, s2;
-
-        signed char *buf = (signed char *) buf1;
-        uint32_t csum;
-
-        s1 = s2 = 0;
-        for (i = 0; i < (len-4); i+=4) {
-                s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3];
-
-                s1 += buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3];
-        }
-
-        for (; i < len; i++) {
-                s1 += buf[i];
-                s2 += s1;
-        }
-
-        csum = (s1 & 0xffff) + (s2 << 16);
-
-        return csum;
+    return adler32(0, buf, len);
 }
 
-
 /*
- * The "strong" checksum required for the rsync algorithm,
- * adapted from the rsync source code.
+ * The "strong" checksum required for the rsync algorithm.
  */
-
 void
-gf_rsync_strong_checksum (char *buf, int32_t len, uint8_t *sum)
+gf_rsync_strong_checksum(unsigned char *data, size_t len,
+                         unsigned char *sha256_md)
 {
-        md_context m;
-
-        md5_begin (&m);
-        md5_update (&m, (unsigned char *) buf, len);
-        md5_result (&m, (unsigned char *) sum);
+    SHA256((const unsigned char *)data, len, sha256_md);
+}
 
-        return;
+void
+gf_rsync_md5_checksum(unsigned char *data, size_t len, unsigned char *md5)
+{
+    MD5(data, len, md5);
 }
diff --git a/libglusterfs/src/checksum.h b/libglusterfs/src/checksum.h
deleted file mode 100644
index 592e99fcb69..00000000000
--- a/libglusterfs/src/checksum.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __CHECKSUM_H__
-#define __CHECKSUM_H__
-
-uint32_t
-gf_rsync_weak_checksum (char *buf, int32_t len);
-
-void
-gf_rsync_strong_checksum (char *buf, int32_t len, uint8_t *sum);
-
-#endif /* __CHECKSUM_H__ */
diff --git a/libglusterfs/src/circ-buff.c b/libglusterfs/src/circ-buff.c
new file mode 100644
index 00000000000..913115c7be1
--- /dev/null
+++ b/libglusterfs/src/circ-buff.c
@@ -0,0 +1,193 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/circ-buff.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+void
+cb_destroy_data(circular_buffer_t *cb, void (*destroy_buffer_data)(void *data))
+{
+    if (destroy_buffer_data)
+        destroy_buffer_data(cb->data);
+    GF_FREE(cb->data);
+    return;
+}
+
+/* hold lock while calling this function */
+int
+__cb_add_entry_buffer(buffer_t *buffer, void *item)
+{
+    circular_buffer_t *ptr = NULL;
+    int ret = -1;
+    // DO we really need the assert here?
+    GF_ASSERT(buffer->used_len <= buffer->size_buffer);
+
+    if (buffer->use_once == _gf_true &&
+        buffer->used_len == buffer->size_buffer) {
+        gf_msg("circ-buff", GF_LOG_WARNING, 0, LG_MSG_BUFFER_ERROR,
+               "buffer %p is use once buffer", buffer);
+        return -1;
+    } else {
+        if (buffer->used_len == buffer->size_buffer) {
+            if (buffer->cb[buffer->w_index]) {
+                ptr = buffer->cb[buffer->w_index];
+                if (ptr->data) {
+                    cb_destroy_data(ptr, buffer->destroy_buffer_data);
+                    ptr->data = NULL;
+                    GF_FREE(ptr);
+                }
+                buffer->cb[buffer->w_index] = NULL;
+                ptr = NULL;
+            }
+        }
+
+        buffer->cb[buffer->w_index] = GF_CALLOC(1, sizeof(circular_buffer_t),
+                                                gf_common_mt_circular_buffer_t);
+        if (!buffer->cb[buffer->w_index])
+            return -1;
+
+        buffer->cb[buffer->w_index]->data = item;
+        ret = gettimeofday(&buffer->cb[buffer->w_index]->tv, NULL);
+        if (ret == -1)
+            gf_msg_callingfn("circ-buff", GF_LOG_WARNING, 0,
+                             LG_MSG_GETTIMEOFDAY_FAILED,
+                             "getting time of the day failed");
+        buffer->w_index++;
+        buffer->w_index %= buffer->size_buffer;
+        // used_buffer size cannot be greater than the total buffer size
+
+        if (buffer->used_len < buffer->size_buffer)
+            buffer->used_len++;
+        return buffer->w_index;
+    }
+}
+
+int
+cb_add_entry_buffer(buffer_t *buffer, void *item)
+{
+    int write_index = -1;
+
+    pthread_mutex_lock(&buffer->lock);
+    {
+        write_index = __cb_add_entry_buffer(buffer, item);
+    }
+    pthread_mutex_unlock(&buffer->lock);
+
+    return write_index;
+}
+
+void
+cb_buffer_show(buffer_t *buffer)
+{
+    pthread_mutex_lock(&buffer->lock);
+    {
+        gf_msg_debug("circ-buff", 0,
+                     "w_index: %d, size: %" GF_PRI_SIZET " used_buffer: %d",
+                     buffer->w_index, buffer->size_buffer, buffer->used_len);
+    }
+    pthread_mutex_unlock(&buffer->lock);
+}
+
+void
+cb_buffer_dump(buffer_t *buffer, void *data,
+               int(fn)(circular_buffer_t *buffer, void *data))
+{
+    int index = 0;
+    circular_buffer_t *entry = NULL;
+    int entries = 0;
+    int ul = 0;
+    int w_ind = 0;
+    int size_buff = 0;
+    int i = 0;
+
+    ul = buffer->used_len;
+    w_ind = buffer->w_index;
+    size_buff = buffer->size_buffer;
+
+    pthread_mutex_lock(&buffer->lock);
+    {
+        if (buffer->use_once == _gf_false) {
+            index = (size_buff + (w_ind - ul)) % size_buff;
+            for (entries = 0; entries < buffer->used_len; entries++) {
+                entry = buffer->cb[index];
+                if (entry)
+                    fn(entry, data);
+                else
+                    gf_msg_callingfn("circ-buff", GF_LOG_WARNING, 0,
+                                     LG_MSG_NULL_PTR,
+                                     "Null entry in "
+                                     "circular buffer at "
+                                     "index %d.",
+                                     index);
+
+                index++;
+                index %= buffer->size_buffer;
+            }
+        } else {
+            for (i = 0; i < buffer->used_len; i++) {
+                entry = buffer->cb[i];
+                fn(entry, data);
+            }
+        }
+    }
+    pthread_mutex_unlock(&buffer->lock);
+}
+
+buffer_t *
+cb_buffer_new(size_t buffer_size, gf_boolean_t use_once,
+              void (*destroy_buffer_data)(void *data))
+{
+    buffer_t *buffer = NULL;
+
+    buffer = GF_CALLOC(1, sizeof(*buffer), gf_common_mt_buffer_t);
+    if (!buffer) {
+        goto out;
+    }
+
+    buffer->cb = GF_CALLOC(buffer_size, sizeof(circular_buffer_t *),
+                           gf_common_mt_circular_buffer_t);
+    if (!buffer->cb) {
+        GF_FREE(buffer);
+        buffer = NULL;
+        goto out;
+    }
+
+    buffer->w_index = 0;
+    buffer->size_buffer = buffer_size;
+    buffer->use_once = use_once;
+    buffer->used_len = 0;
+    buffer->destroy_buffer_data = destroy_buffer_data;
+    pthread_mutex_init(&buffer->lock, NULL);
+
+out:
+    return buffer;
+}
+
+void
+cb_buffer_destroy(buffer_t *buffer)
+{
+    int i = 0;
+    circular_buffer_t *ptr = NULL;
+    if (buffer) {
+        if (buffer->cb) {
+            for (i = 0; i < buffer->used_len; i++) {
+                ptr = buffer->cb[i];
+                if (ptr->data) {
+                    cb_destroy_data(ptr, buffer->destroy_buffer_data);
+                    ptr->data = NULL;
+                    GF_FREE(ptr);
+                }
+            }
+            GF_FREE(buffer->cb);
+        }
+        pthread_mutex_destroy(&buffer->lock);
+        GF_FREE(buffer);
+    }
+}
diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c
new file mode 100644
index 00000000000..9d377c3c2e1
--- /dev/null
+++ b/libglusterfs/src/client_t.c
@@ -0,0 +1,825 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/dict.h"
+#include "glusterfs/statedump.h"
+#include "glusterfs/client_t.h"
+#include "glusterfs/list.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+static int
+gf_client_chain_client_entries(cliententry_t *entries, uint32_t startidx,
+                               uint32_t endcount)
+{
+    uint32_t i = 0;
+
+    if (!entries) {
+        gf_msg_callingfn("client_t", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!entries");
+        return -1;
+    }
+
+    /* Chain only till the second to last entry because we want to
+     * ensure that the last entry has GF_CLIENTTABLE_END.
+     */
+    for (i = startidx; i < (endcount - 1); i++)
+        entries[i].next_free = i + 1;
+
+    /* i has already been incremented up to the last entry. */
+    entries[i].next_free = GF_CLIENTTABLE_END;
+
+    return 0;
+}
+
+static int
+gf_client_clienttable_expand(clienttable_t *clienttable, uint32_t nr)
+{
+    cliententry_t *oldclients = NULL;
+    uint32_t oldmax_clients = -1;
+    int ret = -1;
+
+    if (clienttable == NULL || nr <= clienttable->max_clients) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        ret = EINVAL;
+        goto out;
+    }
+
+    oldclients = clienttable->cliententries;
+    oldmax_clients = clienttable->max_clients;
+
+    clienttable->cliententries = GF_CALLOC(nr, sizeof(cliententry_t),
+                                           gf_common_mt_cliententry_t);
+    if (!clienttable->cliententries) {
+        clienttable->cliententries = oldclients;
+        ret = 0;
+        goto out;
+    }
+    clienttable->max_clients = nr;
+
+    if (oldclients) {
+        uint32_t cpy = oldmax_clients * sizeof(cliententry_t);
+        memcpy(clienttable->cliententries, oldclients, cpy);
+    }
+
+    gf_client_chain_client_entries(clienttable->cliententries, oldmax_clients,
+                                   clienttable->max_clients);
+
+    /* Now that expansion is done, we must update the client list
+     * head pointer so that the client allocation functions can continue
+     * using the expanded table.
+     */
+    clienttable->first_free = oldmax_clients;
+    GF_FREE(oldclients);
+    ret = 0;
+out:
+    return ret;
+}
+
+clienttable_t *
+gf_clienttable_alloc(void)
+{
+    clienttable_t *clienttable = NULL;
+    int result = 0;
+
+    clienttable = GF_CALLOC(1, sizeof(clienttable_t),
+                            gf_common_mt_clienttable_t);
+    if (!clienttable)
+        return NULL;
+
+    LOCK_INIT(&clienttable->lock);
+
+    result = gf_client_clienttable_expand(clienttable,
+                                          GF_CLIENTTABLE_INITIAL_SIZE);
+    if (result != 0) {
+        gf_msg("client_t", GF_LOG_ERROR, 0, LG_MSG_EXPAND_CLIENT_TABLE_FAILED,
+               "gf_client_clienttable_expand failed");
+        GF_FREE(clienttable);
+        return NULL;
+    }
+
+    return clienttable;
+}
+
+/*
+ * Increments ref.bind if the client is already present or creates a new
+ * client with ref.bind = 1,ref.count = 1 it signifies that
+ * as long as ref.bind is > 0 client should be alive.
+ */
+client_t *
+gf_client_get(xlator_t *this, client_auth_data_t *cred, char *client_uid,
+              char *subdir_mount)
+{
+    client_t *client = NULL;
+    cliententry_t *cliententry = NULL;
+    clienttable_t *clienttable = NULL;
+    unsigned int i = 0;
+
+    if (this == NULL || client_uid == NULL) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
+
+    clienttable = this->ctx->clienttable;
+
+    LOCK(&clienttable->lock);
+    {
+        for (; i < clienttable->max_clients; i++) {
+            client = clienttable->cliententries[i].client;
+            if (client == NULL)
+                continue;
+            /*
+             * look for matching client_uid, _and_
+             * if auth was used, matching auth flavour and data
+             */
+            if (strcmp(client_uid, client->client_uid) == 0 &&
+                (cred->flavour && (cred->flavour == client->auth.flavour &&
+                                   (size_t)cred->datalen == client->auth.len &&
+                                   memcmp(cred->authdata, client->auth.data,
+                                          client->auth.len) == 0))) {
+                GF_ATOMIC_INC(client->bind);
+                goto unlock;
+            }
+        }
+
+        client = GF_CALLOC(1, sizeof(client_t), gf_common_mt_client_t);
+        if (client == NULL) {
+            errno = ENOMEM;
+            goto unlock;
+        }
+
+        client->this = this;
+        if (subdir_mount != NULL)
+            client->subdir_mount = gf_strdup(subdir_mount);
+
+        LOCK_INIT(&client->scratch_ctx.lock);
+
+        client->client_uid = gf_strdup(client_uid);
+        if (client->client_uid == NULL) {
+            GF_FREE(client);
+            client = NULL;
+            errno = ENOMEM;
+            goto unlock;
+        }
+        client->scratch_ctx.count = GF_CLIENTCTX_INITIAL_SIZE;
+        client->scratch_ctx.ctx = GF_CALLOC(GF_CLIENTCTX_INITIAL_SIZE,
+                                            sizeof(struct client_ctx),
+                                            gf_common_mt_client_ctx);
+        if (client->scratch_ctx.ctx == NULL) {
+            GF_FREE(client->client_uid);
+            GF_FREE(client);
+            client = NULL;
+            errno = ENOMEM;
+            goto unlock;
+        }
+
+        GF_ATOMIC_INIT(client->bind, 1);
+        GF_ATOMIC_INIT(client->count, 1);
+        GF_ATOMIC_INIT(client->fd_cnt, 0);
+
+        client->auth.flavour = cred->flavour;
+        if (cred->flavour) {
+            client->auth.data = GF_MALLOC(cred->datalen, gf_common_mt_client_t);
+            if (client->auth.data == NULL) {
+                GF_FREE(client->scratch_ctx.ctx);
+                GF_FREE(client->client_uid);
+                GF_FREE(client);
+                client = NULL;
+                errno = ENOMEM;
+                goto unlock;
+            }
+            memcpy(client->auth.data, cred->authdata, cred->datalen);
+            client->auth.len = cred->datalen;
+        }
+
+        client->tbl_index = clienttable->first_free;
+        cliententry = &clienttable->cliententries[clienttable->first_free];
+        if (cliententry->next_free == GF_CLIENTTABLE_END) {
+            int result = gf_client_clienttable_expand(
+                clienttable,
+                clienttable->max_clients + GF_CLIENTTABLE_INITIAL_SIZE);
+            if (result != 0) {
+                GF_FREE(client->scratch_ctx.ctx);
+                GF_FREE(client->client_uid);
+                GF_FREE(client);
+                client = NULL;
+                errno = result;
+                goto unlock;
+            }
+            cliententry = &clienttable->cliententries[client->tbl_index];
+            cliententry->next_free = clienttable->first_free;
+        }
+        cliententry->client = client;
+        clienttable->first_free = cliententry->next_free;
+        cliententry->next_free = GF_CLIENTENTRY_ALLOCATED;
+    }
+unlock:
+    UNLOCK(&clienttable->lock);
+
+    if (client)
+        gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
+                         "%s: bind_ref: %" GF_PRI_ATOMIC
+                         ", ref: "
+                         "%" GF_PRI_ATOMIC,
+                         client->client_uid, GF_ATOMIC_GET(client->bind),
+                         GF_ATOMIC_GET(client->count));
+    return client;
+}
+
+void
+gf_client_put(client_t *client, gf_boolean_t *detached)
+{
+    gf_boolean_t unref = _gf_false;
+    int bind_ref;
+
+    if (client == NULL)
+        goto out;
+
+    if (detached)
+        *detached = _gf_false;
+
+    bind_ref = GF_ATOMIC_DEC(client->bind);
+    if (bind_ref == 0)
+        unref = _gf_true;
+
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_BIND_REF,
+                     "%s: "
+                     "bind_ref: %" GF_PRI_ATOMIC ", ref: %" GF_PRI_ATOMIC
+                     ", "
+                     "unref: %d",
+                     client->client_uid, GF_ATOMIC_GET(client->bind),
+                     GF_ATOMIC_GET(client->count), unref);
+    if (unref) {
+        if (detached)
+            *detached = _gf_true;
+        gf_client_unref(client);
+    }
+
+out:
+    return;
+}
+
+client_t *
+gf_client_ref(client_t *client)
+{
+    if (!client) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "null client");
+        return NULL;
+    }
+
+    GF_ATOMIC_INC(client->count);
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT,
+                     "%s: "
+                     "ref-count %" GF_PRI_ATOMIC,
+                     client->client_uid, GF_ATOMIC_GET(client->count));
+    return client;
+}
+
+static void
+gf_client_destroy_recursive(xlator_t *xl, client_t *client)
+{
+    xlator_list_t *trav;
+
+    if (!xl->call_cleanup && xl->cbks->client_destroy) {
+        xl->cbks->client_destroy(xl, client);
+    }
+
+    for (trav = xl->children; trav; trav = trav->next) {
+        gf_client_destroy_recursive(trav->xlator, client);
+    }
+}
+
+static void
+client_destroy(client_t *client)
+{
+    clienttable_t *clienttable = NULL;
+    glusterfs_graph_t *gtrav = NULL;
+
+    if (client == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        goto out;
+    }
+
+    clienttable = client->this->ctx->clienttable;
+
+    LOCK(&clienttable->lock);
+    {
+        clienttable->cliententries[client->tbl_index].client = NULL;
+        clienttable->cliententries[client->tbl_index]
+            .next_free = clienttable->first_free;
+        clienttable->first_free = client->tbl_index;
+    }
+    UNLOCK(&clienttable->lock);
+
+    list_for_each_entry(gtrav, &client->this->ctx->graphs, list)
+    {
+        gf_client_destroy_recursive(gtrav->top, client);
+    }
+
+    if (client->subdir_inode)
+        inode_unref(client->subdir_inode);
+
+    LOCK_DESTROY(&client->scratch_ctx.lock);
+
+    GF_FREE(client->auth.data);
+    GF_FREE(client->auth.username);
+    GF_FREE(client->auth.passwd);
+    GF_FREE(client->scratch_ctx.ctx);
+    GF_FREE(client->client_uid);
+    GF_FREE(client->subdir_mount);
+    GF_FREE(client->client_name);
+    GF_FREE(client);
+out:
+    return;
+}
+
+static int
+gf_client_disconnect_recursive(xlator_t *xl, client_t *client)
+{
+    int ret = 0;
+    xlator_list_t *trav;
+
+    if (!xl->call_cleanup && xl->cbks->client_disconnect) {
+        ret = xl->cbks->client_disconnect(xl, client);
+    }
+
+    for (trav = xl->children; trav; trav = trav->next) {
+        ret |= gf_client_disconnect_recursive(trav->xlator, client);
+    }
+
+    return ret;
+}
+
+int
+gf_client_disconnect(client_t *client)
+{
+    int ret = 0;
+    glusterfs_graph_t *gtrav = NULL;
+
+    list_for_each_entry(gtrav, &client->this->ctx->graphs, list)
+    {
+        ret |= gf_client_disconnect_recursive(gtrav->top, client);
+    }
+
+    return ret;
+}
+
+void
+gf_client_unref(client_t *client)
+{
+    uint64_t refcount;
+
+    if (!client) {
+        gf_msg_callingfn("client_t", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "client is NULL");
+        return;
+    }
+
+    refcount = GF_ATOMIC_DEC(client->count);
+    gf_msg_callingfn("client_t", GF_LOG_DEBUG, 0, LG_MSG_REF_COUNT,
+                     "%s: "
+                     "ref-count %" GF_PRI_ATOMIC,
+                     client->client_uid, refcount);
+    if (refcount == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, LG_MSG_DISCONNECT_CLIENT,
+               "Shutting down connection %s", client->client_uid);
+        client_destroy(client);
+    }
+}
+
+static int
+__client_ctx_get_int(client_t *client, void *key, void **value)
+{
+    int index = 0;
+    int ret = 0;
+
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (client->scratch_ctx.ctx[index].ctx_key == key)
+            break;
+    }
+
+    if (index == client->scratch_ctx.count) {
+        ret = -1;
+        goto out;
+    }
+
+    if (value)
+        *value = client->scratch_ctx.ctx[index].ctx_value;
+
+out:
+    return ret;
+}
+
+static int
+__client_ctx_set_int(client_t *client, void *key, void *value)
+{
+    int index = 0;
+    int ret = 0;
+    int set_idx = -1;
+
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (!client->scratch_ctx.ctx[index].ctx_key) {
+            if (set_idx == -1)
+                set_idx = index;
+            /* don't break, to check if key already exists
+               further on */
+        }
+        if (client->scratch_ctx.ctx[index].ctx_key == key) {
+            set_idx = index;
+            break;
+        }
+    }
+
+    if (set_idx == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    client->scratch_ctx.ctx[set_idx].ctx_key = key;
+    client->scratch_ctx.ctx[set_idx].ctx_value = value;
+
+out:
+    return ret;
+}
+
+/*will return success with old value if exist*/
+void *
+client_ctx_set(client_t *client, void *key, void *value)
+{
+    int ret = 0;
+    void *ret_value = NULL;
+
+    if (!client || !key || !value)
+        return NULL;
+
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_get_int(client, key, &ret_value);
+        if (!ret && ret_value) {
+            UNLOCK(&client->scratch_ctx.lock);
+            return ret_value;
+        }
+
+        ret = __client_ctx_set_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
+
+    if (ret)
+        return NULL;
+    return value;
+}
+
+int
+client_ctx_get(client_t *client, void *key, void **value)
+{
+    int ret = 0;
+
+    if (!client || !key)
+        return -1;
+
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_get_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
+
+    return ret;
+}
+
+static int
+__client_ctx_del_int(client_t *client, void *key, void **value)
+{
+    int index = 0;
+    int ret = 0;
+
+    for (index = 0; index < client->scratch_ctx.count; index++) {
+        if (client->scratch_ctx.ctx[index].ctx_key == key)
+            break;
+    }
+
+    if (index == client->scratch_ctx.count) {
+        ret = -1;
+        goto out;
+    }
+
+    if (value)
+        *value = client->scratch_ctx.ctx[index].ctx_value;
+
+    client->scratch_ctx.ctx[index].ctx_key = 0;
+    client->scratch_ctx.ctx[index].ctx_value = 0;
+
+out:
+    return ret;
+}
+
+int
+client_ctx_del(client_t *client, void *key, void **value)
+{
+    int ret = 0;
+
+    if (!client || !key)
+        return -1;
+
+    LOCK(&client->scratch_ctx.lock);
+    {
+        ret = __client_ctx_del_int(client, key, value);
+    }
+    UNLOCK(&client->scratch_ctx.lock);
+
+    return ret;
+}
+
+void
+client_ctx_dump(client_t *client, char *prefix)
+{
+#if 0 /* TBD, FIXME */
+        struct client_ctx   *client_ctx = NULL;
+        xlator_t            *xl = NULL;
+        int                  i = 0;
+
+        if ((client == NULL) || (client->ctx == NULL)) {
+                goto out;
+        }
+
+        LOCK (&client->ctx_lock);
+        if (client->ctx != NULL) {
+                client_ctx = GF_CALLOC (client->inode->table->xl->graph->ctx_count,
+                                        sizeof (*client_ctx),
+                                        gf_common_mt_client_ctx);
+                if (client_ctx == NULL) {
+                        goto unlock;
+                }
+
+                for (i = 0; i < client->inode->table->xl->graph->ctx_count; i++) {
+                        client_ctx[i] = client->ctx[i];
+                }
+        }
+unlock:
+        UNLOCK (&client->ctx_lock);
+
+        if (client_ctx == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < client->inode->table->xl->graph->ctx_count; i++) {
+                if (client_ctx[i].xl_key) {
+                        xl = (xlator_t *)(long)client_ctx[i].xl_key;
+                        if (xl->dumpops && xl->dumpops->clientctx)
+                                xl->dumpops->clientctx (xl, client);
+                }
+        }
+out:
+        GF_FREE (client_ctx);
+#endif
+}
+
+/*
+ * the following functions are here to preserve legacy behavior of the
+ * protocol/server xlator dump, but perhaps they should just be folded
+ * into the client dump instead?
+ */
+int
+gf_client_dump_fdtables_to_dict(xlator_t *this, dict_t *dict)
+{
+    clienttable_t *clienttable = NULL;
+    int count = 0;
+    int ret = -1;
+#ifdef NOTYET
+    client_t *client = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+#endif
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        return -1;
+
+#ifdef NOTYET
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->bound_xl &&
+                !strcmp(client->bound_xl->name, this->name)) {
+                snprintf(key, sizeof(key), "conn%d", count++);
+                fdtable_dump_to_dict(client->server_ctx.fdtable, key, dict);
+            }
+        }
+    }
+    UNLOCK(&clienttable->lock);
+#endif
+
+    ret = dict_set_int32(dict, "conncount", count);
+out:
+    return ret;
+}
+
+int
+gf_client_dump_fdtables(xlator_t *this)
+{
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    int count = 1;
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        return -1;
+
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->client_uid) {
+                gf_proc_dump_build_key(key, "conn", "%d.id", count);
+                gf_proc_dump_write(key, "%s", client->client_uid);
+            }
+
+            if (client->subdir_mount) {
+                gf_proc_dump_build_key(key, "conn", "%d.subdir", count);
+                gf_proc_dump_write(key, "%s", client->subdir_mount);
+            }
+            gf_proc_dump_build_key(key, "conn", "%d.ref", count);
+            gf_proc_dump_write(key, "%" GF_PRI_ATOMIC,
+                               GF_ATOMIC_GET(client->count));
+            if (client->bound_xl) {
+                gf_proc_dump_build_key(key, "conn", "%d.bound_xl", count);
+                gf_proc_dump_write(key, "%s", client->bound_xl->name);
+            }
+
+#ifdef NOTYET
+            gf_proc_dump_build_key(key, "conn", "%d.id", count);
+            fdtable_dump(client->server_ctx.fdtable, key);
+#endif
+        }
+    }
+
+    UNLOCK(&clienttable->lock);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+gf_client_dump_inodes_to_dict(xlator_t *this, dict_t *dict)
+{
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    xlator_t *prev_bound_xl = NULL;
+    char key[32] = {
+        0,
+    };
+    int count = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        return -1;
+
+    ret = LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            return -1;
+        }
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (!strcmp(client->bound_xl->name, this->name)) {
+                if (client->bound_xl && client->bound_xl->itable) {
+                    /* Presently every brick contains only
+                     * one bound_xl for all connections.
+                     * This will lead to duplicating of
+                     *  the inode lists, if listing is
+                     * done for every connection. This
+                     * simple check prevents duplication
+                     * in the present case. If need arises
+                     * the check can be improved.
+                     */
+                    if (client->bound_xl == prev_bound_xl)
+                        continue;
+                    prev_bound_xl = client->bound_xl;
+
+                    snprintf(key, sizeof(key), "conn%d", count);
+                    inode_table_dump_to_dict(client->bound_xl->itable, key,
+                                             dict);
+                }
+            }
+        }
+    }
+    UNLOCK(&clienttable->lock);
+
+    ret = dict_set_int32(dict, "conncount", count);
+
+out:
+    if (prev_bound_xl)
+        prev_bound_xl = NULL;
+    return ret;
+}
+
+int
+gf_client_dump_inodes(xlator_t *this)
+{
+    client_t *client = NULL;
+    clienttable_t *clienttable = NULL;
+    xlator_t *prev_bound_xl = NULL;
+    int count = 0;
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    clienttable = this->ctx->clienttable;
+
+    if (!clienttable)
+        goto out;
+
+    ret = TRY_LOCK(&clienttable->lock);
+    {
+        if (ret) {
+            gf_msg("client_t", GF_LOG_WARNING, 0, LG_MSG_LOCK_FAILED,
+                   "Unable to acquire lock");
+            goto out;
+        }
+
+        for (; count < clienttable->max_clients; count++) {
+            if (GF_CLIENTENTRY_ALLOCATED !=
+                clienttable->cliententries[count].next_free)
+                continue;
+            client = clienttable->cliententries[count].client;
+            if (client->bound_xl && client->bound_xl->itable) {
+                /* Presently every brick contains only
+                 * one bound_xl for all connections.
+                 * This will lead to duplicating of
+                 * the inode lists, if listing is
+                 * done for every connection. This
+                 * simple check prevents duplication
+                 * in the present case. If need arises
+                 * the check can be improved.
+                 */
+                if (client->bound_xl == prev_bound_xl)
+                    continue;
+                prev_bound_xl = client->bound_xl;
+
+                gf_proc_dump_build_key(key, "conn", "%d.bound_xl.%s", count,
+                                       client->bound_xl->name);
+                inode_table_dump(client->bound_xl->itable, key);
+            }
+        }
+    }
+    UNLOCK(&clienttable->lock);
+
+    ret = 0;
+out:
+    return ret;
+}
diff --git a/libglusterfs/src/cluster-syncop.c b/libglusterfs/src/cluster-syncop.c
new file mode 100644
index 00000000000..6ee89ddfdcf
--- /dev/null
+++ b/libglusterfs/src/cluster-syncop.c
@@ -0,0 +1,1261 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* Perform fop on all subvolumes represented by list[] array and wait
+   for all callbacks to return */
+
+/* NOTE: Cluster-syncop, like syncop blocks the executing thread until the
+ * responses are gathered if it is not executed as part of synctask. So it
+ * shouldn't be invoked in epoll worker thread */
+#include "glusterfs/cluster-syncop.h"
+#include "glusterfs/defaults.h"
+
+#define FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fop,       \
+                   args...)                                                    \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        cluster_local_t __local = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        void *__old_local = frame->local;                                      \
+                                                                               \
+        __local.replies = replies;                                             \
+        memset(output, 0, numsubvols);                                         \
+        cluster_replies_wipe(replies, numsubvols);                             \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+        if (syncbarrier_init(&__local.barrier))                                \
+            break;                                                             \
+        frame->local = &__local;                                               \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (on[__i]) {                                                     \
+                __count++;                                                     \
+            }                                                                  \
+        }                                                                      \
+        __local.barrier.waitfor = __count;                                     \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (!on[__i])                                                      \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, cluster_##fop##_cbk, (void *)(long)__i,   \
+                              subvols[__i], subvols[__i]->fops->fop, args);    \
+        }                                                                      \
+        syncbarrier_wait(&__local.barrier, __count);                           \
+        syncbarrier_destroy(&__local.barrier);                                 \
+        frame->local = __old_local;                                            \
+        STACK_RESET(frame->root);                                              \
+    } while (0)
+
+#define FOP_SEQ(subvols, on, numsubvols, replies, output, frame, fop, args...) \
+    do {                                                                       \
+        int __i = 0;                                                           \
+                                                                               \
+        cluster_local_t __local = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        void *__old_local = frame->local;                                      \
+        __local.replies = replies;                                             \
+        memset(output, 0, numsubvols);                                         \
+        cluster_replies_wipe(replies, numsubvols);                             \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+        if (syncbarrier_init(&__local.barrier))                                \
+            break;                                                             \
+        frame->local = &__local;                                               \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (!on[__i])                                                      \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, cluster_##fop##_cbk, (void *)(long)__i,   \
+                              subvols[__i], subvols[__i]->fops->fop, args);    \
+            syncbarrier_wait(&__local.barrier, 1);                             \
+        }                                                                      \
+        syncbarrier_destroy(&__local.barrier);                                 \
+        frame->local = __old_local;                                            \
+        STACK_RESET(frame->root);                                              \
+    } while (0)
+
+#define FOP_CBK(fop, frame, cookie, args...)                                   \
+    do {                                                                       \
+        cluster_local_t *__local = frame->local;                               \
+        int __i = (long)cookie;                                                \
+        args_##fop##_cbk_store(&__local->replies[__i], args);                  \
+        __local->replies[__i].valid = 1;                                       \
+        syncbarrier_wake(&__local->barrier);                                   \
+    } while (0)
+
+int32_t
+cluster_fop_success_fill(default_args_cbk_t *replies, int numsubvols,
+                         unsigned char *success)
+{
+    int i = 0;
+    int count = 0;
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret >= 0) {
+            success[i] = 1;
+            count++;
+        } else {
+            success[i] = 0;
+        }
+    }
+
+    return count;
+}
+
+void
+cluster_replies_wipe(default_args_cbk_t *replies, int numsubvols)
+{
+    int i = 0;
+
+    if (!replies)
+        return;
+
+    for (i = 0; i < numsubvols; i++)
+        args_cbk_wipe(&replies[i]);
+    memset(replies, 0, numsubvols * sizeof(*replies));
+}
+
+int32_t
+cluster_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    FOP_CBK(lookup, frame, cookie, op_ret, op_errno, inode, buf, xdata,
+            postparent);
+    return 0;
+}
+
+int32_t
+cluster_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
+{
+    FOP_CBK(stat, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    FOP_CBK(truncate, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    FOP_CBK(ftruncate, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(access, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
+{
+    FOP_CBK(readlink, frame, cookie, op_ret, op_errno, path, buf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(mknod, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(mkdir, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(unlink, frame, cookie, op_ret, op_errno, preparent, postparent,
+            xdata);
+    return 0;
+}
+
+int32_t
+cluster_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(rmdir, frame, cookie, op_ret, op_errno, preparent, postparent,
+            xdata);
+    return 0;
+}
+
+int32_t
+cluster_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(symlink, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    FOP_CBK(rename, frame, cookie, op_ret, op_errno, buf, preoldparent,
+            postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(link, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(create, frame, cookie, op_ret, op_errno, fd, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    FOP_CBK(open, frame, cookie, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+cluster_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
+{
+    FOP_CBK(readv, frame, cookie, op_ret, op_errno, vector, count, stbuf,
+            iobref, xdata);
+    return 0;
+}
+
+int32_t
+cluster_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    FOP_CBK(writev, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_put_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    FOP_CBK(put, frame, cookie, op_ret, op_errno, inode, buf, preparent,
+            postparent, xdata);
+    return 0;
+}
+
+int32_t
+cluster_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(flush, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+    FOP_CBK(fsync, frame, cookie, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    FOP_CBK(fstat, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    FOP_CBK(opendir, frame, cookie, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(fsyncdir, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
+{
+    FOP_CBK(statfs, frame, cookie, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+cluster_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(setxattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(fsetxattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+    FOP_CBK(fgetxattr, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cluster_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+    FOP_CBK(getxattr, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cluster_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    FOP_CBK(xattrop, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+    FOP_CBK(fxattrop, frame, cookie, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cluster_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(removexattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(fremovexattr, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
+{
+    FOP_CBK(lk, frame, cookie, op_ret, op_errno, lock, xdata);
+    return 0;
+}
+
+int32_t
+cluster_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(inodelk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(finodelk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(entrylk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(fentrylk, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                      uint8_t *strong_checksum, dict_t *xdata)
+{
+    FOP_CBK(rchecksum, frame, cookie, op_ret, op_errno, weak_checksum,
+            strong_checksum, xdata);
+    return 0;
+}
+
+int32_t
+cluster_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+    FOP_CBK(readdir, frame, cookie, op_ret, op_errno, entries, xdata);
+    return 0;
+}
+
+int32_t
+cluster_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
+{
+    FOP_CBK(readdirp, frame, cookie, op_ret, op_errno, entries, xdata);
+    return 0;
+}
+
+int32_t
+cluster_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
+{
+    FOP_CBK(setattr, frame, cookie, op_ret, op_errno, statpre, statpost, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+    FOP_CBK(fsetattr, frame, cookie, op_ret, op_errno, statpre, statpost,
+            xdata);
+    return 0;
+}
+
+int32_t
+cluster_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                      struct iatt *post, dict_t *xdata)
+{
+    FOP_CBK(fallocate, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
+
+int32_t
+cluster_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
+{
+    FOP_CBK(discard, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
+
+int32_t
+cluster_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                     struct iatt *post, dict_t *xdata)
+{
+    FOP_CBK(zerofill, frame, cookie, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
+
+int32_t
+cluster_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    FOP_CBK(ipc, frame, cookie, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cluster_fgetxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fgetxattr, fd,
+               name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fsetxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsetxattr, fd,
+               dict, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_setxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, setxattr, loc,
+               dict, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_statfs(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, statfs, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fsyncdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                 dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsyncdir, fd,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_opendir(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, opendir, loc,
+               fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fstat(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fstat, fd,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fsync(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsync, fd,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_flush(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, flush, fd,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_writev(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, writev, fd,
+               vector, count, off, flags, iobref, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_put(xlator_t **subvols, unsigned char *on, int numsubvols,
+            default_args_cbk_t *replies, unsigned char *output,
+            call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+            off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, put, loc, mode,
+               umask, flags, vector, count, offset, iobref, xattr, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_readv(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readv, fd, size,
+               offset, flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_open(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, open, loc,
+               flags, fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_create(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, create, loc,
+               flags, mode, umask, fd, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_link(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, link, oldloc,
+               newloc, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_rename(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rename, oldloc,
+               newloc, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_symlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, symlink,
+               linkpath, loc, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_rmdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rmdir, loc,
+               flags, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_unlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, unlink, loc,
+               xflag, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_mkdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, mkdir, loc,
+               mode, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_mknod(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, mknod, loc,
+               mode, rdev, umask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_readlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readlink, loc,
+               size, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_access(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+               dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, access, loc,
+               mask, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_ftruncate(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, ftruncate, fd,
+               offset, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_getxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, getxattr, loc,
+               name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_xattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, xattrop, loc,
+               flags, dict, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fxattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fxattrop, fd,
+               flags, dict, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_removexattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                    default_args_cbk_t *replies, unsigned char *output,
+                    call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, removexattr,
+               loc, name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fremovexattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                     default_args_cbk_t *replies, unsigned char *output,
+                     call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fremovexattr,
+               fd, name, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_lk(xlator_t **subvols, unsigned char *on, int numsubvols,
+           default_args_cbk_t *replies, unsigned char *output,
+           call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *lock, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, lk, fd, cmd,
+               lock, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_rchecksum(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  int32_t len, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, rchecksum, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_readdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readdir, fd,
+               size, off, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_readdirp(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, readdirp, fd,
+               size, off, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_setattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, setattr, loc,
+               stbuf, valid, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_truncate(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, truncate, loc,
+               offset, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_stat(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, stat, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_lookup(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, lookup, loc,
+               xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fsetattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fsetattr, fd,
+               stbuf, valid, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_fallocate(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, fallocate, fd,
+               keep_size, offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_discard(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, discard, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_zerofill(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, zerofill, fd,
+               offset, len, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int32_t
+cluster_ipc(xlator_t **subvols, unsigned char *on, int numsubvols,
+            default_args_cbk_t *replies, unsigned char *output,
+            call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    FOP_ONLIST(subvols, on, numsubvols, replies, output, frame, ipc, op, xdata);
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_uninodelk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, off_t off, size_t size)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    FOP_ONLIST(subvols, locked_on, numsubvols, replies, output, frame, inodelk,
+               dom, &loc, F_SETLK, &flock, NULL);
+
+    loc_wipe(&loc);
+
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_tryinodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
+                   call_frame_t *frame, xlator_t *this, char *dom,
+                   inode_t *inode, off_t off, size_t size)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    loc_t loc = {0};
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
+
+int
+cluster_inodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                off_t off, size_t size)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_uninodelk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, off, size);
+
+            FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame, inodelk,
+                    dom, &loc, F_SETLKW, &flock, NULL);
+            break;
+        }
+    }
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
+
+int
+cluster_unentrylk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, const char *name)
+{
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    FOP_ONLIST(subvols, locked_on, numsubvols, replies, output, frame, entrylk,
+               dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+
+    loc_wipe(&loc);
+
+    return cluster_fop_success_fill(replies, numsubvols, output);
+}
+
+int
+cluster_tryentrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
+                   call_frame_t *frame, xlator_t *this, char *dom,
+                   inode_t *inode, const char *name)
+{
+    loc_t loc = {0};
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
+
+int
+cluster_entrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                const char *name)
+{
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_unentrylk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, name);
+            FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame, entrylk,
+                    dom, &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
+
+int
+cluster_tiebreaker_inodelk(xlator_t **subvols, unsigned char *on,
+                           int numsubvols, default_args_cbk_t *replies,
+                           unsigned char *locked_on, call_frame_t *frame,
+                           xlator_t *this, char *dom, inode_t *inode, off_t off,
+                           size_t size)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+    int num_success = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, inodelk, dom,
+               &loc, F_SETLK, &flock, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            num_success++;
+            continue;
+        }
+
+        /* TODO: If earlier subvols fail with an error other
+         * than EAGAIN, we could still have 2 clients competing
+         * for the lock*/
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_uninodelk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, off, size);
+
+            if (num_success) {
+                FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame,
+                        inodelk, dom, &loc, F_SETLKW, &flock, NULL);
+            } else {
+                loc_wipe(&loc);
+                memset(locked_on, 0, numsubvols);
+                return 0;
+            }
+            break;
+        }
+    }
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
+
+int
+cluster_tiebreaker_entrylk(xlator_t **subvols, unsigned char *on,
+                           int numsubvols, default_args_cbk_t *replies,
+                           unsigned char *locked_on, call_frame_t *frame,
+                           xlator_t *this, char *dom, inode_t *inode,
+                           const char *name)
+{
+    int i = 0;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+    int num_success = 0;
+
+    output = alloca(numsubvols);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    FOP_ONLIST(subvols, on, numsubvols, replies, locked_on, frame, entrylk, dom,
+               &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < numsubvols; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            num_success++;
+            continue;
+        }
+        if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            cluster_fop_success_fill(replies, numsubvols, locked_on);
+            cluster_unentrylk(subvols, locked_on, numsubvols, replies, output,
+                              frame, this, dom, inode, name);
+            if (num_success) {
+                FOP_SEQ(subvols, on, numsubvols, replies, locked_on, frame,
+                        entrylk, dom, &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK,
+                        NULL);
+            } else {
+                loc_wipe(&loc);
+                memset(locked_on, 0, numsubvols);
+                return 0;
+            }
+            break;
+        }
+    }
+
+    loc_wipe(&loc);
+    return cluster_fop_success_fill(replies, numsubvols, locked_on);
+}
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 6f40e7a6420..682cbf28055 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -1,29 +1,17 @@
 /*
-  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
+#else
+#include "execinfo_compat.h"
 #endif
 
 #include <stdio.h>
@@ -36,1539 +24,5442 @@
 #include <time.h>
 #include <locale.h>
 #include <sys/socket.h>
-#include <sys/wait.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <signal.h>
+#include <assert.h>
+#include <libgen.h> /* for dirname() */
+#include <grp.h>
+
+#if defined(GF_BSD_HOST_OS) || defined(GF_DARWIN_HOST_OS)
+#include <sys/sysctl.h>
+#endif
+#ifndef GF_LINUX_HOST_OS
+#include <sys/resource.h>
+#endif
+#ifdef HAVE_SYNCFS_SYS
+#include <sys/syscall.h>
+#endif
+
+#include "glusterfs/compat-errno.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/revision.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/stack.h"
+#include "glusterfs/lkowner.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/globals.h"
+#define XXH_INLINE_ALL
+#include "xxhash.h"
+#include <ifaddrs.h>
+#include "glusterfs/libglusterfs-messages.h"
+#include "glusterfs/glusterfs-acl.h"
+#ifdef __FreeBSD__
+#include <pthread_np.h>
+#undef BIT_SET
+#endif
 
-#include "logging.h"
-#include "common-utils.h"
-#include "revision.h"
-#include "glusterfs.h"
-#include "stack.h"
+#ifndef AI_ADDRCONFIG
+#define AI_ADDRCONFIG 0
+#endif /* AI_ADDRCONFIG */
+
+char *vol_type_str[] = {
+    "Distribute",
+    "Stripe [NOT SUPPORTED from v6.0]",
+    "Replicate",
+    "Striped-Replicate [NOT SUPPORTED from v6.0]",
+    "Disperse",
+    "Tier [NOT SUPPORTED from v6.0]",
+    "Distributed-Stripe [NOT SUPPORTED from v6.0]",
+    "Distributed-Replicate",
+    "Distributed-Striped-Replicate [NOT SUPPORTED from v6.0]",
+    "Distributed-Disperse",
+};
 
 typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size);
 typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size);
 
+char *xattrs_to_heal[] = {"user.",
+                          POSIX_ACL_ACCESS_XATTR,
+                          POSIX_ACL_DEFAULT_XATTR,
+                          QUOTA_LIMIT_KEY,
+                          QUOTA_LIMIT_OBJECTS_KEY,
+                          GF_SELINUX_XATTR_KEY,
+                          GF_XATTR_MDATA_KEY,
+                          NULL};
 
-struct dnscache6 {
-	struct addrinfo *first;
-	struct addrinfo *next;
-};
+void
+gf_xxh64_wrapper(const unsigned char *data, size_t const len,
+                 unsigned long long const seed, char *xxh64)
+{
+    unsigned short i = 0;
+    const unsigned short lim = GF_XXH64_DIGEST_LENGTH * 2 + 1;
+    XXH64_hash_t hash = 0;
+    XXH64_canonical_t c_hash = {
+        {
+            0,
+        },
+    };
+    const uint8_t *p = (const uint8_t *)&c_hash;
+
+    hash = XXH64(data, len, seed);
+    XXH64_canonicalFromHash(&c_hash, hash);
+
+    for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++)
+        snprintf(xxh64 + i * 2, lim - i * 2, "%02x", p[i]);
+}
+
+/**
+ * This function takes following arguments
+ * @this: xlator
+ * @gfid: The gfid which has to be filled
+ * @hash: the 8 byte hash which has to be filled inside the gfid
+ * @index: the array element of the uuid_t structure (which is
+ *         a array of unsigned char) from where the 8 bytes of
+ *         the hash has to be filled. Since uuid_t contains 16
+ *        char elements in the array, each byte of the hash has
+ *        to be filled in one array element.
+ *
+ * This function is called twice for 2 hashes (of 8 byte each) to
+ * be filled in the gfid.
+ *
+ * The for loop in this function actually is doing these 2 things
+ * for each hash
+ *
+ * 1) One of the hashes
+ *      tmp[0] = (hash_2 >> 56) & 0xff;
+ *      tmp[1] = (hash_2 >> 48) & 0xff;
+ *      tmp[2] = (hash_2 >> 40) & 0xff;
+ *      tmp[3] = (hash_2 >> 32) & 0xff;
+ *      tmp[4] = (hash_2 >> 24) & 0xff;
+ *      tmp[5] = (hash_2 >> 16) & 0xff;
+ *      tmp[6] = (hash_2 >> 8) & 0xff;
+ *      tmp[7] = (hash_2) & 0xff;
+ *
+ * 2) The other hash:
+ *      tmp[8] = (hash_1 >> 56) & 0xff;
+ *      tmp[9] = (hash_1 >> 48) & 0xff;
+ *      tmp[10] = (hash_1 >> 40) & 0xff;
+ *      tmp[11] = (hash_1 >> 32) & 0xff;
+ *      tmp[12] = (hash_1 >> 24) & 0xff;
+ *      tmp[13] = (hash_1 >> 16) & 0xff;
+ *      tmp[14] = (hash_1 >> 8) & 0xff;
+ *      tmp[15] = (hash_1) & 0xff;
+ **/
+static int
+gf_gfid_from_xxh64(xlator_t *this, uuid_t gfid, XXH64_hash_t hash,
+                   unsigned short index)
+{
+    int ret = -1;
+    int i = -1;
+
+    if ((index != 0) && (index != 8)) {
+        gf_msg_callingfn("gfid-from-xxh64", GF_LOG_WARNING, 0,
+                         LG_MSG_INDEX_NOT_FOUND,
+                         "index can only be either 0 or 8, as this"
+                         "function's purpose is to encode a 8 byte "
+                         "hash inside the gfid (index: %d)",
+                         index);
+        goto out;
+    }
+
+    for (i = 0; i < sizeof(hash); i++) {
+        /*
+         * As of now the below statement is equivalent of this.
+         * gfid[index+i] = (hash >> (64 - (8 * (i+1)))) & 0xff;
+         */
+        gfid[index + i] = (hash >> ((sizeof(hash) * 8) - (8 * (i + 1)))) &
+                          (0xff);
+    }
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+/**
+ * This function does the same thing as gf_xxh64_wrapper. But gf_xxh64_wrapper
+ * does not return anything and in this xlator there is a need for both the
+ * actual hash and the canonicalized form of the hash.
+ *
+ * To summarize:
+ * - XXH64_hash_t is needed as return because, those bytes which contain the
+ *   hash can be used for different purposes as needed. One example is
+ *   to have those bytes copied into the uuid_t structure to be used as gfid
+ * - xxh64 string is needed because, it can be used as the key for generating
+ *   the next hash (and any other purpose which might require canonical form
+ *   of the hash).
+ **/
+XXH64_hash_t
+gf_xxh64_hash_wrapper(const unsigned char *data, size_t const len,
+                      unsigned long long const seed, char *xxh64)
+{
+    unsigned short i = 0;
+    const unsigned short lim = GF_XXH64_DIGEST_LENGTH * 2 + 1;
+    XXH64_hash_t hash = 0;
+    XXH64_canonical_t c_hash = {
+        {
+            0,
+        },
+    };
+    const uint8_t *p = (const uint8_t *)&c_hash;
+
+    hash = XXH64(data, len, seed);
+    XXH64_canonicalFromHash(&c_hash, hash);
 
+    for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++)
+        snprintf(xxh64 + i * 2, lim - i * 2, "%02x", p[i]);
 
+    return hash;
+}
+
+/**
+ * This is the algorithm followed for generating new gfid
+ * 1) generate xxh64 hash using snapname and original gfid of the object
+ * 2) Using the canonicalized form of above hash as the key, generate
+ *    another hash
+ * 3) Combine both of the  8 byte hashes to generate a 16 byte uuid_t type
+ * 4) Use the above uuid as the gfid
+ *
+ * Each byte of the hash is stored separately in different elements of the
+ * character array represented by uuid_t
+ * Ex: tmp[0] = (hash_2 >> 56) & 0xFF
+ *     This saves the most significant byte of hash_2 in tmp[0]
+ *     tmp[1] = (hash_2 >> 48) & 0xFF
+ *     This saves next most significant byte of hash_2 in tmp[1]
+ *     .
+ *     .
+ *     So on.
+ *     tmp[0] - tmp[7] holds the contents of hash_2
+ *     tmp[8] - tmp[15] hold the conents of hash_1
+ *
+ * The hash generated (i.e. of type XXH64_hash_t) is 8 bytes long. And for
+ * gfid 16 byte uuid is needed. Hecne the 2 hashes are combined to form
+ * one 16 byte entity.
+ **/
+int
+gf_gfid_generate_from_xxh64(uuid_t gfid, char *key)
+{
+    char xxh64_1[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {
+        0,
+    };
+    char xxh64_2[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {
+        0,
+    };
+    XXH64_hash_t hash_1 = 0;
+    XXH64_hash_t hash_2 = 0;
+    int ret = -1;
+    xlator_t *this = THIS;
+
+    hash_1 = gf_xxh64_hash_wrapper((unsigned char *)key, strlen(key),
+                                   GF_XXHSUM64_DEFAULT_SEED, xxh64_1);
+
+    hash_2 = gf_xxh64_hash_wrapper((unsigned char *)xxh64_1, strlen(xxh64_1),
+                                   GF_XXHSUM64_DEFAULT_SEED, xxh64_2);
+
+    /* hash_2 is saved in 1st 8 elements of uuid_t char array */
+    if (gf_gfid_from_xxh64(this, gfid, hash_2, 0)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         LG_MSG_XXH64_TO_GFID_FAILED,
+                         "failed to encode the hash %llx into the 1st"
+                         "half of gfid",
+                         hash_2);
+        goto out;
+    }
+
+    /* hash_1 is saved in the remaining 8 elements of uuid_t */
+    if (gf_gfid_from_xxh64(this, gfid, hash_1, 8)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         LG_MSG_XXH64_TO_GFID_FAILED,
+                         "failed to encode the hash %llx into the 2nd"
+                         "half of gfid",
+                         hash_1);
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "gfid generated is %s (hash1: %llx) "
+                 "hash2: %llx, xxh64_1: %s xxh64_2: %s",
+                 uuid_utoa(gfid), hash_1, hash_2, xxh64_1, xxh64_2);
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+/* works similar to mkdir(1) -p.
+ */
 int
-log_base2 (unsigned long x)
+mkdir_p(char *path, mode_t mode, gf_boolean_t allow_symlinks)
 {
-        int val = 0;
+    int i = 0;
+    int ret = -1;
+    char dir[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+
+    const int path_len = min(strlen(path), PATH_MAX - 1);
+
+    snprintf(dir, path_len + 1, "%s", path);
+
+    i = (dir[0] == '/') ? 1 : 0;
+    do {
+        if (path[i] != '/' && path[i] != '\0')
+            continue;
+
+        dir[i] = '\0';
+        ret = sys_mkdir(dir, mode);
+        if (ret && errno != EEXIST) {
+            gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED, NULL);
+            goto out;
+        }
+
+        if (ret && errno == EEXIST && !allow_symlinks) {
+            ret = sys_lstat(dir, &stbuf);
+            if (ret)
+                goto out;
 
-        while (x > 1) {
-                x /= 2;
-                val++;
+            if (S_ISLNK(stbuf.st_mode)) {
+                ret = -1;
+                gf_smsg("", GF_LOG_ERROR, 0, LG_MSG_DIR_IS_SYMLINK, "dir=%s",
+                        dir, NULL);
+                goto out;
+            }
         }
+        dir[i] = '/';
+
+    } while (path[i++] != '\0');
+
+    ret = sys_stat(dir, &stbuf);
+    if (ret || !S_ISDIR(stbuf.st_mode)) {
+        if (ret == 0)
+            errno = 0;
+        ret = -1;
+        gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED,
+                "possibly some of the components"
+                " were not directories",
+                NULL);
+        goto out;
+    }
+
+    ret = 0;
+out:
 
-        return val;
+    return ret;
 }
 
+int
+gf_lstat_dir(const char *path, struct stat *stbuf_in)
+{
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+
+    if (path == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = sys_lstat(path, &stbuf);
+    if (ret)
+        goto out;
+
+    if (!S_ISDIR(stbuf.st_mode)) {
+        errno = ENOTDIR;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+out:
+    if (!ret && stbuf_in)
+        *stbuf_in = stbuf;
+
+    return ret;
+}
+
+int
+log_base2(unsigned long x)
+{
+    int val = 0;
+
+    while (x > 1) {
+        x /= 2;
+        val++;
+    }
+
+    return val;
+}
+
+/**
+ * gf_rev_dns_lookup -- Perform a reverse DNS lookup on the IP address.
+ *
+ * @ip: The IP address to perform a reverse lookup on
+ *
+ * @return: success: Allocated string containing the hostname
+ *          failure: NULL
+ */
+char *
+gf_rev_dns_lookup(const char *ip)
+{
+    char *fqdn = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("resolver", ip, out);
+
+    /* Get the FQDN */
+    ret = gf_get_hostname_from_ip((char *)ip, &fqdn);
+    if (ret != 0) {
+        gf_smsg("resolver", GF_LOG_INFO, errno, LG_MSG_RESOLVE_HOSTNAME_FAILED,
+                "hostname=%s", ip, NULL);
+    }
+out:
+    return fqdn;
+}
+
+/**
+ * gf_resolve_path_parent -- Given a path, returns an allocated string
+ *                           containing the parent's path.
+ * @path: Path to parse
+ * @return: The parent path if found, NULL otherwise
+ */
+char *
+gf_resolve_path_parent(const char *path)
+{
+    char *parent = NULL;
+    char *tmp = NULL;
+    char *pathc = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, path, out);
+
+    if (0 == strlen(path)) {
+        gf_msg_callingfn(THIS->name, GF_LOG_DEBUG, 0, LG_MSG_INVALID_STRING,
+                         "invalid string for 'path'");
+        goto out;
+    }
+
+    /* dup the parameter, we don't want to modify it */
+    pathc = strdupa(path);
+    if (!pathc) {
+        goto out;
+    }
+
+    /* Get the parent directory */
+    tmp = dirname(pathc);
+    if (strcmp(tmp, "/") == 0)
+        goto out;
+
+    parent = gf_strdup(tmp);
+out:
+    return parent;
+}
 
 int32_t
-gf_resolve_ip6 (const char *hostname,
-		uint16_t port,
-		int family,
-		void **dnscache,
-		struct addrinfo **addr_info)
-{
-	int32_t ret = 0;
-	struct addrinfo hints;
-	struct dnscache6 *cache = NULL;
-	char service[NI_MAXSERV], host[NI_MAXHOST];
-
-	if (!hostname) {
-		gf_log ("resolver", GF_LOG_WARNING, "hostname is NULL");
-		return -1;
-	}
-
-	if (!*dnscache) {
-		*dnscache = GF_CALLOC (1, sizeof (struct dnscache6),
-                                        gf_common_mt_dnscache6);
-                if (!*dnscache)
-                        return -1;
-	}
-
-	cache = *dnscache;
-	if (cache->first && !cache->next) {
-		freeaddrinfo(cache->first);
-		cache->first = cache->next = NULL;
-		gf_log ("resolver", GF_LOG_TRACE,
-			"flushing DNS cache");
-	}
-
-	if (!cache->first) {
-		char *port_str = NULL;
-		gf_log ("resolver", GF_LOG_TRACE,
-			"DNS cache not present, freshly probing hostname: %s",
-			hostname);
-
-		memset(&hints, 0, sizeof(hints));
-		hints.ai_family   = family;
-		hints.ai_socktype = SOCK_STREAM;
-		hints.ai_flags    = AI_ADDRCONFIG;
-
-		ret = gf_asprintf (&port_str, "%d", port);
-                if (-1 == ret) {
-                        gf_log ("resolver", GF_LOG_ERROR, "asprintf failed");
-                        return -1;
-                }
-		if ((ret = getaddrinfo(hostname, port_str, &hints, &cache->first)) != 0) {
-			gf_log ("resolver", GF_LOG_ERROR,
-				"getaddrinfo failed (%s)", gai_strerror (ret));
-
-			GF_FREE (*dnscache);
-			*dnscache = NULL;
-			GF_FREE (port_str);
-			return -1;
-		}
-		GF_FREE (port_str);
-
-		cache->next = cache->first;
-	}
-
-	if (cache->next) {
-		ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
-				  cache->next->ai_addrlen,
-				  host, sizeof (host),
-				  service, sizeof (service),
-				  NI_NUMERICHOST);
-		if (ret != 0) {
-			gf_log ("resolver",
-				GF_LOG_ERROR,
-				"getnameinfo failed (%s)", gai_strerror (ret));
-			goto err;
-		}
-
-		gf_log ("resolver", GF_LOG_TRACE,
-			"returning ip-%s (port-%s) for hostname: %s and port: %d",
-			host, service, hostname, port);
-
-		*addr_info = cache->next;
-	}
-
-        if (cache->next)
-                cache->next = cache->next->ai_next;
-	if (cache->next) {
-		ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
-				  cache->next->ai_addrlen,
-				  host, sizeof (host),
-				  service, sizeof (service),
-				  NI_NUMERICHOST);
-		if (ret != 0) {
-			gf_log ("resolver",
-				GF_LOG_ERROR,
-				"getnameinfo failed (%s)", gai_strerror (ret));
-			goto err;
-		}
-
-		gf_log ("resolver", GF_LOG_TRACE,
-			"next DNS query will return: ip-%s port-%s", host, service);
-	}
-
-	return 0;
+gf_resolve_ip6(const char *hostname, uint16_t port, int family, void **dnscache,
+               struct addrinfo **addr_info)
+{
+    int32_t ret = 0;
+    struct addrinfo hints;
+    struct dnscache6 *cache = NULL;
+    char service[NI_MAXSERV], host[NI_MAXHOST];
+
+    if (!hostname) {
+        gf_msg_callingfn("resolver", GF_LOG_WARNING, 0, LG_MSG_HOSTNAME_NULL,
+                         "hostname is NULL");
+        return -1;
+    }
+
+    if (!*dnscache) {
+        *dnscache = GF_CALLOC(1, sizeof(struct dnscache6),
+                              gf_common_mt_dnscache6);
+        if (!*dnscache)
+            return -1;
+    }
+
+    cache = *dnscache;
+    if (cache->first && !cache->next) {
+        freeaddrinfo(cache->first);
+        cache->first = cache->next = NULL;
+        gf_msg_trace("resolver", 0, "flushing DNS cache");
+    }
+
+    if (!cache->first) {
+        char *port_str = NULL;
+        gf_msg_trace("resolver", 0,
+                     "DNS cache not present, freshly "
+                     "probing hostname: %s",
+                     hostname);
+
+        memset(&hints, 0, sizeof(hints));
+        hints.ai_family = family;
+        hints.ai_socktype = SOCK_STREAM;
+
+        ret = gf_asprintf(&port_str, "%d", port);
+        if (-1 == ret) {
+            return -1;
+        }
+        if ((ret = getaddrinfo(hostname, port_str, &hints, &cache->first)) !=
+            0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
+                    "family=%d", family, "ret=%s", gai_strerror(ret), NULL);
+
+            GF_FREE(*dnscache);
+            *dnscache = NULL;
+            GF_FREE(port_str);
+            return -1;
+        }
+        GF_FREE(port_str);
+
+        cache->next = cache->first;
+    }
+
+    if (cache->next) {
+        ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
+                          cache->next->ai_addrlen, host, sizeof(host), service,
+                          sizeof(service), NI_NUMERICHOST);
+        if (ret != 0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto err;
+        }
+
+        gf_msg_debug("resolver", 0,
+                     "returning ip-%s (port-%s) for "
+                     "hostname: %s and port: %d",
+                     host, service, hostname, port);
+
+        *addr_info = cache->next;
+    }
+
+    if (cache->next)
+        cache->next = cache->next->ai_next;
+    if (cache->next) {
+        ret = getnameinfo((struct sockaddr *)cache->next->ai_addr,
+                          cache->next->ai_addrlen, host, sizeof(host), service,
+                          sizeof(service), NI_NUMERICHOST);
+        if (ret != 0) {
+            gf_smsg("resolver", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto err;
+        }
+
+        gf_msg_debug("resolver", 0,
+                     "next DNS query will return: "
+                     "ip-%s port-%s",
+                     host, service);
+    }
+
+    return 0;
 
 err:
-	freeaddrinfo (cache->first);
-	cache->first = cache->next = NULL;
-	GF_FREE (cache);
-	*dnscache = NULL;
-	return -1;
+    freeaddrinfo(cache->first);
+    cache->first = cache->next = NULL;
+    GF_FREE(cache);
+    *dnscache = NULL;
+    return -1;
+}
+
+/**
+ * gf_dnscache_init -- Initializes a dnscache struct and sets the ttl
+ *                     to the specified value in the parameter.
+ *
+ * @ttl: the TTL in seconds
+ * @return: SUCCESS: Pointer to an allocated dnscache struct
+ *          FAILURE: NULL
+ */
+struct dnscache *
+gf_dnscache_init(time_t ttl)
+{
+    struct dnscache *cache = GF_MALLOC(sizeof(*cache), gf_common_mt_dnscache);
+    if (!cache)
+        return NULL;
+
+    cache->cache_dict = dict_new();
+    if (!cache->cache_dict) {
+        GF_FREE(cache);
+        cache = NULL;
+    } else {
+        cache->ttl = ttl;
+    }
+
+    return cache;
+}
+
+/**
+ * gf_dnscache_deinit -- cleanup resources used by struct dnscache
+ */
+void
+gf_dnscache_deinit(struct dnscache *cache)
+{
+    if (!cache) {
+        gf_msg_plain(GF_LOG_WARNING, "dnscache is NULL");
+        return;
+    }
+    dict_unref(cache->cache_dict);
+    GF_FREE(cache);
 }
 
+/**
+ * gf_dnscache_entry_init -- Initialize a dnscache entry
+ *
+ * @return: SUCCESS: Pointer to an allocated dnscache entry struct
+ *          FAILURE: NULL
+ */
+struct dnscache_entry *
+gf_dnscache_entry_init()
+{
+    struct dnscache_entry *entry = GF_CALLOC(1, sizeof(*entry),
+                                             gf_common_mt_dnscache_entry);
+    return entry;
+}
 
+/**
+ * gf_dnscache_entry_deinit -- Free memory used by a dnscache entry
+ *
+ * @entry: Pointer to deallocate
+ */
 void
-gf_log_volume_file (FILE *specfp)
+gf_dnscache_entry_deinit(struct dnscache_entry *entry)
+{
+    GF_FREE(entry->ip);
+    GF_FREE(entry->fqdn);
+    GF_FREE(entry);
+}
+
+/**
+ * gf_rev_dns_lookup -- Perform a reverse DNS lookup on the IP address.
+ *
+ * @ip: The IP address to perform a reverse lookup on
+ *
+ * @return: success: Allocated string containing the hostname
+ *          failure: NULL
+ */
+char *
+gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache)
 {
-	extern FILE *gf_log_logfile;
-	int          lcount = 0;
-	char         data[GF_UNIT_KB];
+    char *fqdn = NULL;
+    int ret = 0;
+    dict_t *cache = NULL;
+    data_t *entrydata = NULL;
+    struct dnscache_entry *dnsentry = NULL;
+    gf_boolean_t from_cache = _gf_false;
+
+    if (!dnscache)
+        goto out;
+
+    cache = dnscache->cache_dict;
+
+    /* Quick cache lookup to see if we already hold it */
+    entrydata = dict_get(cache, (char *)ip);
+    if (entrydata) {
+        dnsentry = (struct dnscache_entry *)entrydata->data;
+        /* First check the TTL & timestamp */
+        if (gf_time() - dnsentry->timestamp > dnscache->ttl) {
+            gf_dnscache_entry_deinit(dnsentry);
+            entrydata->data = NULL; /* Mark this as 'null' so
+                                     * dict_del () doesn't try free
+                                     * this after we've already
+                                     * freed it.
+                                     */
+
+            dict_del(cache, (char *)ip); /* Remove this entry */
+        } else {
+            /* Cache entry is valid, get the FQDN and return */
+            fqdn = dnsentry->fqdn;
+            from_cache = _gf_true; /* Mark this as from cache */
+            goto out;
+        }
+    }
+
+    /* Get the FQDN */
+    ret = gf_get_hostname_from_ip((char *)ip, &fqdn);
+    if (ret != 0)
+        goto out;
 
-	fseek (specfp, 0L, SEEK_SET);
+    if (!fqdn) {
+        gf_log_callingfn("resolver", GF_LOG_CRITICAL,
+                         "Allocation failed for the host address");
+        goto out;
+    }
 
-	fprintf (gf_log_logfile, "Given volfile:\n");
-	fprintf (gf_log_logfile,
-		 "+---------------------------------------"
-		 "---------------------------------------+\n");
-	while (fgets (data, GF_UNIT_KB, specfp) != NULL){
-		lcount++;
-		fprintf (gf_log_logfile, "%3d: %s", lcount, data);
-	}
-	fprintf (gf_log_logfile,
-		 "\n+---------------------------------------"
-		 "---------------------------------------+\n");
-	fflush (gf_log_logfile);
-	fseek (specfp, 0L, SEEK_SET);
+    from_cache = _gf_false;
+out:
+    /* Insert into the cache */
+    if (fqdn && !from_cache && ip) {
+        struct dnscache_entry *entry = gf_dnscache_entry_init();
+
+        if (entry) {
+            entry->fqdn = fqdn;
+            entry->ip = gf_strdup(ip);
+            entry->timestamp = gf_time();
+            entrydata = bin_to_data(entry, sizeof(*entry));
+            dict_set(cache, (char *)ip, entrydata);
+        }
+    }
+    return fqdn;
+}
+
+struct xldump {
+    int lineno;
+};
+
+/* to catch any format discrepencies that may arise in code */
+static int
+nprintf(struct xldump *dump, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+static int
+nprintf(struct xldump *dump, const char *fmt, ...)
+{
+    va_list ap;
+    char *msg = NULL;
+    char header[32];
+    int ret = 0;
+
+    ret = snprintf(header, 32, "%3d:", ++dump->lineno);
+    if (ret < 0)
+        goto out;
+
+    va_start(ap, fmt);
+    ret = vasprintf(&msg, fmt, ap);
+    va_end(ap);
+    if (-1 == ret)
+        goto out;
+
+    /* NOTE: No ret value from gf_msg_plain, so unable to compute printed
+     * characters. The return value from nprintf is not used, so for now
+     * living with it */
+    gf_msg_plain(GF_LOG_WARNING, "%s %s", header, msg);
+
+out:
+    FREE(msg);
+    return 0;
+}
+
+static int
+xldump_options(dict_t *this, char *key, data_t *value, void *d)
+{
+    nprintf(d, "    option %s %s", key, value->data);
+    return 0;
 }
 
 static void
-gf_dump_config_flags (int fd)
+xldump_subvolumes(xlator_t *this, void *d)
 {
-        int ret = 0;
+    xlator_list_t *subv = NULL;
+    int len = 0;
+    char *subvstr = NULL;
 
-	ret = write (fd, "configuration details:\n", 23);
-        if (ret == -1)
-                goto out;
+    if (!this->children)
+        return;
+
+    for (subv = this->children; subv; subv = subv->next)
+        len += (strlen(subv->xlator->name) + 1);
+
+    subvstr = GF_MALLOC(len, gf_common_mt_strdup);
+
+    len = 0;
+    for (subv = this->children; subv; subv = subv->next)
+        len += sprintf(subvstr + len, "%s%s", subv->xlator->name,
+                       subv->next ? " " : "");
+
+    nprintf(d, "    subvolumes %s", subvstr);
+
+    GF_FREE(subvstr);
+}
+
+static void
+xldump(xlator_t *each, void *d)
+{
+    nprintf(d, "volume %s", each->name);
+    nprintf(d, "    type %s", each->type);
+    dict_foreach(each->options, xldump_options, d);
+
+    xldump_subvolumes(each, d);
+
+    nprintf(d, "end-volume");
+    nprintf(d, " ");
+}
+
+void
+gf_log_dump_graph(FILE *specfp, glusterfs_graph_t *graph)
+{
+    struct xldump xld = {
+        0,
+    };
+
+    gf_msg_plain(GF_LOG_WARNING, "Final graph:");
+    gf_msg_plain(GF_LOG_WARNING,
+                 "+---------------------------------------"
+                 "---------------------------------------+");
+
+    xlator_foreach_depth_first(graph->top, xldump, &xld);
+
+    gf_msg_plain(GF_LOG_WARNING,
+                 "+---------------------------------------"
+                 "---------------------------------------+");
+}
+
+static void
+gf_dump_config_flags()
+{
+    gf_msg_plain_nomem(GF_LOG_ALERT, "configuration details:");
 
 /* have argp */
 #ifdef HAVE_ARGP
-	ret = write (fd, "argp 1\n", 7);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "argp 1");
 #endif
 
 /* ifdef if found backtrace */
 #ifdef HAVE_BACKTRACE
-	ret = write (fd, "backtrace 1\n", 12);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "backtrace 1");
 #endif
 
 /* Berkeley-DB version has cursor->get() */
 #ifdef HAVE_BDB_CURSOR_GET
-	ret = write (fd, "bdb->cursor->get 1\n", 19);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "bdb->cursor->get 1");
 #endif
 
 /* Define to 1 if you have the <db.h> header file. */
 #ifdef HAVE_DB_H
-	ret = write (fd, "db.h 1\n", 7);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "db.h 1");
 #endif
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #ifdef HAVE_DLFCN_H
-	ret = write (fd, "dlfcn 1\n", 8);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "dlfcn 1");
 #endif
 
 /* define if fdatasync exists */
 #ifdef HAVE_FDATASYNC
-	ret = write (fd, "fdatasync 1\n", 12);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "fdatasync 1");
 #endif
 
 /* Define to 1 if you have the `pthread' library (-lpthread). */
 #ifdef HAVE_LIBPTHREAD
-	ret = write (fd, "libpthread 1\n", 13);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "libpthread 1");
 #endif
 
 /* define if llistxattr exists */
 #ifdef HAVE_LLISTXATTR
-	ret = write (fd, "llistxattr 1\n", 13);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "llistxattr 1");
 #endif
 
 /* define if found setfsuid setfsgid */
 #ifdef HAVE_SET_FSID
-	ret = write (fd, "setfsid 1\n", 10);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "setfsid 1");
 #endif
 
 /* define if found spinlock */
 #ifdef HAVE_SPINLOCK
-	ret = write (fd, "spinlock 1\n", 11);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "spinlock 1");
 #endif
 
 /* Define to 1 if you have the <sys/epoll.h> header file. */
 #ifdef HAVE_SYS_EPOLL_H
-	ret = write (fd, "epoll.h 1\n", 10);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "epoll.h 1");
 #endif
 
 /* Define to 1 if you have the <sys/extattr.h> header file. */
 #ifdef HAVE_SYS_EXTATTR_H
-	ret = write (fd, "extattr.h 1\n", 12);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "extattr.h 1");
 #endif
 
 /* Define to 1 if you have the <sys/xattr.h> header file. */
 #ifdef HAVE_SYS_XATTR_H
-	ret = write (fd, "xattr.h 1\n", 10);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "xattr.h 1");
 #endif
 
 /* define if found st_atim.tv_nsec */
 #ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-	ret = write (fd, "st_atim.tv_nsec 1\n", 18);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "st_atim.tv_nsec 1");
 #endif
 
 /* define if found st_atimespec.tv_nsec */
 #ifdef HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC
-	ret = write (fd, "st_atimespec.tv_nsec 1\n",23);
-        if (ret == -1)
-                goto out;
+    gf_msg_plain_nomem(GF_LOG_ALERT, "st_atimespec.tv_nsec 1");
 #endif
 
 /* Define to the full name and version of this package. */
 #ifdef PACKAGE_STRING
-	{
-		char msg[128];
-		sprintf (msg, "package-string: %s\n", PACKAGE_STRING);
-		ret = write (fd, msg, strlen (msg));
-                if (ret == -1)
-                        goto out;
-	}
+    {
+        char *msg = NULL;
+        int ret = -1;
+
+        ret = gf_asprintf(&msg, "package-string: %s", PACKAGE_STRING);
+        if (ret >= 0) {
+            gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+            GF_FREE(msg);
+        }
+    }
 #endif
 
-out:
-	return;
+    return;
 }
 
-/* Obtain a backtrace and print it to stdout. */
-/* TODO: It looks like backtrace_symbols allocates memory,
-   it may be problem because mostly memory allocation/free causes 'sigsegv' */
+/* Obtain a backtrace and print it to the log */
 void
-gf_print_trace (int32_t signum)
-{
-	extern FILE *gf_log_logfile;
-        struct tm   *tm = NULL;
-        char         msg[1024] = {0,};
-        char         timestr[256] = {0,};
-        time_t       utime = 0;
-        int          ret = 0;
-        int          fd = 0;
-
-        fd = fileno (gf_log_logfile);
-
-	/* Pending frames, (if any), list them in order */
-	ret = write (fd, "pending frames:\n", 16);
-	{
-		glusterfs_ctx_t *ctx = glusterfs_ctx_get ();
-		struct list_head *trav = ((call_pool_t *)ctx->pool)->all_frames.next;
-		while (trav != (&((call_pool_t *)ctx->pool)->all_frames)) {
-			call_frame_t *tmp = (call_frame_t *)(&((call_stack_t *)trav)->frames);
-			if (tmp->root->type == GF_OP_TYPE_FOP)
-				sprintf (msg,"frame : type(%d) op(%s)\n",
-					 tmp->root->type,
-					 gf_fop_list[tmp->root->op]);
-			if (tmp->root->type == GF_OP_TYPE_MGMT)
-				sprintf (msg,"frame : type(%d) op(%s)\n",
-					 tmp->root->type,
-					 gf_mgmt_list[tmp->root->op]);
-
-			ret = write (fd, msg, strlen (msg));
-			trav = trav->next;
-		}
-		ret = write (fd, "\n", 1);
-	}
-
-	sprintf (msg, "patchset: %s\n", GLUSTERFS_REPOSITORY_REVISION);
-	ret = write (fd, msg, strlen (msg));
-
-	sprintf (msg, "signal received: %d\n", signum);
-	ret = write (fd, msg, strlen (msg));
-
+gf_print_trace(int32_t signum, glusterfs_ctx_t *ctx)
+{
+    char msg[1024] = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    call_stack_t *stack = NULL;
+
+    /* Now every gf_log call will just write to a buffer and when the
+     * buffer becomes full, its written to the log-file. Suppose the process
+     * crashes and prints the backtrace in the log-file, then the previous
+     * log information will still be in the buffer itself. So flush the
+     * contents of the buffer to the log file before printing the backtrace
+     * which helps in debugging.
+     */
+    gf_log_flush();
+
+    gf_log_disable_suppression_before_exit(ctx);
+
+    /* Pending frames, (if any), list them in order */
+    gf_msg_plain_nomem(GF_LOG_ALERT, "pending frames:");
+    {
+        /* FIXME: traversing stacks outside pool->lock */
+        list_for_each_entry(stack, &ctx->pool->all_frames, all_frames)
         {
-                /* Dump the timestamp of the crash too, so the previous logs
-                   can be related */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S\n", tm);
-                ret = write (fd, "time of crash: ", 15);
-                ret = write (fd, timestr, strlen (timestr));
+            if (stack->type == GF_OP_TYPE_FOP)
+                sprintf(msg, "frame : type(%d) op(%s)", stack->type,
+                        gf_fop_list[stack->op]);
+            else
+                sprintf(msg, "frame : type(%d) op(%d)", stack->type, stack->op);
+
+            gf_msg_plain_nomem(GF_LOG_ALERT, msg);
         }
+    }
+
+    sprintf(msg, "patchset: %s", GLUSTERFS_REPOSITORY_REVISION);
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+
+    sprintf(msg, "signal received: %d", signum);
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+    {
+        /* Dump the timestamp of the crash too, so the previous logs
+           can be related */
+        gf_time_fmt(timestr, sizeof timestr, gf_time(), gf_timefmt_FT);
+        gf_msg_plain_nomem(GF_LOG_ALERT, "time of crash: ");
+        gf_msg_plain_nomem(GF_LOG_ALERT, timestr);
+    }
+
+    gf_dump_config_flags();
+    gf_msg_backtrace_nomem(GF_LOG_ALERT, 200);
+    sprintf(msg, "---------");
+    gf_msg_plain_nomem(GF_LOG_ALERT, msg);
+
+    /* Send a signal to terminate the process */
+    signal(signum, SIG_DFL);
+    raise(signum);
+}
+
+void
+trap(void)
+{
+}
+
+char *
+gf_trim(char *string)
+{
+    register char *s, *t;
+
+    if (string == NULL) {
+        return NULL;
+    }
 
-	gf_dump_config_flags (fd);
-#if HAVE_BACKTRACE
-	/* Print 'backtrace' */
-	{
-		void *array[200];
-		size_t size;
+    for (s = string; isspace(*s); s++)
+        ;
 
-		size = backtrace (array, 200);
-		backtrace_symbols_fd (&array[1], size-1, fd);
-		sprintf (msg, "---------\n");
-		ret = write (fd, msg, strlen (msg));
-	}
-#endif /* HAVE_BACKTRACE */
+    if (*s == 0)
+        return s;
 
-	/* Send a signal to terminate the process */
-	signal (signum, SIG_DFL);
-	raise (signum);
+    t = s + strlen(s) - 1;
+    while (t > s && isspace(*t))
+        t--;
+    *++t = '\0';
+
+    return s;
 }
 
-void
-trap (void)
+int
+gf_strstr(const char *str, const char *delim, const char *match)
 {
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *tmp_str = NULL;
+
+    int ret = 0;
+
+    tmp_str = strdup(str);
+
+    if (str == NULL || delim == NULL || match == NULL || tmp_str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        ret = -1;
+        goto out;
+    }
+
+    tmp = strtok_r(tmp_str, delim, &save_ptr);
 
+    while (tmp) {
+        ret = strcmp(tmp, match);
+
+        if (ret == 0)
+            break;
+
+        tmp = strtok_r(NULL, delim, &save_ptr);
+    }
+
+out:
+    free(tmp_str);
+
+    return ret;
 }
 
-char *
-gf_trim (char *string)
+int
+gf_volume_name_validate(const char *volume_name)
 {
-	register char *s, *t;
+    const char *vname = NULL;
 
-	if (string == NULL)
-	{
-		return NULL;
-	}
+    if (volume_name == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
 
-	for (s = string; isspace (*s); s++)
-		;
+    if (!isalpha(volume_name[0]))
+        return 1;
 
-	if (*s == 0)
-		return s;
+    for (vname = &volume_name[1]; *vname != '\0'; vname++) {
+        if (!(isalnum(*vname) || *vname == '_'))
+            return 1;
+    }
 
-	t = s + strlen (s) - 1;
-	while (t > s && isspace (*t))
-		t--;
-	*++t = '\0';
+    return 0;
+}
+
+int
+gf_string2time(const char *str, uint32_t *n)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtol(str, &tail, 0);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (((tail[0] == '\0') || ((tail[0] == 's') && (tail[1] == '\0')) ||
+         ((tail[0] == 's') && (tail[1] == 'e') && (tail[2] == 'c') &&
+          (tail[3] == '\0'))))
+        goto out;
+
+    else if (((tail[0] == 'm') && (tail[1] == '\0')) ||
+             ((tail[0] == 'm') && (tail[1] == 'i') && (tail[2] == 'n') &&
+              (tail[3] == '\0'))) {
+        value = value * GF_MINUTE_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'h') && (tail[1] == '\0')) ||
+             ((tail[0] == 'h') && (tail[1] == 'r') && (tail[2] == '\0'))) {
+        value = value * GF_HOUR_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'd') && (tail[1] == '\0')) ||
+             ((tail[0] == 'd') && (tail[1] == 'a') && (tail[2] == 'y') &&
+              (tail[3] == 's') && (tail[4] == '\0'))) {
+        value = value * GF_DAY_IN_SECONDS;
+        goto out;
+    }
+
+    else if (((tail[0] == 'w') && (tail[1] == '\0')) ||
+             ((tail[0] == 'w') && (tail[1] == 'k') && (tail[2] == '\0'))) {
+        value = value * GF_WEEK_IN_SECONDS;
+        goto out;
+    } else {
+        return -1;
+    }
 
-	return s;
+out:
+    *n = value;
+
+    return 0;
 }
 
 int
-gf_strsplit (const char *str, const char *delim,
-	     char ***tokens, int *token_count)
+gf_string2percent(const char *str, double *n)
 {
-	char *_running = NULL;
-	char *running = NULL;
-	char *token = NULL;
-	char **token_list = NULL;
-	int count = 0;
-	int i = 0;
-	int j = 0;
+    double value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (!((tail[0] == '\0') || ((tail[0] == '%') && (tail[1] == '\0'))))
+        return -1;
+
+    *n = value;
+
+    return 0;
+}
 
-	if (str == NULL || delim == NULL || tokens == NULL || token_count == NULL)
-	{
-		return -1;
-	}
+static int
+_gf_string2long(const char *str, long *n, int base)
+{
+    long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-        _running = gf_strdup (str);
-	if (_running == NULL)
-	{
-		return -1;
-	}
-	running = _running;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-	while ((token = strsep (&running, delim)) != NULL)
-	{
-		if (token[0] != '\0')
-			count++;
-	}
-	GF_FREE (_running);
+    old_errno = errno;
+    errno = 0;
+    value = strtol(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
 
-        _running = gf_strdup (str);
-	if (_running == NULL)
-	{
-		return -1;
-	}
-	running = _running;
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
-	if ((token_list = GF_CALLOC (count, sizeof (char *),
-                                        gf_common_mt_char)) == NULL)
-	{
-		GF_FREE (_running);
-		return -1;
-	}
+    if (errno == 0)
+        errno = old_errno;
 
-	while ((token = strsep (&running, delim)) != NULL)
-	{
-		if (token[0] == '\0')
-			continue;
+    if (tail[0] != '\0')
+        return -1;
 
-                token_list[i] = gf_strdup (token);
-		if (token_list[i] == NULL)
-			goto free_exit;
-                i++;
-	}
+    *n = value;
 
-	GF_FREE (_running);
+    return 0;
+}
 
-	*tokens = token_list;
-	*token_count = count;
-	return 0;
+static int
+_gf_string2ulong(const char *str, unsigned long *n, int base)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoul(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (tail[0] != '\0')
+        return -1;
+
+    *n = value;
+
+    return 0;
+}
 
-free_exit:
-	GF_FREE (_running);
-	for (j = 0; j < i; j++)
-	{
-		GF_FREE (token_list[j]);
-	}
-	GF_FREE (token_list);
-	return -1;
+static int
+_gf_string2uint(const char *str, unsigned int *n, int base)
+{
+    unsigned long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoul(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (tail[0] != '\0')
+        return -1;
+
+    *n = (unsigned int)value;
+
+    return 0;
 }
 
-int
-gf_strstr (const char *str, const char *delim, const char *match)
+static int
+_gf_string2double(const char *str, double *n)
 {
-        char *tmp      = NULL;
-        char *save_ptr = NULL;
-        char *tmp_str  = NULL;
+    double value = 0.0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-        int  ret       = 0;
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        tmp_str = strdup (str);
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
 
-        if (str == NULL || delim == NULL || match == NULL || tmp_str == NULL) {
-                ret = -1;
-		goto out;
-	}
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
 
+    if (errno == 0)
+        errno = old_errno;
 
-        tmp = strtok_r (tmp_str, delim, &save_ptr);
+    if (tail[0] != '\0')
+        return -1;
 
-        while (tmp) {
-                ret = strcmp (tmp, match);
+    *n = value;
 
-                if (ret == 0)
-                        break;
+    return 0;
+}
 
-                tmp = strtok_r (NULL, delim, &save_ptr);
-        }
+static int
+_gf_string2longlong(const char *str, long long *n, int base)
+{
+    long long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
 
-out:
-        if (tmp_str)
-                free (tmp_str);
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
 
-        return ret;
+    old_errno = errno;
+    errno = 0;
+    value = strtoll(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (tail[0] != '\0')
+        return -1;
 
+    *n = value;
+
+    return 0;
+}
+
+static int
+_gf_string2ulonglong(const char *str, unsigned long long *n, int base)
+{
+    unsigned long long value = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtoull(str, &tail, base);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (tail[0] != '\0')
+        return -1;
+
+    *n = value;
+
+    return 0;
 }
 
 int
-gf_volume_name_validate (const char *volume_name)
+gf_string2long(const char *str, long *n)
 {
-	const char *vname = NULL;
+    return _gf_string2long(str, n, 0);
+}
 
-	if (volume_name == NULL)
-	{
-		return -1;
-	}
+int
+gf_string2ulong(const char *str, unsigned long *n)
+{
+    return _gf_string2ulong(str, n, 0);
+}
 
-	if (!isalpha (volume_name[0]))
-	{
-		return 1;
-	}
+int
+gf_string2int(const char *str, int *n)
+{
+    long l = 0;
+    int ret = 0;
 
-	for (vname = &volume_name[1]; *vname != '\0'; vname++)
-	{
-		if (!(isalnum (*vname) || *vname == '_'))
-			return 1;
-	}
+    ret = _gf_string2long(str, &l, 0);
 
-	return 0;
+    *n = l;
+    return ret;
 }
 
+int
+gf_string2uint(const char *str, unsigned int *n)
+{
+    return _gf_string2uint(str, n, 0);
+}
 
 int
-gf_string2time (const char *str, uint32_t *n)
+gf_string2double(const char *str, double *n)
 {
-	unsigned long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
+    return _gf_string2double(str, n);
+}
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+int
+gf_string2longlong(const char *str, long long *n)
+{
+    return _gf_string2longlong(str, n, 0);
+}
 
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			return -1;
-		}
-		break;
-	}
+int
+gf_string2ulonglong(const char *str, unsigned long long *n)
+{
+    return _gf_string2ulonglong(str, n, 0);
+}
 
-	old_errno = errno;
-	errno = 0;
-	value = strtol (str, &tail, 0);
+int
+gf_string2int8(const char *str, int8_t *n)
+{
+    long l = 0L;
+    int rv = 0;
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
+    if ((l >= INT8_MIN) && (l <= INT8_MAX)) {
+        *n = (int8_t)l;
+        return 0;
+    }
 
-	if (!((tail[0] == '\0') ||
-	      ((tail[0] == 's') && (tail[1] == '\0')) ||
-	      ((tail[0] == 's') && (tail[1] == 'e') && (tail[2] == 'c') && (tail[3] == '\0'))))
-	{
-		return -1;
-	}
+    errno = ERANGE;
+    return -1;
+}
 
-	*n = value;
+int
+gf_string2int16(const char *str, int16_t *n)
+{
+    long l = 0L;
+    int rv = 0;
 
-	return 0;
-}
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
+    if ((l >= INT16_MIN) && (l <= INT16_MAX)) {
+        *n = (int16_t)l;
+        return 0;
+    }
+
+    errno = ERANGE;
+    return -1;
+}
 
 int
-gf_string2percent (const char *str, uint32_t *n)
+gf_string2int32(const char *str, int32_t *n)
 {
-	unsigned long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
+    long l = 0L;
+    int rv = 0;
+
+    rv = _gf_string2long(str, &l, 0);
+    if (rv != 0)
+        return rv;
+
+    if ((l >= INT32_MIN) && (l <= INT32_MAX)) {
+        *n = (int32_t)l;
+        return 0;
+    }
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+    errno = ERANGE;
+    return -1;
+}
+
+int
+gf_string2int64(const char *str, int64_t *n)
+{
+    long long l = 0LL;
+    int rv = 0;
 
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			return -1;
-		}
-		break;
-	}
+    rv = _gf_string2longlong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	old_errno = errno;
-	errno = 0;
-	value = strtol (str, &tail, 0);
+    *n = (int64_t)l;
+    return 0;
+}
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+int
+gf_string2uint8(const char *str, uint8_t *n)
+{
+    unsigned long l = 0L;
+    int rv = 0;
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	if (!((tail[0] == '\0') ||
-	      ((tail[0] == '%') && (tail[1] == '\0'))))
-	{
-		return -1;
-	}
+    if (l <= UINT8_MAX) {
+        *n = (uint8_t)l;
+        return 0;
+    }
 
-	*n = value;
-
-	return 0;
+    errno = ERANGE;
+    return -1;
 }
 
+int
+gf_string2uint16(const char *str, uint16_t *n)
+{
+    unsigned long l = 0L;
+    int rv = 0;
 
-static int
-_gf_string2long (const char *str, long *n, int base)
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
+
+    if (l <= UINT16_MAX) {
+        *n = (uint16_t)l;
+        return 0;
+    }
+
+    errno = ERANGE;
+    return -1;
+}
+
+int
+gf_string2uint32(const char *str, uint32_t *n)
 {
-	long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+    rv = _gf_string2ulong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	old_errno = errno;
-	errno = 0;
-	value = strtol (str, &tail, base);
+    if (l <= UINT32_MAX) {
+        *n = (uint32_t)l;
+        return 0;
+    }
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+    errno = ERANGE;
+    return -1;
+}
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
+int
+gf_string2uint64(const char *str, uint64_t *n)
+{
+    unsigned long long l = 0ULL;
+    int rv = 0;
 
-	if (tail[0] != '\0')
-	{
-		/* bala: invalid integer format */
-		return -1;
-	}
+    rv = _gf_string2ulonglong(str, &l, 0);
+    if (rv != 0)
+        return rv;
 
-	*n = value;
+    if (l <= UINT64_MAX) {
+        *n = (uint64_t)l;
+        return 0;
+    }
 
-	return 0;
+    errno = ERANGE;
+    return -1;
 }
 
-static int
-_gf_string2ulong (const char *str, unsigned long *n, int base)
-{
-	unsigned long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
-
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
-
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			/* bala: we do not support suffixed (-) sign and
-			   invalid integer format */
-			return -1;
-		}
-		break;
-	}
-
-	old_errno = errno;
-	errno = 0;
-	value = strtoul (str, &tail, base);
-
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
-
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
-
-	if (tail[0] != '\0')
-	{
-		/* bala: invalid integer format */
-		return -1;
-	}
-
-	*n = value;
-
-	return 0;
+int
+gf_string2ulong_base10(const char *str, unsigned long *n)
+{
+    return _gf_string2ulong(str, n, 10);
 }
 
-static int
-_gf_string2uint (const char *str, unsigned int *n, int base)
-{
-	unsigned long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
-
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
-
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			/* bala: we do not support suffixed (-) sign and
-			   invalid integer format */
-			return -1;
-		}
-		break;
-	}
-
-	old_errno = errno;
-	errno = 0;
-	value = strtoul (str, &tail, base);
-
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
-
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
-
-	if (tail[0] != '\0')
-	{
-		/* bala: invalid integer format */
-		return -1;
-	}
-
-	*n = (unsigned int)value;
-
-	return 0;
+int
+gf_string2uint_base10(const char *str, unsigned int *n)
+{
+    return _gf_string2uint(str, n, 10);
 }
 
-static int
-_gf_string2double (const char *str, double *n)
+int
+gf_string2uint8_base10(const char *str, uint8_t *n)
 {
-	double value     = 0.0;
-	char   *tail     = NULL;
-	int    old_errno = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-	if (str == NULL || n == NULL) {
-		errno = EINVAL;
-		return -1;
-	}
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-	old_errno = errno;
-	errno = 0;
-	value = strtod (str, &tail);
+    if (l <= UINT8_MAX) {
+        *n = (uint8_t)l;
+        return 0;
+    }
 
-	if (errno == ERANGE || errno == EINVAL)	{
-		return -1;
-	}
+    errno = ERANGE;
+    return -1;
+}
 
-	if (errno == 0)	{
-		errno = old_errno;
-	}
+int
+gf_string2uint16_base10(const char *str, uint16_t *n)
+{
+    unsigned long l = 0L;
+    int rv = 0;
 
-	if (tail[0] != '\0') {
-		return -1;
-	}
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-	*n = value;
+    if (l <= UINT16_MAX) {
+        *n = (uint16_t)l;
+        return 0;
+    }
 
-	return 0;
+    errno = ERANGE;
+    return -1;
 }
 
-static int
-_gf_string2longlong (const char *str, long long *n, int base)
+int
+gf_string2uint32_base10(const char *str, uint32_t *n)
 {
-	long long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
+    unsigned long l = 0L;
+    int rv = 0;
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+    rv = _gf_string2ulong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-	old_errno = errno;
-	errno = 0;
-	value = strtoll (str, &tail, base);
+    if (l <= UINT32_MAX) {
+        *n = (uint32_t)l;
+        return 0;
+    }
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+    errno = ERANGE;
+    return -1;
+}
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
+int
+gf_string2uint64_base10(const char *str, uint64_t *n)
+{
+    unsigned long long l = 0ULL;
+    int rv = 0;
 
-	if (tail[0] != '\0')
-	{
-		/* bala: invalid integer format */
-		return -1;
-	}
+    rv = _gf_string2ulonglong(str, &l, 10);
+    if (rv != 0)
+        return rv;
 
-	*n = value;
+    if (l <= UINT64_MAX) {
+        *n = (uint64_t)l;
+        return 0;
+    }
 
-	return 0;
+    errno = ERANGE;
+    return -1;
 }
 
-static int
-_gf_string2ulonglong (const char *str, unsigned long long *n, int base)
+char *
+gf_uint64_2human_readable(uint64_t n)
 {
-	unsigned long long value = 0;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
+    int ret = 0;
+    char *str = NULL;
+
+    if (n >= GF_UNIT_PB) {
+        ret = gf_asprintf(&str, "%.1lfPB", ((double)n) / GF_UNIT_PB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_TB) {
+        ret = gf_asprintf(&str, "%.1lfTB", ((double)n) / GF_UNIT_TB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_GB) {
+        ret = gf_asprintf(&str, "%.1lfGB", ((double)n) / GF_UNIT_GB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_MB) {
+        ret = gf_asprintf(&str, "%.1lfMB", ((double)n) / GF_UNIT_MB);
+        if (ret < 0)
+            goto err;
+    } else if (n >= GF_UNIT_KB) {
+        ret = gf_asprintf(&str, "%.1lfKB", ((double)n) / GF_UNIT_KB);
+        if (ret < 0)
+            goto err;
+    } else {
+        ret = gf_asprintf(&str, "%" PRIu64 "Bytes", n);
+        if (ret < 0)
+            goto err;
+    }
+    return str;
+err:
+    return NULL;
+}
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+int
+gf_string2bytesize_range(const char *str, uint64_t *n, uint64_t umax)
+{
+    double value = 0.0;
+    int64_t int_value = 0;
+    uint64_t unit = 0;
+    int64_t max = 0;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+    gf_boolean_t fraction = _gf_false;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    max = umax & 0x7fffffffffffffffLL;
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    if (strrchr(str, '.'))
+        fraction = _gf_true;
+
+    old_errno = errno;
+    errno = 0;
+    if (fraction)
+        value = strtod(str, &tail);
+    else
+        int_value = strtoll(str, &tail, 10);
+
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, GF_UNIT_KB_STRING) == 0)
+            unit = GF_UNIT_KB;
+        else if (strcasecmp(tail, GF_UNIT_MB_STRING) == 0)
+            unit = GF_UNIT_MB;
+        else if (strcasecmp(tail, GF_UNIT_GB_STRING) == 0)
+            unit = GF_UNIT_GB;
+        else if (strcasecmp(tail, GF_UNIT_TB_STRING) == 0)
+            unit = GF_UNIT_TB;
+        else if (strcasecmp(tail, GF_UNIT_PB_STRING) == 0)
+            unit = GF_UNIT_PB;
+        else if (strcasecmp(tail, GF_UNIT_B_STRING) != 0)
+            return -1;
+
+        if (unit > 0) {
+            if (fraction)
+                value *= unit;
+            else
+                int_value *= unit;
+        }
+    }
 
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			/* bala: we do not support suffixed (-) sign and
-			   invalid integer format */
-			return -1;
-		}
-		break;
-	}
+    if (fraction) {
+        if ((max - value) < 0) {
+            errno = ERANGE;
+            return -1;
+        }
+        *n = (uint64_t)value;
+    } else {
+        if ((max - int_value) < 0) {
+            errno = ERANGE;
+            return -1;
+        }
+        *n = int_value;
+    }
 
-	old_errno = errno;
-	errno = 0;
-	value = strtoull (str, &tail, base);
+    return 0;
+}
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+int
+gf_string2bytesize_uint64(const char *str, uint64_t *n)
+{
+    return gf_string2bytesize_range(str, n, UINT64_MAX);
+}
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
+int
+gf_string2bytesize_int64(const char *str, int64_t *n)
+{
+    uint64_t u64 = 0;
+    int ret = 0;
 
-	if (tail[0] != '\0')
-	{
-		/* bala: invalid integer format */
-		return -1;
-	}
+    ret = gf_string2bytesize_range(str, &u64, INT64_MAX);
+    *n = (int64_t)u64;
+    return ret;
+}
 
-	*n = value;
+int
+gf_string2percent_or_bytesize(const char *str, double *n,
+                              gf_boolean_t *is_percent)
+{
+    double value = 0ULL;
+    char *tail = NULL;
+    int old_errno = 0;
+    const char *s = NULL;
+
+    if (str == NULL || n == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        errno = EINVAL;
+        return -1;
+    }
+
+    for (s = str; *s != '\0'; s++) {
+        if (isspace(*s))
+            continue;
+        if (*s == '-')
+            return -1;
+        break;
+    }
+
+    old_errno = errno;
+    errno = 0;
+    value = strtod(str, &tail);
+    if (str == tail)
+        errno = EINVAL;
+
+    if (errno == ERANGE || errno == EINVAL)
+        return -1;
+
+    if (errno == 0)
+        errno = old_errno;
+
+    /*Maximum accepted value for 64 bit OS will be (2^14 -1)PB*/
+    if (tail[0] != '\0') {
+        if (strcasecmp(tail, GF_UNIT_KB_STRING) == 0)
+            value *= GF_UNIT_KB;
+        else if (strcasecmp(tail, GF_UNIT_MB_STRING) == 0)
+            value *= GF_UNIT_MB;
+        else if (strcasecmp(tail, GF_UNIT_GB_STRING) == 0)
+            value *= GF_UNIT_GB;
+        else if (strcasecmp(tail, GF_UNIT_TB_STRING) == 0)
+            value *= GF_UNIT_TB;
+        else if (strcasecmp(tail, GF_UNIT_PB_STRING) == 0)
+            value *= GF_UNIT_PB;
+        else if (strcasecmp(tail, GF_UNIT_PERCENT_STRING) == 0)
+            *is_percent = _gf_true;
+        else
+            return -1;
+    }
+
+    /* Error out if we cannot store the value in uint64 */
+    if ((UINT64_MAX - value) < 0) {
+        errno = ERANGE;
+        return -1;
+    }
+
+    *n = value;
+
+    return 0;
+}
 
-	return 0;
+int64_t
+gf_str_to_long_long(const char *number)
+{
+    int64_t unit = 1;
+    int64_t ret = 0;
+    char *endptr = NULL;
+    if (!number)
+        return 0;
+
+    ret = strtoll(number, &endptr, 0);
+
+    if (endptr) {
+        switch (*endptr) {
+            case 'G':
+            case 'g':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024 * 1024 * 1024;
+                break;
+            case 'M':
+            case 'm':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024 * 1024;
+                break;
+            case 'K':
+            case 'k':
+                if ((*(endptr + 1) == 'B') || (*(endptr + 1) == 'b'))
+                    unit = 1024;
+                break;
+            case '%':
+                unit = 1;
+                break;
+            default:
+                unit = 1;
+                break;
+        }
+    }
+    return ret * unit;
 }
 
 int
-gf_string2long (const char *str, long *n)
+gf_string2boolean(const char *str, gf_boolean_t *b)
 {
-	return _gf_string2long (str, n, 0);
+    if (str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
+
+    if ((strcasecmp(str, "1") == 0) || (strcasecmp(str, "on") == 0) ||
+        (strcasecmp(str, "yes") == 0) || (strcasecmp(str, "true") == 0) ||
+        (strcasecmp(str, "enable") == 0)) {
+        *b = _gf_true;
+        return 0;
+    }
+
+    if ((strcasecmp(str, "0") == 0) || (strcasecmp(str, "off") == 0) ||
+        (strcasecmp(str, "no") == 0) || (strcasecmp(str, "false") == 0) ||
+        (strcasecmp(str, "disable") == 0)) {
+        *b = _gf_false;
+        return 0;
+    }
+
+    return -1;
 }
 
 int
-gf_string2ulong (const char *str, unsigned long *n)
+gf_strn2boolean(const char *str, const int len, gf_boolean_t *b)
 {
-	return _gf_string2ulong (str, n, 0);
+    if (str == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return -1;
+    }
+
+    switch (len) {
+        case 1:
+            if (strcasecmp(str, "1") == 0) {
+                *b = _gf_true;
+                return 0;
+            } else if (strcasecmp(str, "0") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 2:
+            if (strcasecmp(str, "on") == 0) {
+                *b = _gf_true;
+                return 0;
+            } else if (strcasecmp(str, "no") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 3:
+            if (strcasecmp(str, "yes") == 0) {
+                *b = _gf_true;
+                return 0;
+            } else if (strcasecmp(str, "off") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 4:
+            if (strcasecmp(str, "true") == 0) {
+                *b = _gf_true;
+                return 0;
+            }
+            break;
+        case 5:
+            if (strcasecmp(str, "false") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        case 6:
+            if (strcasecmp(str, "enable") == 0) {
+                *b = _gf_true;
+                return 0;
+            }
+            break;
+        case 7:
+            if (strcasecmp(str, "disable") == 0) {
+                *b = _gf_false;
+                return 0;
+            }
+            break;
+        default:
+            return -1;
+            break;
+    }
+    return -1;
 }
 
 int
-gf_string2int (const char *str, int *n)
+gf_lockfd(int fd)
 {
-	long l = 0;
-	int  ret = 0;
+    struct gf_flock fl;
 
-	ret = _gf_string2long (str, &l, 0);
+    fl.l_type = F_WRLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 0;
+    fl.l_len = 0;
 
-	*n = l;
-	return ret;
+    return fcntl(fd, F_SETLK, &fl);
 }
 
 int
-gf_string2uint (const char *str, unsigned int *n)
+gf_unlockfd(int fd)
 {
-	return _gf_string2uint (str, n, 0);
+    struct gf_flock fl;
+
+    fl.l_type = F_UNLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 0;
+    fl.l_len = 0;
+
+    return fcntl(fd, F_SETLK, &fl);
 }
 
-int
-gf_string2double (const char *str, double *n)
+static void
+compute_checksum(char *buf, const ssize_t size, uint32_t *checksum)
 {
-	return _gf_string2double (str, n);
+    int ret = -1;
+    char *checksum_buf = NULL;
+
+    checksum_buf = (char *)(checksum);
+
+    if (!(*checksum)) {
+        checksum_buf[0] = 0xba;
+        checksum_buf[1] = 0xbe;
+        checksum_buf[2] = 0xb0;
+        checksum_buf[3] = 0x0b;
+    }
+
+    for (ret = 0; ret < (size - 4); ret += 4) {
+        checksum_buf[0] ^= (buf[ret]);
+        checksum_buf[1] ^= (buf[ret + 1] << 1);
+        checksum_buf[2] ^= (buf[ret + 2] << 2);
+        checksum_buf[3] ^= (buf[ret + 3] << 3);
+    }
+
+    for (ret = 0; ret <= (size % 4); ret++) {
+        checksum_buf[ret] ^= (buf[(size - 4) + ret] << ret);
+    }
+
+    return;
 }
 
+#define GF_CHECKSUM_BUF_SIZE 1024
+
 int
-gf_string2longlong (const char *str, long long *n)
+get_checksum_for_file(int fd, uint32_t *checksum, int op_version)
 {
-	return _gf_string2longlong (str, n, 0);
+    int ret = -1;
+    char buf[GF_CHECKSUM_BUF_SIZE] = {
+        0,
+    };
+
+    /* goto first place */
+    sys_lseek(fd, 0L, SEEK_SET);
+    do {
+        ret = sys_read(fd, &buf, GF_CHECKSUM_BUF_SIZE);
+        if (ret > 0) {
+            if (op_version < GD_OP_VERSION_5_4)
+                compute_checksum(buf, GF_CHECKSUM_BUF_SIZE, checksum);
+            else
+                compute_checksum(buf, ret, checksum);
+        }
+    } while (ret > 0);
+
+    /* set it back */
+    sys_lseek(fd, 0L, SEEK_SET);
+
+    return ret;
 }
 
 int
-gf_string2ulonglong (const char *str, unsigned long long *n)
+get_checksum_for_path(char *path, uint32_t *checksum, int op_version)
 {
-	return _gf_string2ulonglong (str, n, 0);
+    int ret = -1;
+    int fd = -1;
+
+    GF_ASSERT(path);
+    GF_ASSERT(checksum);
+
+    fd = open(path, O_RDWR);
+
+    if (fd == -1) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, LG_MSG_PATH_OPEN_FAILED,
+                "path=%s", path, NULL);
+        goto out;
+    }
+
+    ret = get_checksum_for_file(fd, checksum, op_version);
+
+out:
+    if (fd != -1)
+        sys_close(fd);
+
+    return ret;
 }
 
+/**
+ * get_file_mtime -- Given a path, get the mtime for the file
+ *
+ * @path: The filepath to check the mtime on
+ * @stamp: The parameter to set after we get the mtime
+ *
+ * @returns: success: 0
+ *           errors : Errors returned by the stat () call
+ */
 int
-gf_string2int8 (const char *str, int8_t *n)
+get_file_mtime(const char *path, time_t *stamp)
 {
-	long l = 0L;
-	int rv = 0;
+    struct stat f_stat = {0};
+    int ret = -EINVAL;
 
-	rv = _gf_string2long (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    GF_VALIDATE_OR_GOTO(THIS->name, path, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, stamp, out);
 
-	if (l >= INT8_MIN && l <= INT8_MAX)
-	{
-		*n = (int8_t) l;
-		return 0;
-	}
+    ret = sys_stat(path, &f_stat);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, LG_MSG_FILE_STAT_FAILED,
+                "path=%s", path, NULL);
+        goto out;
+    }
 
-	errno = ERANGE;
-	return -1;
+    /* Set the mtime */
+    *stamp = f_stat.st_mtime;
+out:
+    return ret;
 }
 
-int
-gf_string2int16 (const char *str, int16_t *n)
+/**
+ * gf_is_ip_in_net -- Checks if an IP Address is in a network.
+ *                    A network should be specified by something like
+ *                    '10.5.153.0/24' (in CIDR notation).
+ *
+ * @result : Sets to true if the IP is in the network
+ * @ip_str : The IP to check
+ * @network: The network to check the IP against.
+ *
+ * @return: success: _gf_true
+ *          failure: -EINVAL for bad args, retval of inet_pton otherwise
+ */
+gf_boolean_t
+gf_is_ip_in_net(const char *network, const char *ip_str)
 {
-	long l = 0L;
-	int rv = 0;
+    unsigned long ip_buf = 0;
+    unsigned long net_ip_buf = 0;
+    unsigned long subnet_mask = 0;
+    int ret = -EINVAL;
+    char *slash = NULL;
+    char *net_ip = NULL;
+    char *subnet = NULL;
+    char *net_str = NULL;
+    int family = AF_INET;
+    gf_boolean_t result = _gf_false;
+
+    GF_ASSERT(network);
+    GF_ASSERT(ip_str);
+
+    if (strchr(network, ':'))
+        family = AF_INET6;
+    else if (strchr(network, '.'))
+        family = AF_INET;
+    else {
+        goto out;
+    }
+
+    net_str = strdupa(network);
+    slash = strchr(net_str, '/');
+    if (!slash)
+        goto out;
+    *slash = '\0';
+
+    subnet = slash + 1;
+    net_ip = net_str;
+
+    /* Convert IP address to a long */
+    ret = inet_pton(family, ip_str, &ip_buf);
+    if (ret < 0)
+        gf_smsg("common-utils", GF_LOG_ERROR, errno, LG_MSG_INET_PTON_FAILED,
+                NULL);
+
+    /* Convert network IP address to a long */
+    ret = inet_pton(family, net_ip, &net_ip_buf);
+    if (ret < 0) {
+        gf_smsg("common-utils", GF_LOG_ERROR, errno, LG_MSG_INET_PTON_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* Converts /x into a mask */
+    subnet_mask = (1 << atoi(subnet)) - 1;
+
+    result = ((ip_buf & subnet_mask) == (net_ip_buf & subnet_mask));
+out:
+    return result;
+}
+
+char *
+strtail(char *str, const char *pattern)
+{
+    int i = 0;
+
+    for (i = 0; str[i] == pattern[i] && str[i]; i++)
+        ;
+
+    if (pattern[i] == '\0')
+        return str + i;
+
+    return NULL;
+}
+
+void
+skipwhite(char **s)
+{
+    while (isspace(**s))
+        (*s)++;
+}
+
+void
+gf_strTrim(char **s)
+{
+    char *end = NULL;
+
+    end = *s + strlen(*s) - 1;
+    while (end > *s && isspace((unsigned char)*end))
+        end--;
 
-	rv = _gf_string2long (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    *(end + 1) = '\0';
 
-	if (l >= INT16_MIN && l <= INT16_MAX)
-	{
-		*n = (int16_t) l;
-		return 0;
-	}
+    while (isspace(**s))
+        (*s)++;
 
-	errno = ERANGE;
-	return -1;
+    return;
+}
+
+char *
+nwstrtail(char *str, char *pattern)
+{
+    for (;;) {
+        skipwhite(&str);
+        skipwhite(&pattern);
+
+        if (*str != *pattern || !*str)
+            break;
+
+        str++;
+        pattern++;
+    }
+
+    return *pattern ? NULL : str;
+}
+
+/**
+ * token_iter_init -- initialize tokenization
+ *
+ * @str: string to be tokenized
+ * @sep: token separator character
+ * @tit: pointer to iteration state
+ *
+ * @return: token string
+ *
+ * The returned token string and tit are
+ * not to be used directly, but through
+ * next_token().
+ */
+char *
+token_iter_init(char *str, char sep, token_iter_t *tit)
+{
+    tit->end = str + strlen(str);
+    tit->sep = sep;
+
+    return str;
+}
+
+/**
+ * next_token -- fetch next token in tokenization
+ * inited by token_iter_init().
+ *
+ * @tokenp: pointer to token
+ * @tit:    pointer to iteration state
+ *
+ * @return: true if iteration ends, else false
+ *
+ * The token pointed by @tokenp can be used
+ * after a call to next_token(). When next_token()
+ * returns true the iteration is to be stopped
+ * and the string with which the tokenization
+ * was inited (see token_iter_init() is restored,
+ * apart from dropped tokens (see drop_token()).
+ */
+gf_boolean_t
+next_token(char **tokenp, token_iter_t *tit)
+{
+    char *cursor = NULL;
+    gf_boolean_t is_last = _gf_false;
+
+    for (cursor = *tokenp; *cursor; cursor++)
+        ;
+    if (cursor < tit->end) {
+        /*
+         * We detect that in between current token and end a zero
+         * marker has already been inserted. This means that the
+         * token has already been returned. We restore the
+         * separator and move ahead.
+         */
+        *cursor = tit->sep;
+        *tokenp = cursor + 1;
+    }
+
+    for (cursor = *tokenp; *cursor && *cursor != tit->sep; cursor++)
+        ;
+    /* If the cursor ended up on a zero byte, then it's the last token. */
+    is_last = !*cursor;
+    /* Zero-terminate the token. */
+    *cursor = 0;
+
+    return is_last;
+}
+
+/*
+ * drop_token -- drop a token during iterated calls of next_token().
+ *
+ * Sample program that uses these functions to tokenize
+ * a comma-separated first argument while dropping the
+ * rest of the arguments if they occur as token:
+ *
+ * #include <stdio.h>
+ * #include <stdlib.h>
+ * #include <string.h>
+ * #include "glusterfs/common-utils.h"
+ *
+ * int
+ * main (int argc, char **argv)
+ * {
+ *         char *buf;
+ *         char *token;
+ *         token_iter_t tit;
+ *         int i;
+ *         gf_boolean_t iter_end;
+ *
+ *         if (argc <= 1)
+ *                 abort();
+ *
+ *         buf = strdup (argv[1]);
+ *         if (!buf)
+ *                 abort();
+ *
+ *         for (token = token_iter_init (buf, ',', &tit) ;;) {
+ *                 iter_end = next_token (&token, &tit);
+ *                 printf("found token: '%s'\n", token);
+ *                 for (i = 2; i < argc; i++) {
+ *                         if (strcmp (argv[i], token) == 0) {
+ *                                 printf ("%s\n", "dropping token!");
+ *                                 drop_token (token, &tit);
+ *                                 break;
+ *                          }
+ *                 }
+ *                 if (iter_end)
+ *                         break;
+ *         }
+ *
+ *         printf ("finally: '%s'\n", buf);
+ *
+ *         return 0;
+ * }
+ */
+void
+drop_token(char *token, token_iter_t *tit)
+{
+    char *cursor = NULL;
+
+    for (cursor = token; *cursor; cursor++)
+        ;
+    if (cursor < tit->end) {
+        /*
+         * We detect a zero inserted by next_token().
+         * Step the cursor and copy what comes after
+         * to token.
+         */
+        for (cursor++; cursor < tit->end; *token++ = *cursor++)
+            ;
+    }
+
+    /*
+     * Zero out the remainder of the buffer.
+     * It would be enough to insert just a single zero,
+     * but we continue 'till the end to have cleaner
+     * memory content.
+     */
+    for (cursor = token; cursor < tit->end; *cursor++ = 0)
+        ;
+
+    /* Adjust the end to point to the new terminating zero. */
+    tit->end = token;
+}
+
+/* Syntax formed according to RFC 1912 (RFC 1123 & 952 are more restrictive)  *
+   <hname> ::= <gen-name>*["."<gen-name>]                                     *
+   <gen-name> ::= <let-or-digit> <[*[<let-or-digit-or-hyphen>]<let-or-digit>] */
+char
+valid_host_name(char *address, int length)
+{
+    int i = 0;
+    int str_len = 0;
+    char ret = 1;
+    char *dup_addr = NULL;
+    char *temp_str = NULL;
+    char *save_ptr = NULL;
+
+    if ((length > _POSIX_HOST_NAME_MAX) || (length < 1)) {
+        ret = 0;
+        goto out;
+    }
+
+    dup_addr = gf_strdup(address);
+    if (!dup_addr) {
+        ret = 0;
+        goto out;
+    }
+
+    if (!isalnum(dup_addr[length - 1]) && (dup_addr[length - 1] != '*')) {
+        ret = 0;
+        goto out;
+    }
+
+    /* Check for consecutive dots, which is invalid in a hostname and is
+     * ignored by strtok()
+     */
+    if (strstr(dup_addr, "..")) {
+        ret = 0;
+        goto out;
+    }
+
+    /* gen-name */
+    temp_str = strtok_r(dup_addr, ".", &save_ptr);
+    do {
+        str_len = strlen(temp_str);
+
+        if (!isalnum(temp_str[0]) || !isalnum(temp_str[str_len - 1])) {
+            ret = 0;
+            goto out;
+        }
+        for (i = 1; i < str_len; i++) {
+            if (!isalnum(temp_str[i]) && (temp_str[i] != '-')) {
+                ret = 0;
+                goto out;
+            }
+        }
+    } while ((temp_str = strtok_r(NULL, ".", &save_ptr)));
+
+out:
+    GF_FREE(dup_addr);
+    return ret;
+}
+
+/*  Matches all ipv4 address, if wildcard_acc is true  '*' wildcard pattern for*
+  subnets is considered as valid strings as well */
+char
+valid_ipv4_address(char *address, int length, gf_boolean_t wildcard_acc)
+{
+    int octets = 0;
+    int value = 0;
+    char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
+    char ret = 1;
+    int is_wildcard = 0;
+
+    tmp = gf_strdup(address);
+
+    /*
+     * To prevent cases where last character is '.' and which have
+     * consecutive dots like ".." as strtok ignore consecutive
+     * delimiters.
+     */
+    if (length <= 0 || (strstr(address, "..")) ||
+        (!isdigit(tmp[length - 1]) && (tmp[length - 1] != '*'))) {
+        ret = 0;
+        goto out;
+    }
+
+    prev = strtok_r(tmp, ".", &ptr);
+
+    while (prev != NULL) {
+        octets++;
+        if (wildcard_acc && !strcmp(prev, "*")) {
+            is_wildcard = 1;
+        } else {
+            value = strtol(prev, &endptr, 10);
+            if ((value > 255) || (value < 0) ||
+                (endptr != NULL && *endptr != '\0')) {
+                ret = 0;
+                goto out;
+            }
+        }
+        prev = strtok_r(NULL, ".", &ptr);
+    }
+
+    if ((octets > 4) || (octets < 4 && !is_wildcard)) {
+        ret = 0;
+    }
+
+out:
+    GF_FREE(tmp);
+    return ret;
+}
+
+char
+valid_cidr_address(char *cidr_address, gf_boolean_t wildcard_acc)
+{
+    unsigned int net_mask = 0, len = 0;
+    char *temp = NULL, *cidr_str = NULL, ret = 1;
+
+    cidr_str = strdupa(cidr_address);
+    temp = strstr(cidr_str, "/");
+    if (temp == NULL)
+        return 0; /* Since Invalid cidr ip address we return 0 */
+
+    *temp = '\0';
+    temp++;
+    net_mask = (unsigned int)atoi(temp);
+
+    if (net_mask > 32 || net_mask < 1)
+        return 0; /* Since Invalid cidr ip address we return 0*/
+
+    len = strlen(cidr_str);
+
+    ret = valid_ipv4_address(cidr_str, len, wildcard_acc);
+
+    return ret;
+}
+
+/**
+ * valid_ipv4_subnetwork() takes the pattern and checks if it contains
+ * a valid ipv4 subnetwork pattern i.e. xx.xx.xx.xx/n. IPv4 address
+ * part (xx.xx.xx.xx) and mask bits length part (n). The mask bits length
+ * must be in 0-32 range (ipv4 addr is 32 bit). The pattern must be
+ * in this format.
+ *
+ * Returns _gf_true if both IP addr and mask bits len are valid
+ *         _gf_false otherwise.
+ */
+gf_boolean_t
+valid_ipv4_subnetwork(const char *address)
+{
+    char *slash = NULL;
+    char *paddr = NULL;
+    char *endptr = NULL;
+    long prefixlen = -1;
+    gf_boolean_t retv = _gf_true;
+
+    if (address == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return _gf_false;
+    }
+
+    paddr = gf_strdup(address);
+    if (paddr == NULL) /* ENOMEM */
+        return _gf_false;
+
+    /*
+     * INVALID: If '/' is not present OR
+     *          Nothing specified after '/'
+     */
+    slash = strchr(paddr, '/');
+    if ((slash == NULL) || (slash[1] == '\0')) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 "
+                         "subnetwork format");
+        retv = _gf_false;
+        goto out;
+    }
+
+    *slash = '\0';
+    retv = valid_ipv4_address(paddr, strlen(paddr), _gf_false);
+    if (retv == _gf_false) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 subnetwork address");
+        goto out;
+    }
+    /*
+     * Reset errno before checking it
+     */
+    errno = 0;
+    prefixlen = strtol(slash + 1, &endptr, 10);
+    if ((errno != 0) || (*endptr != '\0') || (prefixlen < 0) ||
+        (prefixlen > IPv4_ADDR_SIZE)) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INVALID_IPV4_FORMAT,
+                         "Invalid IPv4 subnetwork mask");
+        retv = _gf_false;
+        goto out;
+    }
+
+    retv = _gf_true;
+out:
+    GF_FREE(paddr);
+    return retv;
+}
+
+char
+valid_ipv6_address(char *address, int length, gf_boolean_t wildcard_acc)
+{
+    int hex_numbers = 0;
+    int value = 0;
+    int i = 0;
+    char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
+    char ret = 1;
+    int is_wildcard = 0;
+    int is_compressed = 0;
+
+    tmp = gf_strdup(address);
+
+    /* Check for '%' for link local addresses */
+    endptr = strchr(tmp, '%');
+    if (endptr) {
+        *endptr = '\0';
+        length = strlen(tmp);
+        endptr = NULL;
+    }
+
+    /* Check for compressed form */
+    if (length <= 0 || tmp[length - 1] == ':') {
+        ret = 0;
+        goto out;
+    }
+    for (i = 0; i < (length - 1); i++) {
+        if (tmp[i] == ':' && tmp[i + 1] == ':') {
+            if (is_compressed == 0)
+                is_compressed = 1;
+            else {
+                ret = 0;
+                goto out;
+            }
+        }
+    }
+
+    prev = strtok_r(tmp, ":", &ptr);
+
+    while (prev != NULL) {
+        hex_numbers++;
+        if (wildcard_acc && !strcmp(prev, "*")) {
+            is_wildcard = 1;
+        } else {
+            value = strtol(prev, &endptr, 16);
+            if ((value > 0xffff) || (value < 0) ||
+                (endptr != NULL && *endptr != '\0')) {
+                ret = 0;
+                goto out;
+            }
+        }
+        prev = strtok_r(NULL, ":", &ptr);
+    }
+
+    if ((hex_numbers > 8) ||
+        (hex_numbers < 8 && !is_wildcard && !is_compressed)) {
+        ret = 0;
+    }
+
+out:
+    GF_FREE(tmp);
+    return ret;
+}
+
+char
+valid_internet_address(char *address, gf_boolean_t wildcard_acc,
+                       gf_boolean_t cidr)
+{
+    char ret = 0;
+    int length = 0;
+
+    if (address == NULL) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        goto out;
+    }
+
+    length = strlen(address);
+    if (length == 0)
+        goto out;
+
+    if (cidr && valid_cidr_address(address, wildcard_acc)) {
+        ret = 1;
+    }
+
+    if (valid_ipv4_address(address, length, wildcard_acc) ||
+        valid_ipv6_address(address, length, wildcard_acc) ||
+        valid_host_name(address, length))
+        ret = 1;
+
+out:
+    return ret;
+}
+
+/**
+ * valid_mount_auth_address - Validate the rpc-auth.addr.allow/reject pattern
+ *
+ * @param address - Pattern to be validated
+ *
+ * @return _gf_true if "address" is "*" (anonymous) 'OR'
+ *                  if "address" is valid FQDN or valid IPv4/6 address 'OR'
+ *                  if "address" contains wildcard chars e.g. "'*' or '?' or
+ * '['" if "address" is valid ipv4 subnet pattern (xx.xx.xx.xx/n) _gf_false
+ * otherwise
+ *
+ *
+ * NB: If the user/admin set for wildcard pattern, then it does not have
+ *     to be validated. Make it similar to the way exportfs (kNFS) works.
+ */
+gf_boolean_t
+valid_mount_auth_address(char *address)
+{
+    int length = 0;
+    char *cp = NULL;
+
+    /* 1. Check for "NULL and empty string */
+    if ((address == NULL) || (address[0] == '\0')) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "argument invalid");
+        return _gf_false;
+    }
+
+    /* 2. Check for Anonymous */
+    if (strcmp(address, "*") == 0)
+        return _gf_true;
+
+    for (cp = address; *cp; cp++) {
+        /* 3. Check for wildcard pattern */
+        if (*cp == '*' || *cp == '?' || *cp == '[') {
+            return _gf_true;
+        }
+
+        /*
+         * 4. check for IPv4 subnetwork i.e. xx.xx.xx.xx/n
+         * TODO: check for IPv6 subnetwork
+         * NB: Wildcard must not be mixed with subnetwork.
+         */
+        if (*cp == '/') {
+            return valid_ipv4_subnetwork(address);
+        }
+    }
+
+    /* 5. Check for v4/v6 IP addr and FQDN/hostname */
+    length = strlen(address);
+    if ((valid_ipv4_address(address, length, _gf_false)) ||
+        (valid_ipv6_address(address, length, _gf_false)) ||
+        (valid_host_name(address, length))) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+/**
+ * gf_sock_union_equal_addr - check if two given gf_sock_unions have same addr
+ *
+ * @param a - first sock union
+ * @param b - second sock union
+ * @return _gf_true if a and b have same ipv{4,6} addr, _gf_false otherwise
+ */
+gf_boolean_t
+gf_sock_union_equal_addr(union gf_sock_union *a, union gf_sock_union *b)
+{
+    if (!a || !b) {
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
+                "gf_sock_union_equal_addr", NULL);
+        return _gf_false;
+    }
+
+    if (a->storage.ss_family != b->storage.ss_family)
+        return _gf_false;
+
+    switch (a->storage.ss_family) {
+        case AF_INET:
+            if (a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr)
+                return _gf_true;
+            else
+                return _gf_false;
+
+        case AF_INET6:
+            if (memcmp((void *)(&a->sin6.sin6_addr),
+                       (void *)(&b->sin6.sin6_addr), sizeof(a->sin6.sin6_addr)))
+                return _gf_false;
+            else
+                return _gf_true;
+
+        default:
+            gf_msg_debug("common-utils", 0,
+                         "Unsupported/invalid address "
+                         "family");
+            break;
+    }
+
+    return _gf_false;
+}
+
+/*
+ * Check if both have same network address.
+ * Extract the network address from the sockaddr(s) addr by applying the
+ * network mask. If they match, return boolean _gf_true, _gf_false otherwise.
+ *
+ * (x == y) <=> (x ^ y == 0)
+ * (x & y) ^ (x & z) <=> x & (y ^ z)
+ *
+ * ((ip1 & mask) == (ip2 & mask)) <=> ((mask & (ip1 ^ ip2)) == 0)
+ */
+gf_boolean_t
+mask_match(const uint32_t a, const uint32_t b, const uint32_t m)
+{
+    return (((a ^ b) & m) == 0);
+}
+
+/*Thread safe conversion function*/
+char *
+uuid_utoa(uuid_t uuid)
+{
+    char *uuid_buffer = glusterfs_uuid_buf_get();
+    gf_uuid_unparse(uuid, uuid_buffer);
+    return uuid_buffer;
+}
+
+/*Re-entrant conversion function*/
+char *
+uuid_utoa_r(uuid_t uuid, char *dst)
+{
+    if (!dst)
+        return NULL;
+    gf_uuid_unparse(uuid, dst);
+    return dst;
+}
+
+/*Thread safe conversion function*/
+char *
+lkowner_utoa(gf_lkowner_t *lkowner)
+{
+    char *lkowner_buffer = glusterfs_lkowner_buf_get();
+    lkowner_unparse(lkowner, lkowner_buffer, GF_LKOWNER_BUF_SIZE);
+    return lkowner_buffer;
+}
+
+/*Re-entrant conversion function*/
+char *
+lkowner_utoa_r(gf_lkowner_t *lkowner, char *dst, int len)
+{
+    if (!dst)
+        return NULL;
+    lkowner_unparse(lkowner, dst, len);
+    return dst;
+}
+
+gf_boolean_t
+is_valid_lease_id(const char *lease_id)
+{
+    int i = 0;
+    gf_boolean_t valid = _gf_false;
+
+    for (i = 0; i < LEASE_ID_SIZE; i++) {
+        if (lease_id[i] != 0) {
+            valid = _gf_true;
+            goto out;
+        }
+    }
+out:
+    return valid;
+}
+
+/* Lease_id can be a either in printable or non printable binary
+ * format. This function can be used to print any lease_id.
+ *
+ * This function returns a pointer to a buf, containing the ascii
+ * representation of the value in lease_id, in the following format:
+ * 4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum
+ *
+ * Eg: If lease_id = "lid1-clnt1" the printable string would be:
+ * 6c69-6431-2d63-6c6e-7431-0000-0000-0000
+ *
+ * Note: The pointer returned should not be stored for further use, as any
+ * subsequent call to this function will override the same buffer.
+ */
+char *
+leaseid_utoa(const char *lease_id)
+{
+    char *buf = NULL;
+    int i = 0;
+    int j = 0;
+
+    buf = glusterfs_leaseid_buf_get();
+    if (!buf)
+        goto out;
+
+    for (i = 0; i < LEASE_ID_SIZE; i++) {
+        if (i && !(i % 2)) {
+            buf[j] = '-';
+            j++;
+        }
+        sprintf(&buf[j], "%02hhx", lease_id[i]);
+        j += 2;
+        if (j == GF_LEASE_ID_BUF_SIZE)
+            break;
+    }
+    buf[GF_LEASE_ID_BUF_SIZE - 1] = '\0';
+out:
+    return buf;
+}
+
+char *
+gf_leaseid_get()
+{
+    return glusterfs_leaseid_buf_get();
+}
+
+char *
+gf_existing_leaseid()
+{
+    return glusterfs_leaseid_exist();
+}
+
+void *
+gf_array_elem(void *a, int index, size_t elem_size)
+{
+    uint8_t *ptr = a;
+    return (void *)(ptr + index * elem_size);
+}
+
+void
+gf_elem_swap(void *x, void *y, size_t l)
+{
+    uint8_t *a = x, *b = y, c;
+    while (l--) {
+        c = *a;
+        *a++ = *b;
+        *b++ = c;
+    }
+}
+
+void
+gf_array_insertionsort(void *A, int l, int r, size_t elem_size, gf_cmp cmp)
+{
+    int i = l;
+    int N = r + 1;
+    void *Temp = NULL;
+    int j = 0;
+
+    for (i = l; i < N; i++) {
+        Temp = gf_array_elem(A, i, elem_size);
+        j = i - 1;
+        while (j >= 0 && (cmp(Temp, gf_array_elem(A, j, elem_size)) < 0)) {
+            gf_elem_swap(Temp, gf_array_elem(A, j, elem_size), elem_size);
+            Temp = gf_array_elem(A, j, elem_size);
+            j = j - 1;
+        }
+    }
 }
 
 int
-gf_string2int32 (const char *str, int32_t *n)
+gf_is_str_int(const char *value)
 {
-	long l = 0L;
-	int rv = 0;
+    int flag = 0;
+    char *str = NULL;
+    char *fptr = NULL;
 
-	rv = _gf_string2long (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    GF_VALIDATE_OR_GOTO(THIS->name, value, out);
 
-	if (l >= INT32_MIN && l <= INT32_MAX)
-	{
-		*n = (int32_t) l;
-		return 0;
-	}
+    str = gf_strdup(value);
+    if (!str)
+        goto out;
+
+    fptr = str;
+
+    while (*str) {
+        if (!isdigit(*str)) {
+            flag = 1;
+            goto out;
+        }
+        str++;
+    }
 
-	errno = ERANGE;
-	return -1;
+out:
+    GF_FREE(fptr);
+
+    return flag;
+}
+/*
+ * rounds up nr to power of two. If nr is already a power of two, just returns
+ * nr
+ */
+
+int32_t
+gf_roundup_power_of_two(int32_t nr)
+{
+    int32_t result = 1;
+
+    if (nr < 0) {
+        gf_smsg("common-utils", GF_LOG_WARNING, 0, LG_MSG_NEGATIVE_NUM_PASSED,
+                NULL);
+        result = -1;
+        goto out;
+    }
+
+    while (result < nr)
+        result *= 2;
+
+out:
+    return result;
+}
+
+/*
+ * rounds up nr to next power of two. If nr is already a power of two, next
+ * power of two is returned.
+ */
+
+int32_t
+gf_roundup_next_power_of_two(int32_t nr)
+{
+    int32_t result = 1;
+
+    if (nr < 0) {
+        gf_smsg("common-utils", GF_LOG_WARNING, 0, LG_MSG_NEGATIVE_NUM_PASSED,
+                NULL);
+        result = -1;
+        goto out;
+    }
+
+    while (result <= nr)
+        result *= 2;
+
+out:
+    return result;
 }
 
 int
-gf_string2int64 (const char *str, int64_t *n)
+validate_brick_name(char *brick)
 {
-	long long l = 0LL;
-	int rv = 0;
+    char *delimiter = NULL;
+    int ret = 0;
+    delimiter = strrchr(brick, ':');
+    if (!delimiter || delimiter == brick || *(delimiter + 1) != '/')
+        ret = -1;
 
-	rv = _gf_string2longlong (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    return ret;
+}
 
-	if (l >= INT64_MIN && l <= INT64_MAX)
-	{
-		*n = (int64_t) l;
-		return 0;
-	}
+char *
+get_host_name(char *word, char **host)
+{
+    char *delimiter = NULL;
+    delimiter = strrchr(word, ':');
+    if (delimiter)
+        *delimiter = '\0';
+    else
+        return NULL;
+    *host = word;
+    return *host;
+}
 
-	errno = ERANGE;
-	return -1;
+char *
+get_path_name(char *word, char **path)
+{
+    char *delimiter = NULL;
+    delimiter = strchr(word, '/');
+    if (!delimiter)
+        return NULL;
+    *path = delimiter;
+    return *path;
+}
+
+void
+gf_path_strip_trailing_slashes(char *path)
+{
+    int i = 0;
+    int len = 0;
+
+    if (!path)
+        return;
+
+    len = strlen(path);
+    for (i = len - 1; i > 0; i--) {
+        if (path[i] != '/')
+            break;
+    }
+
+    if (i < (len - 1))
+        path[i + 1] = '\0';
+
+    return;
+}
+
+uint64_t
+get_mem_size()
+{
+    uint64_t memsize = -1;
+
+#if defined GF_LINUX_HOST_OS || defined GF_SOLARIS_HOST_OS
+
+    uint64_t page_size = 0;
+    uint64_t num_pages = 0;
+
+    page_size = sysconf(_SC_PAGESIZE);
+    num_pages = sysconf(_SC_PHYS_PAGES);
+
+    memsize = page_size * num_pages;
+#endif
+
+#if defined GF_DARWIN_HOST_OS || defined __FreeBSD__
+
+    size_t len = sizeof(memsize);
+    int name[] = {CTL_HW, HW_PHYSMEM};
+
+    sysctl(name, 2, &memsize, &len, NULL, 0);
+#endif
+
+#if defined __NetBSD__
+
+    size_t len = sizeof(memsize);
+    int name64[] = {CTL_HW, HW_PHYSMEM64};
+
+    sysctl(name64, 2, &memsize, &len, NULL, 0);
+    if (memsize == -1)
+        sysctl(name64, 2, &memsize, &len, NULL, 0);
+#endif
+    return memsize;
 }
 
+/* Strips all whitespace characters in a string and returns length of new string
+ * on success
+ */
 int
-gf_string2uint8 (const char *str, uint8_t *n)
+gf_strip_whitespace(char *str, int len)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    int i = 0;
+    int new_len = 0;
+    char *new_str = NULL;
 
-	rv = _gf_string2ulong (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    GF_ASSERT(str);
 
-	if (l >= 0 && l <= UINT8_MAX)
-	{
-		*n = (uint8_t) l;
-		return 0;
-	}
+    new_str = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (new_str == NULL)
+        return -1;
 
-	errno = ERANGE;
-	return -1;
+    for (i = 0; i < len; i++) {
+        if (!isspace(str[i]))
+            new_str[new_len++] = str[i];
+    }
+    new_str[new_len] = '\0';
+
+    if (new_len != len) {
+        snprintf(str, new_len + 1, "%s", new_str);
+    }
+
+    GF_FREE(new_str);
+    return new_len;
 }
 
 int
-gf_string2uint16 (const char *str, uint16_t *n)
+gf_canonicalize_path(char *path)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    int ret = -1;
+    int path_len = 0;
+    int dir_path_len = 0;
+    char *tmppath = NULL;
+    char *dir = NULL;
+    char *tmpstr = NULL;
+
+    if (!path || *path != '/')
+        goto out;
+
+    if (!strcmp(path, "/"))
+        return 0;
+
+    tmppath = gf_strdup(path);
+    if (!tmppath)
+        goto out;
+
+    /* Strip the extra slashes and return */
+    bzero(path, strlen(path));
+    path[0] = '/';
+    dir = strtok_r(tmppath, "/", &tmpstr);
+
+    while (dir) {
+        dir_path_len = strlen(dir);
+        memcpy((path + path_len + 1), dir, dir_path_len);
+        path_len += dir_path_len + 1;
+        dir = strtok_r(NULL, "/", &tmpstr);
+        if (dir) {
+            path[path_len] = '/';
+        }
+    }
+    path[path_len] = '\0';
+    ret = 0;
+
+out:
+    if (ret)
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_PATH_ERROR, NULL);
+
+    GF_FREE(tmppath);
+
+    return ret;
+}
 
-	rv = _gf_string2ulong (str, &l, 0);
-	if (rv != 0)
-		return rv;
+static const char *__gf_timefmts[] = {
+    "%F %T", "%Y/%m/%d-%T", "%b %d %T", "%F %H%M%S", "%Y-%m-%d-%T", "%s",
+};
 
-	if (l >= 0 && l <= UINT16_MAX)
-	{
-		*n = (uint16_t) l;
-		return 0;
-	}
+static const char *__gf_zerotimes[] = {
+    "0000-00-00 00:00:00", "0000/00/00-00:00:00", "xxx 00 00:00:00",
+    "0000-00-00 000000",   "0000-00-00-00:00:00", "0",
+};
 
-	errno = ERANGE;
-	return -1;
+void
+_gf_timestuff(const char ***fmts, const char ***zeros)
+{
+    *fmts = __gf_timefmts;
+    *zeros = __gf_zerotimes;
+}
+
+char *
+generate_glusterfs_ctx_id(void)
+{
+    uuid_t ctxid;
+    char *tmp = NULL;
+
+    gf_uuid_generate(ctxid);
+    tmp = uuid_utoa(ctxid);
+
+    return gf_strdup(tmp);
+}
+
+char *
+gf_get_reserved_ports()
+{
+    char *ports_info = NULL;
+#if defined GF_LINUX_HOST_OS
+    int proc_fd = -1;
+    char *proc_file = "/proc/sys/net/ipv4/ip_local_reserved_ports";
+    char buffer[4096] = {
+        0,
+    };
+    int32_t ret = -1;
+
+    proc_fd = open(proc_file, O_RDONLY);
+    if (proc_fd == -1) {
+        /* What should be done in this case? error out from here
+         * and thus stop the glusterfs process from starting or
+         * continue with older method of using any of the available
+         * port? For now 2nd option is considered.
+         */
+        gf_smsg("glusterfs", GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                " /proc/sys/net/ipv4/ip_local_reserved_ports", NULL);
+        goto out;
+    }
+
+    ret = sys_read(proc_fd, buffer, sizeof(buffer) - 1);
+    if (ret < 0) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                "file=%s", proc_file, NULL);
+        goto out;
+    }
+
+    buffer[ret] = '\0';
+    ports_info = gf_strdup(buffer);
+
+out:
+    if (proc_fd != -1)
+        sys_close(proc_fd);
+#endif /* GF_LINUX_HOST_OS */
+    return ports_info;
 }
 
 int
-gf_string2uint32 (const char *str, uint32_t *n)
+gf_process_reserved_ports(unsigned char *ports, uint32_t ceiling)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    int ret = -1;
+
+    memset(ports, 0, GF_PORT_ARRAY_SIZE);
+
+#if defined GF_LINUX_HOST_OS
+    char *ports_info = NULL;
+    char *tmp = NULL;
+    char *blocked_port = NULL;
+
+    ports_info = gf_get_reserved_ports();
+    if (!ports_info) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, 0, LG_MSG_RESERVED_PORTS_ERROR,
+                NULL);
+        goto out;
+    }
 
-	rv = _gf_string2ulong (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    blocked_port = strtok_r(ports_info, ",\n", &tmp);
 
-	if (l >= 0 && l <= UINT32_MAX)
-	{
-		*n = (uint32_t) l;
-		return 0;
-	}
+    while (blocked_port) {
+        gf_ports_reserved(blocked_port, ports, ceiling);
+        blocked_port = strtok_r(NULL, ",\n", &tmp);
+    }
 
-	errno = ERANGE;
-	return -1;
+    ret = 0;
+
+out:
+    GF_FREE(ports_info);
+
+#else  /* FIXME: Non Linux Host */
+    ret = 0;
+#endif /* GF_LINUX_HOST_OS */
+
+    return ret;
 }
 
+gf_boolean_t
+gf_ports_reserved(char *blocked_port, unsigned char *ports, uint32_t ceiling)
+{
+    gf_boolean_t result = _gf_false;
+    char *range_port = NULL;
+    int32_t tmp_port1 = -1;
+    int32_t tmp_port2 = -1;
+
+    if (strstr(blocked_port, "-") == NULL) {
+        /* get rid of the new line character*/
+        if (blocked_port[strlen(blocked_port) - 1] == '\n')
+            blocked_port[strlen(blocked_port) - 1] = '\0';
+        if (gf_string2int32(blocked_port, &tmp_port1) == 0) {
+            if (tmp_port1 > GF_PORT_MAX || tmp_port1 < 0) {
+                gf_smsg("glusterfs-socket", GF_LOG_WARNING, 0,
+                        LG_MSG_INVALID_PORT, "port=%d", tmp_port1, NULL);
+                result = _gf_true;
+                goto out;
+            } else {
+                gf_msg_debug("glusterfs", 0,
+                             "blocking port "
+                             "%d",
+                             tmp_port1);
+                BIT_SET(ports, tmp_port1);
+            }
+        } else {
+            gf_smsg("glusterfs-socket", GF_LOG_WARNING, 0, LG_MSG_INVALID_PORT,
+                    "port=%s", blocked_port, NULL);
+            result = _gf_true;
+            goto out;
+        }
+    } else {
+        range_port = strtok(blocked_port, "-");
+        if (!range_port) {
+            result = _gf_true;
+            goto out;
+        }
+        if (gf_string2int32(range_port, &tmp_port1) == 0) {
+            if (tmp_port1 > ceiling)
+                tmp_port1 = ceiling;
+            if (tmp_port1 < 0)
+                tmp_port1 = 0;
+        }
+        range_port = strtok(NULL, "-");
+        if (!range_port) {
+            result = _gf_true;
+            goto out;
+        }
+        /* get rid of the new line character*/
+        if (range_port[strlen(range_port) - 1] == '\n')
+            range_port[strlen(range_port) - 1] = '\0';
+        if (gf_string2int32(range_port, &tmp_port2) == 0) {
+            if (tmp_port2 > ceiling)
+                tmp_port2 = ceiling;
+            if (tmp_port2 < 0)
+                tmp_port2 = 0;
+        }
+        gf_msg_debug("glusterfs", 0, "lower: %d, higher: %d", tmp_port1,
+                     tmp_port2);
+        for (; tmp_port1 <= tmp_port2; tmp_port1++)
+            BIT_SET(ports, tmp_port1);
+    }
+
+out:
+    return result;
+}
+
+/* Takes in client ip{v4,v6} and returns associated hostname, if any
+ * Also, allocates memory for the hostname.
+ * Returns: 0 for success, -1 for failure
+ */
 int
-gf_string2uint64 (const char *str, uint64_t *n)
+gf_get_hostname_from_ip(char *client_ip, char **hostname)
+{
+    int ret = -1;
+    struct sockaddr *client_sockaddr = NULL;
+    struct sockaddr_in client_sock_in = {0};
+    struct sockaddr_in6 client_sock_in6 = {0};
+    char client_hostname[NI_MAXHOST] = {0};
+    char *client_ip_copy = NULL;
+    char *tmp = NULL;
+    char *ip = NULL;
+    size_t addr_sz = 0;
+
+    /* if ipv4, reverse lookup the hostname to
+     * allow FQDN based rpc authentication
+     */
+    if (!valid_ipv6_address(client_ip, strlen(client_ip), 0) &&
+        !valid_ipv4_address(client_ip, strlen(client_ip), 0)) {
+        /* most times, we get a.b.c.d:port form, so check that */
+        client_ip_copy = gf_strdup(client_ip);
+        if (!client_ip_copy)
+            goto out;
+
+        ip = strtok_r(client_ip_copy, ":", &tmp);
+    } else {
+        ip = client_ip;
+    }
+
+    if (valid_ipv4_address(ip, strlen(ip), 0) == _gf_true) {
+        client_sockaddr = (struct sockaddr *)&client_sock_in;
+        addr_sz = sizeof(client_sock_in);
+        client_sock_in.sin_family = AF_INET;
+        ret = inet_pton(AF_INET, ip, (void *)&client_sock_in.sin_addr.s_addr);
+
+    } else if (valid_ipv6_address(ip, strlen(ip), 0) == _gf_true) {
+        client_sockaddr = (struct sockaddr *)&client_sock_in6;
+        addr_sz = sizeof(client_sock_in6);
+
+        client_sock_in6.sin6_family = AF_INET6;
+        ret = inet_pton(AF_INET6, ip, (void *)&client_sock_in6.sin6_addr);
+    } else {
+        goto out;
+    }
+
+    if (ret != 1) {
+        ret = -1;
+        goto out;
+    }
+
+    /* You cannot just use sizeof (*client_sockaddr), as per the man page
+     * the (getnameinfo) size must be the size of the underlying sockaddr
+     * struct e.g. sockaddr_in6 or sockaddr_in.  Failure to do so will
+     * break IPv6 hostname resolution (IPv4 will work only because
+     * the sockaddr_in struct happens to be of the correct size).
+     */
+    ret = getnameinfo(client_sockaddr, addr_sz, client_hostname,
+                      sizeof(client_hostname), NULL, 0, 0);
+    if (ret) {
+        gf_smsg("common-utils", GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                "ip=%s", client_ip, "ret=%s", gai_strerror(ret), NULL);
+        ret = -1;
+        goto out;
+    }
+
+    *hostname = gf_strdup((char *)client_hostname);
+out:
+    if (client_ip_copy)
+        GF_FREE(client_ip_copy);
+
+    return ret;
+}
+
+gf_boolean_t
+gf_interface_search(char *ip)
+{
+    int32_t ret = -1;
+    gf_boolean_t found = _gf_false;
+    struct ifaddrs *ifaddr, *ifa;
+    int family;
+    char host[NI_MAXHOST];
+    xlator_t *this = NULL;
+    char *pct = NULL;
+
+    this = THIS;
+
+    ret = getifaddrs(&ifaddr);
+
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETIFADDRS_FAILED, "ret=%s",
+                gai_strerror(ret), NULL);
+        goto out;
+    }
+
+    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+        if (!ifa->ifa_addr) {
+            /*
+             * This seemingly happens if an interface hasn't
+             * been bound to a particular protocol (seen with
+             * TUN devices).
+             */
+            continue;
+        }
+        family = ifa->ifa_addr->sa_family;
+
+        if (family != AF_INET && family != AF_INET6)
+            continue;
+
+        ret = getnameinfo(ifa->ifa_addr,
+                          (family == AF_INET) ? sizeof(struct sockaddr_in)
+                                              : sizeof(struct sockaddr_in6),
+                          host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETNAMEINFO_FAILED,
+                    "ret=%s", gai_strerror(ret), NULL);
+            goto out;
+        }
+
+        /*
+         * Sometimes the address comes back as addr%eth0 or
+         * similar.  Since % is an invalid character, we can
+         * strip it out with confidence that doing so won't
+         * harm anything.
+         */
+        pct = index(host, '%');
+        if (pct) {
+            *pct = '\0';
+        }
+
+        if (strncmp(ip, host, NI_MAXHOST) == 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s is local address at "
+                         "interface %s",
+                         ip, ifa->ifa_name);
+            found = _gf_true;
+            goto out;
+        }
+    }
+out:
+    if (ifaddr)
+        freeifaddrs(ifaddr);
+    return found;
+}
+
+char *
+get_ip_from_addrinfo(struct addrinfo *addr, char **ip)
 {
-	unsigned long long l = 0ULL;
-	int rv = 0;
+    char buf[64];
+    void *in_addr = NULL;
+    struct sockaddr_in *s4 = NULL;
+    struct sockaddr_in6 *s6 = NULL;
+
+    switch (addr->ai_family) {
+        case AF_INET:
+            s4 = (struct sockaddr_in *)addr->ai_addr;
+            in_addr = &s4->sin_addr;
+            break;
+
+        case AF_INET6:
+            s6 = (struct sockaddr_in6 *)addr->ai_addr;
+            in_addr = &s6->sin6_addr;
+            break;
+
+        default:
+            gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_INVALID_FAMILY, NULL);
+            return NULL;
+    }
+
+    if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_CONVERSION_FAILED, NULL);
+        return NULL;
+    }
 
-	rv = _gf_string2ulonglong (str, &l, 0);
-	if (rv != 0)
-		return rv;
+    *ip = gf_strdup(buf);
+    return *ip;
+}
 
-	if (l >= 0 && l <= UINT64_MAX)
-	{
-		*n = (uint64_t) l;
-		return 0;
-	}
+gf_boolean_t
+gf_is_loopback_localhost(const struct sockaddr *sa, char *hostname)
+{
+    GF_ASSERT(sa);
+
+    gf_boolean_t is_local = _gf_false;
+    const struct in_addr *addr4 = NULL;
+    const struct in6_addr *addr6 = NULL;
+    uint8_t *ap = NULL;
+    struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
+
+    switch (sa->sa_family) {
+        case AF_INET:
+            addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
+            ap = (uint8_t *)&addr4->s_addr;
+            if (ap[0] == 127)
+                is_local = _gf_true;
+            break;
+
+        case AF_INET6:
+            addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
+            if (memcmp(addr6, &loopbackaddr6, sizeof(loopbackaddr6)) == 0)
+                is_local = _gf_true;
+            break;
+
+        default:
+            if (hostname)
+                gf_smsg("glusterd", GF_LOG_ERROR, 0, LG_MSG_INVALID_FAMILY,
+                        "family=%d", sa->sa_family, "hostname=%s", hostname,
+                        NULL);
+            break;
+    }
+
+    return is_local;
+}
 
-	errno = ERANGE;
-	return -1;
+gf_boolean_t
+gf_is_local_addr(char *hostname)
+{
+    int32_t ret = -1;
+    struct addrinfo *result = NULL;
+    struct addrinfo *res = NULL;
+    gf_boolean_t found = _gf_false;
+    char *ip = NULL;
+    xlator_t *this = NULL;
+    struct addrinfo hints;
+
+    this = THIS;
+
+    memset(&hints, 0, sizeof(hints));
+    /*
+     * Removing AI_ADDRCONFIG from default_hints
+     * for being able to use link local ipv6 addresses
+     */
+    hints.ai_family = AF_UNSPEC;
+
+    ret = getaddrinfo(hostname, NULL, &hints, &result);
+
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, LG_MSG_GETADDRINFO_FAILED,
+                "ret=%s", gai_strerror(ret), NULL);
+        goto out;
+    }
+
+    for (res = result; res != NULL; res = res->ai_next) {
+        get_ip_from_addrinfo(res, &ip);
+        gf_msg_debug(this->name, 0, "%s ", ip);
+
+        if (ip) {
+            found = (gf_is_loopback_localhost(res->ai_addr, hostname) ||
+                     gf_interface_search(ip));
+        }
+        if (found) {
+            GF_FREE(ip);
+            goto out;
+        }
+        GF_FREE(ip);
+        /* the above free will not set ip to NULL, and hence, there is
+           double free possible as the loop continues. set ip to NULL. */
+        ip = NULL;
+    }
+
+out:
+    if (result)
+        freeaddrinfo(result);
+
+    if (!found)
+        gf_msg_debug(this->name, 0, "%s is not local", hostname);
+
+    return found;
 }
 
+gf_boolean_t
+gf_is_same_address(char *name1, char *name2)
+{
+    struct addrinfo *addr1 = NULL;
+    struct addrinfo *addr2 = NULL;
+    struct addrinfo *p = NULL;
+    struct addrinfo *q = NULL;
+    gf_boolean_t ret = _gf_false;
+    int gai_err = 0;
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
+
+    gai_err = getaddrinfo(name1, NULL, &hints, &addr1);
+    if (gai_err != 0) {
+        gf_smsg(name1, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED, "error=%s",
+                gai_strerror(gai_err), NULL);
+        goto out;
+    }
+
+    gai_err = getaddrinfo(name2, NULL, &hints, &addr2);
+    if (gai_err != 0) {
+        gf_smsg(name2, GF_LOG_WARNING, 0, LG_MSG_GETADDRINFO_FAILED, "error=%s",
+                gai_strerror(gai_err), NULL);
+        goto out;
+    }
+
+    for (p = addr1; p; p = p->ai_next) {
+        for (q = addr2; q; q = q->ai_next) {
+            if (p->ai_addrlen != q->ai_addrlen) {
+                continue;
+            }
+            if (memcmp(p->ai_addr, q->ai_addr, p->ai_addrlen)) {
+                continue;
+            }
+            ret = _gf_true;
+            goto out;
+        }
+    }
+
+out:
+    if (addr1) {
+        freeaddrinfo(addr1);
+    }
+    if (addr2) {
+        freeaddrinfo(addr2);
+    }
+    return ret;
+}
+
+/*
+ * Processes list of volfile servers.
+ * Format: <host1>:<port1> <host2>:<port2>...
+ */
 int
-gf_string2ulong_base10 (const char *str, unsigned long *n)
+gf_process_getspec_servers_list(cmd_args_t *cmd_args, const char *servers_list)
 {
-	return _gf_string2ulong (str, n, 10);
+    char *tmp = NULL;
+    char *address = NULL;
+    char *host = NULL;
+    char *last_colon = NULL;
+    char *save_ptr = NULL;
+    int port = 0;
+    int ret = -1;
+
+    tmp = gf_strdup(servers_list);
+    if (!tmp) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    address = strtok_r(tmp, " ", &save_ptr);
+    if (!address) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    while (1) {
+        last_colon = strrchr(address, ':');
+        if (!last_colon) {
+            errno = EINVAL;
+            ret = -1;
+            break;
+        }
+        *last_colon = '\0';
+        host = address;
+        port = atoi(last_colon + 1);
+        if (port <= 0) {
+            errno = EINVAL;
+            ret = -1;
+            break;
+        }
+        ret = gf_set_volfile_server_common(cmd_args, host,
+                                           GF_DEFAULT_VOLFILE_TRANSPORT, port);
+        if (ret && errno != EEXIST) {
+            break;
+        }
+        address = strtok_r(NULL, " ", &save_ptr);
+        if (!address) {
+            errno = 0;
+            ret = 0;
+            break;
+        }
+    }
+
+out:
+    if (tmp) {
+        GF_FREE(tmp);
+    }
+
+    return ret;
 }
 
 int
-gf_string2uint_base10 (const char *str, unsigned int *n)
+gf_set_volfile_server_common(cmd_args_t *cmd_args, const char *host,
+                             const char *transport, int port)
 {
-	return _gf_string2uint (str,  n, 10);
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, host, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, transport, out);
+
+    server = GF_CALLOC(1, sizeof(server_cmdline_t),
+                       gf_common_mt_server_cmdline_t);
+    if (!server) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&server->list);
+
+    server->volfile_server = gf_strdup(host);
+    if (!server->volfile_server) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    server->transport = gf_strdup(transport);
+    if (!server->transport) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    server->port = port;
+
+    if (!cmd_args->volfile_server) {
+        cmd_args->volfile_server = server->volfile_server;
+        cmd_args->volfile_server_transport = server->transport;
+        cmd_args->volfile_server_port = server->port;
+        cmd_args->curr_server = server;
+    }
+
+    list_for_each_entry(tmp, &cmd_args->volfile_servers, list)
+    {
+        if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
+             !strcmp(tmp->transport, server->transport) &&
+             (tmp->port == server->port))) {
+            /* Duplicate option given, log and ignore */
+            gf_smsg("gluster", GF_LOG_INFO, EEXIST, LG_MSG_DUPLICATE_ENTRY,
+                    NULL);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    list_add_tail(&server->list, &cmd_args->volfile_servers);
+
+    ret = 0;
+out:
+    if (-1 == ret) {
+        if (server) {
+            GF_FREE(server->volfile_server);
+            GF_FREE(server->transport);
+            GF_FREE(server);
+        }
+    }
+
+    return ret;
 }
 
+/* Sets log file path from user provided arguments */
 int
-gf_string2uint8_base10 (const char *str, uint8_t *n)
+gf_set_log_file_path(cmd_args_t *cmd_args, glusterfs_ctx_t *ctx)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int tmp_len = 0;
+    char tmp_str[1024] = {
+        0,
+    };
+
+    if (!cmd_args)
+        goto done;
+
+    if (cmd_args->mount_point) {
+        j = 0;
+        i = 0;
+        if (cmd_args->mount_point[0] == '/')
+            i = 1;
+        for (; i < strlen(cmd_args->mount_point); i++, j++) {
+            tmp_str[j] = cmd_args->mount_point[i];
+            if (cmd_args->mount_point[i] == '/')
+                tmp_str[j] = '-';
+        }
 
-	rv = _gf_string2ulong (str, &l, 10);
-	if (rv != 0)
-		return rv;
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", tmp_str);
+        if (ret > 0)
+            ret = 0;
+        goto done;
+    }
+
+    if (ctx && GF_GLUSTERD_PROCESS == ctx->process_mode) {
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", GLUSTERD_NAME);
+        if (ret > 0)
+            ret = 0;
+
+        goto done;
+    }
+
+    if (cmd_args->volfile) {
+        j = 0;
+        i = 0;
+        if (cmd_args->volfile[0] == '/')
+            i = 1;
+        for (; i < strlen(cmd_args->volfile); i++, j++) {
+            tmp_str[j] = cmd_args->volfile[i];
+            if (cmd_args->volfile[i] == '/')
+                tmp_str[j] = '-';
+        }
+        ret = gf_asprintf(&cmd_args->log_file,
+                          DEFAULT_LOG_FILE_DIRECTORY "/%s.log", tmp_str);
+        if (ret > 0)
+            ret = 0;
+        goto done;
+    }
+
+    if (cmd_args->volfile_server) {
+        if (strncmp(cmd_args->volfile_server_transport, "unix", 4) == 0) {
+            if (cmd_args->volfile_server[0] == '/')
+                i = 1;
+            tmp_len = strlen(cmd_args->volfile_server);
+            for (j = 0; i < tmp_len; i++, j++) {
+                tmp_str[j] = cmd_args->volfile_server[i];
+                if (cmd_args->volfile_server[i] == '/')
+                    tmp_str[j] = '-';
+            }
+            ret = gf_asprintf(&cmd_args->log_file, "%s/%s-%s-%d.log",
+                              DEFAULT_LOG_FILE_DIRECTORY, tmp_str,
+                              cmd_args->volfile_id, getpid());
+        } else {
+            ret = gf_asprintf(&cmd_args->log_file, "%s/%s-%s-%d.log",
+                              DEFAULT_LOG_FILE_DIRECTORY,
+                              cmd_args->volfile_server, cmd_args->volfile_id,
+                              getpid());
+        }
+        if (ret > 0)
+            ret = 0;
+    }
+done:
+    return ret;
+}
 
-	if (l >= 0 && l <= UINT8_MAX)
-	{
-		*n = (uint8_t) l;
-		return 0;
-	}
+int
+gf_set_log_ident(cmd_args_t *cmd_args)
+{
+    int ret = 0;
+    char *ptr = NULL;
+
+    if (cmd_args->log_file == NULL) {
+        /* no ident source */
+        return 0;
+    }
+
+    /* TODO: Some idents would look like, etc-glusterfs-glusterd.vol, which
+     * seems ugly and can be bettered? */
+    /* just get the filename as the ident */
+    if (NULL != (ptr = strrchr(cmd_args->log_file, '/'))) {
+        ret = gf_asprintf(&cmd_args->log_ident, "%s", ptr + 1);
+    } else {
+        ret = gf_asprintf(&cmd_args->log_ident, "%s", cmd_args->log_file);
+    }
+
+    if (ret > 0)
+        ret = 0;
+    else
+        return ret;
 
-	errno = ERANGE;
-	return -1;
+    /* remove .log suffix */
+    if (NULL != (ptr = strrchr(cmd_args->log_ident, '.'))) {
+        if (strcmp(ptr, ".log") == 0) {
+            ptr[0] = '\0';
+        }
+    }
+
+    return ret;
 }
 
 int
-gf_string2uint16_base10 (const char *str, uint16_t *n)
+gf_thread_cleanup_xint(pthread_t thread)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    int ret = 0;
+    void *res = NULL;
+
+    ret = pthread_cancel(thread);
+    if (ret != 0)
+        goto error_return;
 
-	rv = _gf_string2ulong (str, &l, 10);
-	if (rv != 0)
-		return rv;
+    ret = pthread_join(thread, &res);
+    if (ret != 0)
+        goto error_return;
+
+    if (res != PTHREAD_CANCELED)
+        goto error_return;
+
+    ret = 0;
+
+error_return:
+    return ret;
+}
+
+void
+gf_thread_set_vname(pthread_t thread, const char *name, va_list args)
+{
+    char thread_name[GF_THREAD_NAME_LIMIT];
+    int ret;
+
+    /* Initialize the thread name with the prefix (not NULL terminated). */
+    memcpy(thread_name, GF_THREAD_NAME_PREFIX,
+           sizeof(GF_THREAD_NAME_PREFIX) - 1);
+
+    ret = vsnprintf(thread_name + sizeof(GF_THREAD_NAME_PREFIX) - 1,
+                    sizeof(thread_name) - sizeof(GF_THREAD_NAME_PREFIX) + 1,
+                    name, args);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_PTHREAD_NAMING_FAILED,
+                "name=%s", name, NULL);
+        return;
+    }
+
+    if (ret >= sizeof(thread_name)) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_THREAD_NAME_TOO_LONG,
+                "name=%s", thread_name, NULL);
+    }
+
+#ifdef GF_LINUX_HOST_OS
+    ret = pthread_setname_np(thread, thread_name);
+#elif defined(__NetBSD__)
+    ret = pthread_setname_np(thread, thread_name, NULL);
+#elif defined(__FreeBSD__)
+    pthread_set_name_np(thread, thread_name);
+    ret = 0;
+#else
+    ret = ENOSYS;
+#endif
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, ret, LG_MSG_SET_THREAD_FAILED,
+                "name=%s", thread_name, NULL);
+    }
+}
 
-	if (l >= 0 && l <= UINT16_MAX)
-	{
-		*n = (uint16_t) l;
-		return 0;
-	}
+void
+gf_thread_set_name(pthread_t thread, const char *name, ...)
+{
+    va_list args;
 
-	errno = ERANGE;
-	return -1;
+    va_start(args, name);
+    gf_thread_set_vname(thread, name, args);
+    va_end(args);
 }
 
 int
-gf_string2uint32_base10 (const char *str, uint32_t *n)
+gf_thread_vcreate(pthread_t *thread, const pthread_attr_t *attr,
+                  void *(*start_routine)(void *), void *arg, const char *name,
+                  va_list args)
 {
-	unsigned long l = 0L;
-	int rv = 0;
+    sigset_t set, old;
+    int ret;
+
+    sigemptyset(&old);
+    sigfillset(&set);
+    sigdelset(&set, SIGSEGV);
+    sigdelset(&set, SIGBUS);
+    sigdelset(&set, SIGILL);
+    sigdelset(&set, SIGSYS);
+    sigdelset(&set, SIGFPE);
+    sigdelset(&set, SIGABRT);
+
+    pthread_sigmask(SIG_BLOCK, &set, &old);
+
+    ret = pthread_create(thread, attr, start_routine, arg);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ret, LG_MSG_THREAD_CREATE_FAILED,
+                NULL);
+        ret = -1;
+    } else if (name != NULL) {
+        gf_thread_set_vname(*thread, name, args);
+    }
+
+    pthread_sigmask(SIG_SETMASK, &old, NULL);
+
+    return ret;
+}
 
-	rv = _gf_string2ulong (str, &l, 10);
-	if (rv != 0)
-		return rv;
+int
+gf_thread_create(pthread_t *thread, const pthread_attr_t *attr,
+                 void *(*start_routine)(void *), void *arg, const char *name,
+                 ...)
+{
+    va_list args;
+    int ret;
 
-	if (l >= 0 && l <= UINT32_MAX)
-	{
-		*n = (uint32_t) l;
-		return 0;
-	}
+    va_start(args, name);
+    ret = gf_thread_vcreate(thread, attr, start_routine, arg, name, args);
+    va_end(args);
 
-	errno = ERANGE;
-	return -1;
+    return ret;
 }
 
 int
-gf_string2uint64_base10 (const char *str, uint64_t *n)
+gf_thread_create_detached(pthread_t *thread, void *(*start_routine)(void *),
+                          void *arg, const char *name, ...)
 {
-	unsigned long long l = 0ULL;
-	int rv = 0;
+    pthread_attr_t attr;
+    va_list args;
+    int ret = -1;
+
+    ret = pthread_attr_init(&attr);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ret, LG_MSG_PTHREAD_ATTR_INIT_FAILED,
+                NULL);
+        return -1;
+    }
+
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-	rv = _gf_string2ulonglong (str, &l, 10);
-	if (rv != 0)
-		return rv;
+    va_start(args, name);
+    ret = gf_thread_vcreate(thread, &attr, start_routine, arg, name, args);
+    va_end(args);
 
-	if (l >= 0 && l <= UINT64_MAX)
-	{
-		*n = (uint64_t) l;
-		return 0;
-	}
+    pthread_attr_destroy(&attr);
 
-	errno = ERANGE;
-	return -1;
+    return ret;
 }
 
 int
-gf_string2bytesize (const char *str, uint64_t *n)
+gf_skip_header_section(int fd, int header_len)
 {
-	uint64_t value = 0ULL;
-	char *tail = NULL;
-	int old_errno = 0;
-	const char *s = NULL;
+    int ret = -1;
 
-	if (str == NULL || n == NULL)
-	{
-		errno = EINVAL;
-		return -1;
-	}
+    ret = sys_lseek(fd, header_len, SEEK_SET);
+    if (ret == (off_t)-1) {
+        gf_smsg("", GF_LOG_ERROR, 0, LG_MSG_SKIP_HEADER_FAILED, NULL);
+    } else {
+        ret = 0;
+    }
 
-	for (s = str; *s != '\0'; s++)
-	{
-		if (isspace (*s))
-		{
-			continue;
-		}
-		if (*s == '-')
-		{
-			/* bala: we do not support suffixed (-) sign and
-			   invalid integer format */
-			return -1;
-		}
-		break;
-	}
+    return ret;
+}
 
-	old_errno = errno;
-	errno = 0;
-	value = strtoull (str, &tail, 10);
+/* Below function is use to check at runtime if pid is running */
 
-	if (errno == ERANGE || errno == EINVAL)
-	{
-		return -1;
-	}
+gf_boolean_t
+gf_is_pid_running(int pid)
+{
+#ifdef __FreeBSD__
+    int ret = -1;
+
+    ret = sys_kill(pid, 0);
+    if (ret < 0) {
+        return _gf_false;
+    }
+#else
+    char fname[32] = {
+        0,
+    };
+    int fd = -1;
+
+    snprintf(fname, sizeof(fname), "/proc/%d/cmdline", pid);
+
+    fd = sys_open(fname, O_RDONLY, 0);
+    if (fd < 0) {
+        return _gf_false;
+    }
+
+    sys_close(fd);
+#endif
+    return _gf_true;
+}
 
-	if (errno == 0)
-	{
-		errno = old_errno;
-	}
-
-	if (tail[0] != '\0')
-	{
-		if (strcasecmp (tail, GF_UNIT_KB_STRING) == 0)
-		{
-			value *= GF_UNIT_KB;
-		}
-		else if (strcasecmp (tail, GF_UNIT_MB_STRING) == 0)
-		{
-			value *= GF_UNIT_MB;
-		}
-		else if (strcasecmp (tail, GF_UNIT_GB_STRING) == 0)
-		{
-			value *= GF_UNIT_GB;
-		}
-		else if (strcasecmp (tail, GF_UNIT_TB_STRING) == 0)
-		{
-			value *= GF_UNIT_TB;
-		}
-		else if (strcasecmp (tail, GF_UNIT_PB_STRING) == 0)
-		{
-			value *= GF_UNIT_PB;
-		}
-		else
-		{
-			/* bala: invalid integer format */
-			return -1;
-		}
-	}
-
-	*n = value;
-
-	return 0;
+gf_boolean_t
+gf_is_service_running(char *pidfile, int *pid)
+{
+    FILE *file = NULL;
+    gf_boolean_t running = _gf_false;
+    int ret = 0;
+    int fno = 0;
+
+    file = fopen(pidfile, "r+");
+    if (!file) {
+        goto out;
+    }
+
+    fno = fileno(file);
+    ret = lockf(fno, F_TEST, 0);
+    if (ret == -1) {
+        running = _gf_true;
+    }
+
+    ret = fscanf(file, "%d", pid);
+    if (ret <= 0) {
+        gf_smsg("", GF_LOG_ERROR, errno, LG_MSG_FILE_OP_FAILED, "pidfile=%s",
+                pidfile, NULL);
+        *pid = -1;
+        running = _gf_false;
+        goto out;
+    }
+
+    running = gf_is_pid_running(*pid);
+out:
+    if (file)
+        fclose(file);
+    return running;
 }
 
-int64_t
-gf_str_to_long_long (const char *number)
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid(const char *pid, int length)
 {
-	int64_t unit = 1;
-	int64_t ret = 0;
-	char *endptr = NULL ;
-	if (!number)
-		return 0;
+    gf_boolean_t ret = _gf_true;
+    pid_t value = 0;
+    char *end_ptr = NULL;
+
+    if (length <= 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    value = strtol(pid, &end_ptr, 10);
+    if (value <= 0) {
+        ret = _gf_false;
+    }
+out:
+    return ret;
+}
 
-	ret = strtoll (number, &endptr, 0);
+static int
+dht_is_linkfile_key(dict_t *this, char *key, data_t *value, void *data)
+{
+    gf_boolean_t *linkfile_key_found = NULL;
+
+    if (!data)
+        goto out;
+
+    linkfile_key_found = data;
 
-	if (endptr) {
-		switch (*endptr) {
-		case 'G':
-		case 'g':
-			if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-				unit = 1024 * 1024 * 1024;
-			break;
-		case 'M':
-		case 'm':
-			if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-				unit = 1024 * 1024;
-			break;
-		case 'K':
-		case 'k':
-			if ((* (endptr + 1) == 'B') ||(* (endptr + 1) == 'b'))
-				unit = 1024;
-			break;
-		case '%':
-			unit = 1;
-			break;
-		default:
-			unit = 1;
-			break;
-		}
-	}
-	return ret * unit;
+    *linkfile_key_found = _gf_true;
+out:
+    return 0;
 }
 
-int
-gf_string2boolean (const char *str, gf_boolean_t *b)
+gf_boolean_t
+dht_is_linkfile(struct iatt *buf, dict_t *dict)
 {
-	if (str == NULL) {
-		return -1;
-	}
+    gf_boolean_t linkfile_key_found = _gf_false;
 
-	if ((strcasecmp (str, "1") == 0) ||
-	    (strcasecmp (str, "on") == 0) ||
-	    (strcasecmp (str, "yes") == 0) ||
-	    (strcasecmp (str, "true") == 0) ||
-	    (strcasecmp (str, "enable") == 0)) {
-		*b = _gf_true;
-		return 0;
-	}
+    if (!IS_DHT_LINKFILE_MODE(buf))
+        return _gf_false;
 
-	if ((strcasecmp (str, "0") == 0) ||
-	    (strcasecmp (str, "off") == 0) ||
-	    (strcasecmp (str, "no") == 0) ||
-	    (strcasecmp (str, "false") == 0) ||
-	    (strcasecmp (str, "disable") == 0)) {
-		*b = _gf_false;
-		return 0;
-	}
+    dict_foreach_fnmatch(dict, "*." DHT_LINKFILE_STR, dht_is_linkfile_key,
+                         &linkfile_key_found);
 
-	return -1;
+    return linkfile_key_found;
 }
 
+int
+gf_check_log_format(const char *value)
+{
+    int log_format = -1;
+
+    if (!strcasecmp(value, GF_LOG_FORMAT_NO_MSG_ID))
+        log_format = gf_logformat_traditional;
+    else if (!strcasecmp(value, GF_LOG_FORMAT_WITH_MSG_ID))
+        log_format = gf_logformat_withmsgid;
+
+    if (log_format == -1)
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
+                "possible_values=" GF_LOG_FORMAT_NO_MSG_ID
+                "|" GF_LOG_FORMAT_WITH_MSG_ID,
+                NULL);
+
+    return log_format;
+}
 
 int
-gf_lockfd (int fd)
+gf_check_logger(const char *value)
 {
-	struct gf_flock fl;
+    int logger = -1;
+
+    if (!strcasecmp(value, GF_LOGGER_GLUSTER_LOG))
+        logger = gf_logger_glusterlog;
+    else if (!strcasecmp(value, GF_LOGGER_SYSLOG))
+        logger = gf_logger_syslog;
+
+    if (logger == -1)
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INVALID_LOG,
+                "possible_values=" GF_LOGGER_GLUSTER_LOG "|" GF_LOGGER_SYSLOG,
+                NULL);
 
-	fl.l_type = F_WRLCK;
-	fl.l_whence = SEEK_SET;
-	fl.l_start = 0;
-	fl.l_len = 0;
+    return logger;
+}
 
-	return fcntl (fd, F_SETLK, &fl);
+/* gf_compare_sockaddr compares the given addresses @addr1 and @addr2 for
+ * equality, ie. if they both refer to the same address.
+ *
+ * This was inspired by sock_addr_cmp_addr() from
+ * https://www.opensource.apple.com/source/postfix/postfix-197/postfix/src/util/sock_addr.c
+ */
+gf_boolean_t
+gf_compare_sockaddr(const struct sockaddr *addr1, const struct sockaddr *addr2)
+{
+    GF_ASSERT(addr1 != NULL);
+    GF_ASSERT(addr2 != NULL);
+
+    /* Obviously, the addresses don't match if their families are different
+     */
+    if (addr1->sa_family != addr2->sa_family)
+        return _gf_false;
+
+    if (AF_INET == addr1->sa_family) {
+        if (((struct sockaddr_in *)addr1)->sin_addr.s_addr ==
+            ((struct sockaddr_in *)addr2)->sin_addr.s_addr)
+            return _gf_true;
+
+    } else if (AF_INET6 == addr1->sa_family) {
+        if (memcmp((char *)&((struct sockaddr_in6 *)addr1)->sin6_addr,
+                   (char *)&((struct sockaddr_in6 *)addr2)->sin6_addr,
+                   sizeof(struct in6_addr)) == 0)
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
+/*
+ * gf_set_timestamp:
+ *      It sets the mtime and atime of 'dest' file as of 'src'.
+ */
 
 int
-gf_unlockfd (int fd)
+gf_set_timestamp(const char *src, const char *dest)
+{
+    struct stat sb = {
+        0,
+    };
+#if defined(HAVE_UTIMENSAT)
+    struct timespec new_time[2] = {{
+                                       0,
+                                   },
+                                   {
+                                       0,
+                                   }};
+#else
+    struct timeval new_time[2] = {{
+                                      0,
+                                  },
+                                  {
+                                      0,
+                                  }};
+#endif
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(src);
+    GF_ASSERT(dest);
+
+    ret = sys_stat(src, &sb);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_FILE_STAT_FAILED,
+                "stat=%s", src, NULL);
+        goto out;
+    }
+    /* The granularity is nano seconds if `utimensat()` is available,
+     * and micro seconds otherwise.
+     */
+#if defined(HAVE_UTIMENSAT)
+    new_time[0].tv_sec = sb.st_atime;
+    new_time[0].tv_nsec = ST_ATIM_NSEC(&sb);
+
+    new_time[1].tv_sec = sb.st_mtime;
+    new_time[1].tv_nsec = ST_MTIM_NSEC(&sb);
+
+    /* dirfd = 0 is ignored because `dest` is an absolute path. */
+    ret = sys_utimensat(AT_FDCWD, dest, new_time, AT_SYMLINK_NOFOLLOW);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_UTIMENSAT_FAILED,
+                "dest=%s", dest, NULL);
+    }
+#else
+    new_time[0].tv_sec = sb.st_atime;
+    new_time[0].tv_usec = ST_ATIM_NSEC(&sb) / 1000;
+
+    new_time[1].tv_sec = sb.st_mtime;
+    new_time[1].tv_usec = ST_MTIM_NSEC(&sb) / 1000;
+
+    ret = sys_utimes(dest, new_time);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, LG_MSG_UTIMES_FAILED,
+                "dest=%s", dest, NULL);
+    }
+#endif
+out:
+    return ret;
+}
+
+static void
+gf_backtrace_end(char *buf, size_t frames)
 {
-	struct gf_flock fl;
+    size_t pos = 0;
+
+    if (!buf)
+        return;
+
+    pos = strlen(buf);
+
+    frames = min(frames, GF_BACKTRACE_LEN - pos - 1);
 
-	fl.l_type = F_UNLCK;
-	fl.l_whence = SEEK_SET;
-	fl.l_start = 0;
-	fl.l_len = 0;
+    if (0 == frames)
+        return;
 
-	return fcntl (fd, F_SETLK, &fl);
+    memset(buf + pos, ')', frames);
+    buf[pos + frames] = '\0';
 }
 
-static void
-compute_checksum (char *buf, size_t size, uint32_t *checksum)
+/*Returns bytes written*/
+static int
+gf_backtrace_append(char *buf, size_t pos, char *framestr)
 {
-        int  ret = -1;
-        char *checksum_buf = NULL;
+    if (pos >= GF_BACKTRACE_LEN)
+        return -1;
+    return snprintf(buf + pos, GF_BACKTRACE_LEN - pos, "(--> %s ", framestr);
+}
 
-        checksum_buf = (char *)(checksum);
+static int
+gf_backtrace_fillframes(char *buf)
+{
+    void *array[GF_BACKTRACE_FRAME_COUNT];
+    size_t frames = 0;
+    FILE *fp = NULL;
+    char callingfn[GF_BACKTRACE_FRAME_COUNT - 2][1024] = {
+        {0},
+    };
+    int ret = -1;
+    int fd = -1;
+    size_t idx = 0;
+    size_t pos = 0;
+    size_t inc = 0;
+    char tmpl[] = "/tmp/glfs-bt-XXXXXX";
+
+    frames = backtrace(array, GF_BACKTRACE_FRAME_COUNT);
+    if (!frames)
+        return -1;
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    fd = mkstemp(tmpl);
+    if (fd == -1)
+        return -1;
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(tmpl);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_INFO, 0, LG_MSG_FILE_DELETE_FAILED,
+                "temporary_file=%s", tmpl, NULL);
+    }
+
+    /*The most recent two frames are the calling function and
+     * gf_backtrace_save, which we can infer.*/
+
+    backtrace_symbols_fd(&array[2], frames - 2, fd);
+
+    fp = fdopen(fd, "r");
+    if (!fp) {
+        sys_close(fd);
+        goto out;
+    }
+
+    ret = fseek(fp, 0L, SEEK_SET);
+    if (ret)
+        goto out;
+
+    pos = 0;
+    for (idx = 0; idx < frames - 2; idx++) {
+        ret = fscanf(fp, "%1023s", callingfn[idx]);
+        if (ret == EOF)
+            break;
+        inc = gf_backtrace_append(buf, pos, callingfn[idx]);
+        if (inc == -1)
+            break;
+        pos += inc;
+    }
+    gf_backtrace_end(buf, idx);
 
-        if (!(*checksum)) {
-                checksum_buf [0] = 0xba;
-                checksum_buf [1] = 0xbe;
-                checksum_buf [2] = 0xb0;
-                checksum_buf [3] = 0x0b;
-        }
+out:
+    if (fp)
+        fclose(fp);
 
-        for (ret = 0; ret < (size - 4); ret += 4) {
-                checksum_buf[0] ^= (buf[ret]);
-                checksum_buf[1] ^= (buf[ret + 1] << 1) ;
-                checksum_buf[2] ^= (buf[ret + 2] << 2);
-                checksum_buf[3] ^= (buf[ret + 3] << 3);
-        }
+    return (idx > 0) ? 0 : -1;
+}
+
+/* Optionally takes @buf to save backtrace.  If @buf is NULL, uses the
+ * pre-allocated ctx->btbuf to avoid allocating memory while printing
+ * backtrace.
+ * TODO: This API doesn't provide flexibility in terms of no. of frames
+ * of the backtrace is being saved in the buffer. Deferring fixing it
+ * when there is a real-use for that.*/
+
+char *
+gf_backtrace_save(char *buf)
+{
+    char *bt = NULL;
 
-        for (ret = 0; ret <= (size % 4); ret++) {
-                checksum_buf[ret] ^= (buf[(size - 4) + ret] << ret);
+    if (!buf) {
+        bt = THIS->ctx->btbuf;
+        GF_ASSERT(bt);
+
+    } else {
+        bt = buf;
+    }
+
+    if ((0 == gf_backtrace_fillframes(bt)))
+        return bt;
+
+    gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_BACKTRACE_SAVE_FAILED, NULL);
+    return NULL;
+}
+
+gf_loglevel_t
+fop_log_level(glusterfs_fop_t fop, int op_errno)
+{
+    /* if gfid doesn't exist ESTALE comes */
+    if (op_errno == ENOENT || op_errno == ESTALE)
+        return GF_LOG_DEBUG;
+
+    if ((fop == GF_FOP_ENTRYLK) || (fop == GF_FOP_FENTRYLK) ||
+        (fop == GF_FOP_FINODELK) || (fop == GF_FOP_INODELK) ||
+        (fop == GF_FOP_LK)) {
+        /*
+         * if non-blocking lock fails EAGAIN comes
+         * if locks xlator is not loaded ENOSYS comes
+         */
+        if (op_errno == EAGAIN || op_errno == ENOSYS)
+            return GF_LOG_DEBUG;
+    }
+
+    if ((fop == GF_FOP_GETXATTR) || (fop == GF_FOP_FGETXATTR)) {
+        if (op_errno == ENOTSUP || op_errno == ENODATA)
+            return GF_LOG_DEBUG;
+    }
+
+    if ((fop == GF_FOP_SETXATTR) || (fop == GF_FOP_FSETXATTR) ||
+        (fop == GF_FOP_REMOVEXATTR) || (fop == GF_FOP_FREMOVEXATTR)) {
+        if (op_errno == ENOTSUP)
+            return GF_LOG_DEBUG;
+    }
+
+    if (fop == GF_FOP_MKNOD || fop == GF_FOP_MKDIR)
+        if (op_errno == EEXIST)
+            return GF_LOG_DEBUG;
+
+    if (fop == GF_FOP_SEEK) {
+#ifdef HAVE_SEEK_HOLE
+        if (op_errno == ENXIO) {
+            return GF_LOG_DEBUG;
         }
+#else
+        return GF_LOG_DEBUG;
+#endif
+    }
 
-        return;
+    return GF_LOG_ERROR;
 }
 
-#define GF_CHECKSUM_BUF_SIZE 1024
+/* This function will build absolute path of file/directory from the
+ * current location and relative path given from the current location
+ * For example consider our current path is /a/b/c/ and relative path
+ * from current location is ./../x/y/z .After parsing through this
+ * function the absolute path becomes /a/b/x/y/z/.
+ *
+ * The function gives a pointer to absolute path if it is successful
+ * and also returns zero.
+ * Otherwise function gives NULL pointer with returning an err value.
+ *
+ * So the user need to free memory allocated for path.
+ *
+ */
+
+int32_t
+gf_build_absolute_path(char *current_path, char *relative_path, char **path)
+{
+    char *absolute_path = NULL;
+    char *token = NULL;
+    char *component = NULL;
+    char *saveptr = NULL;
+    char *end = NULL;
+    int ret = 0;
+    size_t relativepath_len = 0;
+    size_t currentpath_len = 0;
+    size_t max_absolutepath_len = 0;
+
+    GF_ASSERT(current_path);
+    GF_ASSERT(relative_path);
+    GF_ASSERT(path);
+
+    if (!path || !current_path || !relative_path) {
+        ret = -EFAULT;
+        goto err;
+    }
+    /* Check for current and relative path
+     * current path should be absolute one and  start from '/'
+     * relative path should not start from '/'
+     */
+    currentpath_len = strlen(current_path);
+    if (current_path[0] != '/' || (currentpath_len > PATH_MAX)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_WRONG_VALUE,
+                "current-path=%s", current_path, NULL);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    relativepath_len = strlen(relative_path);
+    if (relative_path[0] == '/' || (relativepath_len > PATH_MAX)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_WRONG_VALUE,
+                "relative-path=%s", relative_path, NULL);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    /* It is maximum possible value for absolute path */
+    max_absolutepath_len = currentpath_len + relativepath_len + 2;
+
+    absolute_path = GF_CALLOC(1, max_absolutepath_len, gf_common_mt_char);
+    if (!absolute_path) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    absolute_path[0] = '\0';
+
+    /* If current path is root i.e contains only "/", we do not
+     * need to copy it
+     */
+    if (strcmp(current_path, "/") != 0) {
+        strcpy(absolute_path, current_path);
+
+        /* We trim '/' at the end for easier string manipulation */
+        gf_path_strip_trailing_slashes(absolute_path);
+    }
+
+    /* Used to spilt relative path based on '/' */
+    component = gf_strdup(relative_path);
+    if (!component) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    /* In the relative path, we want to consider ".." and "."
+     * if token is ".." , we just need to reduce one level hierarchy
+     * if token is "." , we just ignore it
+     * if token is NULL , end of relative path
+     * if absolute path becomes '\0' and still "..", then it is a bad
+     * relative path,  it points to out of boundary area and stop
+     * building the absolute path
+     * All other cases we just concatenate token to the absolute path
+     */
+    for (token = strtok_r(component, "/", &saveptr),
+        end = strchr(absolute_path, '\0');
+         token; token = strtok_r(NULL, "/", &saveptr)) {
+        if (strcmp(token, ".") == 0)
+            continue;
+
+        else if (strcmp(token, "..") == 0) {
+            if (absolute_path[0] == '\0') {
+                ret = -EACCES;
+                goto err;
+            }
+
+            end = strrchr(absolute_path, '/');
+            *end = '\0';
+        } else {
+            ret = snprintf(end, max_absolutepath_len - strlen(absolute_path),
+                           "/%s", token);
+            end = strchr(absolute_path, '\0');
+        }
+    }
+
+    if (strlen(absolute_path) > PATH_MAX) {
+        ret = -EINVAL;
+        goto err;
+    }
+    *path = gf_strdup(absolute_path);
 
+err:
+    if (component)
+        GF_FREE(component);
+    if (absolute_path)
+        GF_FREE(absolute_path);
+    return ret;
+}
+
+/* This is an utility function which will recursively delete
+ * a folder and its contents.
+ *
+ * @param delete_path folder to be deleted.
+ *
+ * @return 0 on success and -1 on failure.
+ */
 int
-get_checksum_for_file (int fd, uint32_t *checksum)
+recursive_rmdir(const char *delete_path)
 {
-        int ret = -1;
-        char buf[GF_CHECKSUM_BUF_SIZE] = {0,};
+    int ret = -1;
+    char path[PATH_MAX] = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, delete_path, out);
+
+    dir = sys_opendir(delete_path);
+    if (!dir) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to open directory %s. "
+                     "Reason : %s",
+                     delete_path, strerror(errno));
+        ret = 0;
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+        ret = sys_lstat(path, &st);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to stat entry %s :"
+                         " %s",
+                         path, strerror(errno));
+            (void)sys_closedir(dir);
+            goto out;
+        }
 
-        /* goto first place */
-        lseek (fd, 0L, SEEK_SET);
-        do {
-                ret = read (fd, &buf, GF_CHECKSUM_BUF_SIZE);
-                if (ret > 0)
-                        compute_checksum (buf, GF_CHECKSUM_BUF_SIZE,
-                                          checksum);
-        } while (ret > 0);
+        if (S_ISDIR(st.st_mode))
+            ret = recursive_rmdir(path);
+        else
+            ret = sys_unlink(path);
 
-        /* set it back */
-        lseek (fd, 0L, SEEK_SET);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         " Failed to remove %s. "
+                         "Reason : %s",
+                         path, strerror(errno));
+        }
 
-        return ret;
+        gf_msg_debug(this->name, 0, "%s %s",
+                     ret ? "Failed to remove" : "Removed", entry->d_name);
+    }
+
+    ret = sys_closedir(dir);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to close dir %s. Reason :"
+                     " %s",
+                     delete_path, strerror(errno));
+    }
+
+    ret = sys_rmdir(delete_path);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s,err: %s", delete_path,
+                     strerror(errno));
+    }
+
+out:
+    return ret;
 }
+/*
+ * Input: Array of strings 'array' terminating in NULL
+ *        string 'elem' to be searched in the array
+ *
+ * Output: Index of the element in the array if found, '-1' otherwise
+ */
+int
+gf_get_index_by_elem(char **array, char *elem)
+{
+    int i = 0;
 
+    for (i = 0; array[i]; i++) {
+        if (strcmp(elem, array[i]) == 0)
+            return i;
+    }
+
+    return -1;
+}
 
-/* One should pass the command here with command with full path,
-   otherwise, execv will fail */
+static int
+get_pathinfo_host(char *pathinfo, char *hostname, size_t size)
+{
+    char *start = NULL;
+    char *end = NULL;
+    int ret = -1;
+    int i = 0;
+
+    if (!pathinfo)
+        goto out;
+
+    start = strchr(pathinfo, ':');
+    if (!start)
+        goto out;
+
+    end = strrchr(pathinfo, ':');
+    if (start == end)
+        goto out;
+
+    memset(hostname, 0, size);
+    i = 0;
+    while (++start != end)
+        hostname[i++] = *start;
+    ret = 0;
+out:
+    return ret;
+}
+
+/*Note: 'pathinfo' should be gathered only from one brick*/
 int
-gf_system (const char *command)
+glusterfs_is_local_pathinfo(char *pathinfo, gf_boolean_t *is_local)
 {
-        int    ret    = -1;
-        pid_t  pid    = 0;
-        int    status = 0;
-        int    idx    = 0;
-        char  *dupcmd = NULL;
-        char  *arg    = NULL;
-        char  *tmp    = NULL;
-        char  *argv[100] = { NULL, };
+    int ret = 0;
+    char pathinfohost[1024] = {0};
+    char localhost[1024] = {0};
 
-        dupcmd = gf_strdup (command);
-        if (!dupcmd)
-                goto out;
+    *is_local = _gf_false;
+    ret = get_pathinfo_host(pathinfo, pathinfohost, sizeof(pathinfohost));
+    if (ret)
+        goto out;
+
+    ret = gethostname(localhost, sizeof(localhost));
+    if (ret)
+        goto out;
+
+    if (!strcmp(localhost, pathinfohost))
+        *is_local = _gf_true;
+out:
+    return ret;
+}
 
-        pid = fork ();
-        if (pid < 0) {
-                /* failure */
+ssize_t
+gf_nread(int fd, void *buf, size_t count)
+{
+    ssize_t ret = 0;
+    ssize_t read_bytes = 0;
+
+    for (read_bytes = 0; read_bytes < count; read_bytes += ret) {
+        ret = sys_read(fd, buf + read_bytes, count - read_bytes);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            if (errno == EINTR)
+                ret = 0;
+            else
                 goto out;
         }
-        if (pid == 0) {
-                /* Child process */
-                /* Step 0: Prepare the argv */
-                arg = strtok_r (dupcmd, " ", &tmp);
-                while (arg) {
-                        argv[idx] = arg;
-                        arg = strtok_r (NULL, " ", &tmp);
-                        idx++;
-                }
-                /* Step 1: Close all 'fd' */
-                for (idx = 3; idx < 65536; idx++) {
-                        close (idx);
-                }
-                /* Step 2: execv (); */
-                ret = execvp (argv[0], argv);
-
-                /* Code will not come here at all */
-                gf_log ("", GF_LOG_ERROR, "execv of (%s) failed", command);
-
-                kill (getpid(), SIGKILL);
-        }
-        if (pid > 0) {
-                /* Current, ie, parent process */
-                pid = waitpid (pid, &status, 0);
-                if (WIFEXITED(status) && WEXITSTATUS(status) == EXIT_SUCCESS)
-                        ret = 0;
-                else
-                        ret = -1;
+    }
+
+    ret = read_bytes;
+out:
+    return ret;
+}
+
+ssize_t
+gf_nwrite(int fd, const void *buf, size_t count)
+{
+    ssize_t ret = 0;
+    ssize_t written = 0;
+
+    for (written = 0; written != count; written += ret) {
+        ret = sys_write(fd, buf + written, count - written);
+        if (ret < 0) {
+            if (errno == EINTR)
+                ret = 0;
+            else
+                goto out;
         }
+    }
+
+    ret = written;
 out:
-        if (dupcmd)
-                GF_FREE (dupcmd);
+    return ret;
+}
 
-        return ret;
+void
+gf_free_mig_locks(lock_migration_info_t *locks)
+{
+    lock_migration_info_t *current = NULL;
+    lock_migration_info_t *temp = NULL;
+
+    if (!locks)
+        return;
+
+    if (list_empty(&locks->list))
+        return;
+
+    list_for_each_entry_safe(current, temp, &locks->list, list)
+    {
+        list_del_init(&current->list);
+        GF_FREE(current->client_uid);
+        GF_FREE(current);
+    }
 }
 
-int
-get_checksum_for_path (char *path, uint32_t *checksum)
+void
+_mask_cancellation(void)
 {
-        int     ret = -1;
-        int     fd = -1;
+    (void)pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
+}
 
-        GF_ASSERT (path);
-        GF_ASSERT (checksum);
+void
+_unmask_cancellation(void)
+{
+    (void)pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+}
 
-        fd = open (path, O_RDWR);
+/* This is a wrapper function to add a pointer to a list,
+ * which doesn't contain list member
+ */
+struct list_node *
+_list_node_add(void *ptr, struct list_head *list,
+               int (*compare)(struct list_head *, struct list_head *))
+{
+    struct list_node *node = NULL;
 
-        if (fd == -1) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open %s, errno: %d",
-                        path, errno);
-                goto out;
-        }
+    if (ptr == NULL || list == NULL)
+        goto out;
 
-        ret = get_checksum_for_file (fd, checksum);
+    node = GF_CALLOC(1, sizeof(struct list_node), gf_common_list_node);
 
+    if (node == NULL)
+        goto out;
+
+    node->ptr = ptr;
+    if (compare)
+        list_add_order(&node->list, list, compare);
+    else
+        list_add_tail(&node->list, list);
 out:
-        if (fd != -1)
-                close (fd);
+    return node;
+}
+
+struct list_node *
+list_node_add(void *ptr, struct list_head *list)
+{
+    return _list_node_add(ptr, list, NULL);
+}
 
+struct list_node *
+list_node_add_order(void *ptr, struct list_head *list,
+                    int (*compare)(struct list_head *, struct list_head *))
+{
+    return _list_node_add(ptr, list, compare);
+}
+
+void
+list_node_del(struct list_node *node)
+{
+    if (node == NULL)
+        return;
+
+    list_del_init(&node->list);
+    GF_FREE(node);
+}
+
+const char *
+fop_enum_to_pri_string(glusterfs_fop_t fop)
+{
+    switch (fop) {
+        case GF_FOP_OPEN:
+        case GF_FOP_STAT:
+        case GF_FOP_FSTAT:
+        case GF_FOP_LOOKUP:
+        case GF_FOP_ACCESS:
+        case GF_FOP_READLINK:
+        case GF_FOP_OPENDIR:
+        case GF_FOP_STATFS:
+        case GF_FOP_READDIR:
+        case GF_FOP_READDIRP:
+        case GF_FOP_GETACTIVELK:
+        case GF_FOP_SETACTIVELK:
+        case GF_FOP_ICREATE:
+        case GF_FOP_NAMELINK:
+            return "HIGH";
+
+        case GF_FOP_CREATE:
+        case GF_FOP_FLUSH:
+        case GF_FOP_LK:
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+        case GF_FOP_UNLINK:
+        case GF_FOP_SETATTR:
+        case GF_FOP_FSETATTR:
+        case GF_FOP_MKNOD:
+        case GF_FOP_MKDIR:
+        case GF_FOP_RMDIR:
+        case GF_FOP_SYMLINK:
+        case GF_FOP_RENAME:
+        case GF_FOP_LINK:
+        case GF_FOP_SETXATTR:
+        case GF_FOP_GETXATTR:
+        case GF_FOP_FGETXATTR:
+        case GF_FOP_FSETXATTR:
+        case GF_FOP_REMOVEXATTR:
+        case GF_FOP_FREMOVEXATTR:
+        case GF_FOP_IPC:
+        case GF_FOP_LEASE:
+            return "NORMAL";
+
+        case GF_FOP_READ:
+        case GF_FOP_WRITE:
+        case GF_FOP_FSYNC:
+        case GF_FOP_TRUNCATE:
+        case GF_FOP_FTRUNCATE:
+        case GF_FOP_FSYNCDIR:
+        case GF_FOP_XATTROP:
+        case GF_FOP_FXATTROP:
+        case GF_FOP_RCHECKSUM:
+        case GF_FOP_ZEROFILL:
+        case GF_FOP_FALLOCATE:
+        case GF_FOP_SEEK:
+            return "LOW";
+
+        case GF_FOP_NULL:
+        case GF_FOP_FORGET:
+        case GF_FOP_RELEASE:
+        case GF_FOP_RELEASEDIR:
+        case GF_FOP_GETSPEC:
+        case GF_FOP_MAXVALUE:
+        case GF_FOP_DISCARD:
+            return "LEAST";
+        default:
+            return "UNKNOWN";
+    }
+}
+
+const char *
+gf_inode_type_to_str(ia_type_t type)
+{
+    static const char *const str_ia_type[] = {
+        "UNKNOWN",      "REGULAR FILE",     "DIRECTORY", "LINK",
+        "BLOCK DEVICE", "CHARACTER DEVICE", "PIPE",      "SOCKET"};
+    return str_ia_type[type];
+}
+
+gf_boolean_t
+gf_is_zero_filled_stat(struct iatt *buf)
+{
+    if (!buf)
+        return 1;
+
+    /* Do not use st_dev because it is transformed to store the xlator id
+     * in place of the device number. Do not use st_ino because by this time
+     * we've already mapped the root ino to 1 so it is not guaranteed to be
+     * 0.
+     */
+    if ((buf->ia_nlink == 0) && (buf->ia_ctime == 0))
+        return 1;
+
+    return 0;
+}
+
+void
+gf_zero_fill_stat(struct iatt *buf)
+{
+    buf->ia_nlink = 0;
+    buf->ia_ctime = 0;
+}
+
+gf_boolean_t
+gf_is_valid_xattr_namespace(char *key)
+{
+    static char *xattr_namespaces[] = {"trusted.", "system.", "user.",
+                                       "security.", NULL};
+    int i = 0;
+
+    for (i = 0; xattr_namespaces[i]; i++) {
+        if (strncmp(key, xattr_namespaces[i], strlen(xattr_namespaces[i])) == 0)
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+ino_t
+gfid_to_ino(uuid_t gfid)
+{
+    ino_t ino = 0;
+    int32_t i;
+
+    for (i = 8; i < 16; i++) {
+        ino <<= 8;
+        ino += (uint8_t)gfid[i];
+    }
+
+    return ino;
+}
+
+int
+gf_bits_count(uint64_t n)
+{
+    int val = 0;
+#if defined(__GNUC__) || defined(__clang__)
+    val = __builtin_popcountll(n);
+#else
+    n -= (n >> 1) & 0x5555555555555555ULL;
+    n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL);
+    n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+    n += n >> 8;
+    n += n >> 16;
+    n += n >> 32;
+    val = n & 0xFF;
+#endif
+    return val;
+}
+
+int
+gf_bits_index(uint64_t n)
+{
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_ffsll(n) - 1;
+#else
+    return ffsll(n) - 1;
+#endif
+}
+
+const char *
+gf_fop_string(glusterfs_fop_t fop)
+{
+    if ((fop > GF_FOP_NULL) && (fop < GF_FOP_MAXVALUE))
+        return gf_fop_list[fop];
+    return "INVALID";
+}
+
+int
+gf_fop_int(char *fop)
+{
+    int i = 0;
+
+    for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++) {
+        if (strcasecmp(fop, gf_fop_list[i]) == 0)
+            return i;
+    }
+    return -1;
+}
+
+int
+close_fds_except(int *fdv, size_t count)
+{
+    int i = 0;
+    size_t j = 0;
+    gf_boolean_t should_close = _gf_true;
+#ifdef GF_LINUX_HOST_OS
+    DIR *d = NULL;
+    struct dirent *de = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char *e = NULL;
+
+    d = sys_opendir("/proc/self/fd");
+    if (!d)
+        return -1;
+
+    for (;;) {
+        should_close = _gf_true;
+
+        errno = 0;
+        de = sys_readdir(d, scratch);
+        if (!de || errno != 0)
+            break;
+        i = strtoul(de->d_name, &e, 10);
+        if (*e != '\0' || i == dirfd(d))
+            continue;
+
+        for (j = 0; j < count; j++) {
+            if (i == fdv[j]) {
+                should_close = _gf_false;
+                break;
+            }
+        }
+        if (should_close)
+            sys_close(i);
+    }
+    sys_closedir(d);
+#else  /* !GF_LINUX_HOST_OS */
+    struct rlimit rl;
+    int ret = -1;
+
+    ret = getrlimit(RLIMIT_NOFILE, &rl);
+    if (ret)
         return ret;
+
+    for (i = 0; i < rl.rlim_cur; i++) {
+        should_close = _gf_true;
+        for (j = 0; j < count; j++) {
+            if (i == fdv[j]) {
+                should_close = _gf_false;
+                break;
+            }
+        }
+        if (should_close)
+            sys_close(i);
+    }
+#endif /* !GF_LINUX_HOST_OS */
+    return 0;
+}
+
+/**
+ * gf_getgrouplist - get list of groups to which a user belongs
+ *
+ * A convenience wrapper for getgrouplist(3).
+ *
+ * @param user - same as in getgrouplist(3)
+ * @param group - same as in getgrouplist(3)
+ * @param groups - pointer to a gid_t pointer
+ *
+ * gf_getgrouplist allocates a gid_t buffer which is big enough to
+ * hold the list of auxiliary group ids for user, up to the GF_MAX_AUX_GROUPS
+ * threshold. Upon successful invocation groups will be pointed to that buffer.
+ *
+ * @return success: the number of auxiliary group ids retrieved
+ *         failure: -1
+ */
+int
+gf_getgrouplist(const char *user, gid_t group, gid_t **groups)
+{
+    int ret = -1;
+    int ngroups = SMALL_GROUP_COUNT;
+
+    *groups = GF_CALLOC(sizeof(gid_t), ngroups, gf_common_mt_groups_t);
+    if (!*groups)
+        return -1;
+
+    /*
+     * We are running getgrouplist() in a loop until we succeed (or hit
+     * certain exit conditions, see the comments below). This is because
+     * the indicated number of auxiliary groups that we obtain in case of
+     * the failure of the first invocation is not guaranteed to keep its
+     * validity upon the next invocation with a gid buffer of that size.
+     */
+    for (;;) {
+        int ngroups_old = ngroups;
+        ret = getgrouplist(user, group, *groups, &ngroups);
+        if (ret != -1)
+            break;
+
+        if (ngroups >= GF_MAX_AUX_GROUPS) {
+            /*
+             * This should not happen as GF_MAX_AUX_GROUPS is set
+             * to the max value of number of supported auxiliary
+             * groups across all platforms supported by GlusterFS.
+             * However, if it still happened some way, we wouldn't
+             * care about the incompleteness of the result, we'd
+             * just go on with what we got.
+             */
+            return GF_MAX_AUX_GROUPS;
+        } else if (ngroups <= ngroups_old) {
+            /*
+             * There is an edge case that getgrouplist() fails but
+             * ngroups remains the same. This is actually not
+             * specified in getgrouplist(3), but implementations
+             * can do this upon internal failure[1]. To avoid
+             * falling into an infinite loop when this happens, we
+             * break the loop if the getgrouplist call failed
+             * without an increase in the indicated group number.
+             *
+             * [1]
+             * https://sourceware.org/git/?p=glibc.git;a=blob;f=grp/initgroups.c;hb=refs/heads/release/2.25/master#l168
+             */
+            GF_FREE(*groups);
+            return -1;
+        }
+
+        *groups = GF_REALLOC(*groups, ngroups * sizeof(gid_t));
+        if (!*groups)
+            return -1;
+    }
+    return ret;
 }
 
+int
+glusterfs_compute_sha256(const unsigned char *content, size_t size,
+                         char *sha256_hash)
+{
+    SHA256_CTX sha256;
+
+    SHA256_Init(&sha256);
+    SHA256_Update(&sha256, (const unsigned char *)(content), size);
+    SHA256_Final((unsigned char *)sha256_hash, &sha256);
+
+    return 0;
+}
+
+/* * Safe wrapper function for strncpy.
+ * This wrapper makes sure that when there is no null byte among the first n in
+ * source srting for strncpy function call, the string placed in dest will be
+ * null-terminated.
+ */
+
 char *
-strtail (char *str, const char *pattern)
+gf_strncpy(char *dest, const char *src, const size_t dest_size)
 {
-        int i = 0;
+    strncpy(dest, src, dest_size - 1);
+    dest[dest_size - 1] = '\0';
+    return dest;
+}
 
-        for (i = 0; str[i] == pattern[i] && str[i]; i++);
+int
+gf_replace_old_iatt_in_dict(dict_t *xdata)
+{
+    int ret;
+    struct old_iatt *o_iatt; /* old iatt structure */
+    struct iatt *c_iatt;     /* current iatt */
 
-        if (pattern[i] == '\0')
-                return str + i;
+    if (!xdata) {
+        return 0;
+    }
 
-        return NULL;
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&c_iatt);
+    if (ret < 0) {
+        return 0;
+    }
+
+    o_iatt = GF_CALLOC(1, sizeof(struct old_iatt), gf_common_mt_char);
+    if (!o_iatt) {
+        return -1;
+    }
+
+    oldiatt_from_iatt(o_iatt, c_iatt);
+
+    ret = dict_set_bin(xdata, DHT_IATT_IN_XDATA_KEY, o_iatt,
+                       sizeof(struct old_iatt));
+    if (ret) {
+        GF_FREE(o_iatt);
+    }
+
+    return ret;
+}
+
+int
+gf_replace_new_iatt_in_dict(dict_t *xdata)
+{
+    int ret;
+    struct old_iatt *o_iatt; /* old iatt structure */
+    struct iatt *c_iatt;     /* new iatt */
+
+    if (!xdata) {
+        return 0;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&o_iatt);
+    if (ret < 0) {
+        return 0;
+    }
+
+    c_iatt = GF_CALLOC(1, sizeof(struct iatt), gf_common_mt_char);
+    if (!c_iatt) {
+        return -1;
+    }
+
+    iatt_from_oldiatt(c_iatt, o_iatt);
+
+    ret = dict_set_bin(xdata, DHT_IATT_IN_XDATA_KEY, c_iatt,
+                       sizeof(struct iatt));
+    if (ret) {
+        GF_FREE(c_iatt);
+    }
+
+    return ret;
+}
+
+xlator_cmdline_option_t *
+find_xlator_option_in_cmd_args_t(const char *option_name, cmd_args_t *args)
+{
+    xlator_cmdline_option_t *pos = NULL;
+    xlator_cmdline_option_t *tmp = NULL;
+
+    list_for_each_entry_safe(pos, tmp, &args->xlator_options, cmd_args)
+    {
+        if (strcmp(pos->key, option_name) == 0)
+            return pos;
+    }
+    return NULL;
+}
+
+int
+gf_d_type_from_ia_type(ia_type_t type)
+{
+    switch (type) {
+        case IA_IFDIR:
+            return DT_DIR;
+        case IA_IFCHR:
+            return DT_CHR;
+        case IA_IFBLK:
+            return DT_BLK;
+        case IA_IFIFO:
+            return DT_FIFO;
+        case IA_IFLNK:
+            return DT_LNK;
+        case IA_IFREG:
+            return DT_REG;
+        case IA_IFSOCK:
+            return DT_SOCK;
+        default:
+            return DT_UNKNOWN;
+    }
+}
+
+int
+gf_nanosleep(uint64_t nsec)
+{
+    struct timespec req;
+    struct timespec rem;
+    int ret = -1;
+
+    req.tv_sec = nsec / GF_SEC_IN_NS;
+    req.tv_nsec = nsec % GF_SEC_IN_NS;
+
+    do {
+        ret = nanosleep(&req, &rem);
+        req = rem;
+    } while (ret == -1 && errno == EINTR);
+
+    return ret;
+}
+
+int
+gf_syncfs(int fd)
+{
+    int ret = 0;
+#if defined(HAVE_SYNCFS)
+    /* Linux with glibc recent enough. */
+    ret = syncfs(fd);
+#elif defined(HAVE_SYNCFS_SYS)
+    /* Linux with no library function. */
+    ret = syscall(SYS_syncfs, fd);
+#else
+    /* Fallback to generic UNIX stuff. */
+    sync();
+#endif
+    return ret;
+}
+
+char **
+get_xattrs_to_heal()
+{
+    return xattrs_to_heal;
 }
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
deleted file mode 100644
index 4371888d034..00000000000
--- a/libglusterfs/src/common-utils.h
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _COMMON_UTILS_H
-#define _COMMON_UTILS_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <sys/uio.h>
-#include <netdb.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <pthread.h>
-#ifndef GF_BSD_HOST_OS
-#include <alloca.h>
-#endif 
-
-void trap (void);
-
-#define GF_UNIVERSAL_ANSWER 42    /* :O */
-
-/* To solve type punned error */
-#define VOID(ptr) ((void **) ((void *) ptr))
-
-#include "logging.h"
-#include "glusterfs.h"
-#include "locking.h"
-#include "mem-pool.h"
-
-
-#define min(a,b) ((a)<(b)?(a):(b))
-#define max(a,b) ((a)>(b)?(a):(b))
-#define roof(a,b) ((((a)+(b)-1)/((b)?(b):1))*(b))
-#define floor(a,b) (((a)/((b)?(b):1))*(b))
-
-
-#define GF_UNIT_KB    1024ULL
-#define GF_UNIT_MB    1048576ULL
-#define GF_UNIT_GB    1073741824ULL
-#define GF_UNIT_TB    1099511627776ULL
-#define GF_UNIT_PB    1125899906842624ULL
-
-#define GF_UNIT_KB_STRING    "KB"
-#define GF_UNIT_MB_STRING    "MB"
-#define GF_UNIT_GB_STRING    "GB"
-#define GF_UNIT_TB_STRING    "TB"
-#define GF_UNIT_PB_STRING    "PB"
-
-
-enum _gf_boolean 
-{
-	_gf_false = 0, 
-	_gf_true = 1
-};
-
-typedef enum _gf_boolean gf_boolean_t;
-
-void gf_global_variable_init(void);
-
-in_addr_t gf_resolve_ip (const char *hostname, void **dnscache);
-
-void gf_log_volume_file (FILE *specfp);
-void gf_print_trace (int32_t signal);
-
-extern char *gf_fop_list[GF_FOP_MAXVALUE];
-extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
-
-#define VECTORSIZE(count) (count * (sizeof (struct iovec)))
-
-#define STRLEN_0(str) (strlen(str) + 1)
-#define VALIDATE_OR_GOTO(arg,label)   do {				\
-		if (!arg) {						\
-			errno = EINVAL;					\
-			gf_log ((this ? this->name : "(Govinda! Govinda!)"), \
-				GF_LOG_ERROR,				\
-				"invalid argument: " #arg);		\
-			goto label;					\
-		}							\
-	} while (0); 
-
-#define GF_VALIDATE_OR_GOTO(name,arg,label)   do {		\
-		if (!arg) {					\
-			errno = EINVAL;   			\
-			gf_log (name, GF_LOG_ERROR,		\
-				"invalid argument: " #arg);	\
-			goto label;				\
-		}						\
-	} while (0); 
-
-#define GF_VALIDATE_OR_GOTO_WITH_ERROR(name, arg, label, errno, error) do { \
-                if (!arg) {                                                 \
-                        errno = error;                                  \
-                        gf_log (name, GF_LOG_ERROR,                     \
-                                "invalid argument: " #arg);             \
-                        goto label;                                     \
-                }                                                       \
-        }while (0);
-
-#define GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO(name,arg,label)       \
-        do {                                                    \
-                GF_VALIDATE_OR_GOTO (name, arg, label);         \
-                if ((arg[0]) != '/') {                          \
-                        errno = EINVAL;                         \
-			gf_log (name, GF_LOG_ERROR,	        \
-				"invalid argument: " #arg);	\
-                        goto label;                             \
-                }                                               \
-	} while (0);
-
-#define GF_FILE_CONTENT_REQUESTED(_xattr_req,_content_limit) \
-	(dict_get_uint64 (_xattr_req, "glusterfs.content", _content_limit) == 0)
-
-#define GF_ASSERT(x) \
-        do {                                                    \
-                if (!(x)) {                                     \
-                        gf_log_callingfn ("", GF_LOG_ERROR,     \
-                                  "Assertion failed: " #x);     \
-                }                                               \
-        } while (0);
-
-static inline void
-iov_free (struct iovec *vector, int count)
-{
-	int i;
-
-	for (i = 0; i < count; i++)
-		FREE (vector[i].iov_base);
-
-	GF_FREE (vector);
-}
-
-
-static inline int
-iov_length (const struct iovec *vector, int count)
-{
-	int     i = 0;
-	size_t  size = 0;
-
-	for (i = 0; i < count; i++)
-		size += vector[i].iov_len;
-
-	return size;
-}
-
-
-static inline struct iovec *
-iov_dup (struct iovec *vector, int count)
-{
-	int           bytecount = 0;
-	int           i;
-	struct iovec *newvec = NULL;
-
-	bytecount = (count * sizeof (struct iovec));
-	newvec = GF_MALLOC (bytecount, gf_common_mt_iovec);
-	if (!newvec)
-		return NULL;
-
-	for (i = 0; i < count; i++) {
-		newvec[i].iov_len  = vector[i].iov_len;
-		newvec[i].iov_base = vector[i].iov_base;
-	}
-
-	return newvec;
-}
-
-
-static inline int
-iov_subset (struct iovec *orig, int orig_count,
-	    off_t src_offset, off_t dst_offset,
-	    struct iovec *new)
-{
-	int    new_count = 0;
-	int    i;
-	off_t  offset = 0;
-	size_t start_offset = 0;
-	size_t end_offset = 0;
-
-
-	for (i = 0; i < orig_count; i++) {
-		if ((offset + orig[i].iov_len < src_offset)
-		    || (offset > dst_offset)) {
-			goto not_subset;
-		}
-
-		if (!new) {
-			goto count_only;
-		}
-
-		start_offset = 0;
-		end_offset = orig[i].iov_len;
-
-		if (src_offset >= offset) {
-			start_offset = (src_offset - offset);
-		}
-
-		if (dst_offset <= (offset + orig[i].iov_len)) {
-			end_offset = (dst_offset - offset);
-		}
-
-		new[new_count].iov_base = orig[i].iov_base + start_offset;
-		new[new_count].iov_len = end_offset - start_offset;
-
-	count_only:
-		new_count++;
-
-	not_subset:
-		offset += orig[i].iov_len;
-	}
-
-	return new_count;
-}
-
-
-static inline void
-iov_unload (char *buf, const struct iovec *vector, int count)
-{
-	int i;
-	int copied = 0;
-
-	for (i = 0; i < count; i++) {
-		memcpy (buf + copied, vector[i].iov_base, vector[i].iov_len);
-		copied += vector[i].iov_len;
-	}
-}
-
-
-static inline int
-mem_0filled (const char *buf, size_t size)
-{
-	int i = 0;
-	int ret = 0;
-
-	for (i = 0; i < size; i++) {
-		ret = buf[i];
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-
-static inline int
-iov_0filled (struct iovec *vector, int count)
-{
-	int i = 0;
-	int ret = 0;
-
-	for (i = 0; i < count; i++) {
-		ret = mem_0filled (vector[i].iov_base, vector[i].iov_len);
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-
-static inline void *
-memdup (const void *ptr, size_t size)
-{
-	void *newptr = NULL;
-
-	newptr = GF_MALLOC (size, gf_common_mt_memdup);
-	if (!newptr)
-		return NULL;
-
-	memcpy (newptr, ptr, size);
-	return newptr;
-}
-
-
-char *gf_trim (char *string);
-int gf_strsplit (const char *str, const char *delim, 
-		 char ***tokens, int *token_count);
-int gf_volume_name_validate (const char *volume_name);
-
-int gf_string2long (const char *str, long *n);
-int gf_string2ulong (const char *str, unsigned long *n);
-int gf_string2int (const char *str, int *n);
-int gf_string2uint (const char *str, unsigned int *n);
-int gf_string2double (const char *str, double *n);
-int gf_string2longlong (const char *str, long long *n);
-int gf_string2ulonglong (const char *str, unsigned long long *n);
-
-int gf_string2int8 (const char *str, int8_t *n);
-int gf_string2int16 (const char *str, int16_t *n);
-int gf_string2int32 (const char *str, int32_t *n);
-int gf_string2int64 (const char *str, int64_t *n);
-int gf_string2uint8 (const char *str, uint8_t *n);
-int gf_string2uint16 (const char *str, uint16_t *n);
-int gf_string2uint32 (const char *str, uint32_t *n);
-int gf_string2uint64 (const char *str, uint64_t *n);
-
-int gf_strstr (const char *str, const char *delim, const char *match);
-
-int gf_string2ulong_base10 (const char *str, unsigned long *n);
-int gf_string2uint_base10 (const char *str, unsigned int *n);
-int gf_string2uint8_base10 (const char *str, uint8_t *n);
-int gf_string2uint16_base10 (const char *str, uint16_t *n);
-int gf_string2uint32_base10 (const char *str, uint32_t *n);
-int gf_string2uint64_base10 (const char *str, uint64_t *n);
-
-int gf_string2bytesize (const char *str, uint64_t *n);
-
-int gf_string2boolean (const char *str, gf_boolean_t *b);
-int gf_string2percent (const char *str, uint32_t *n);
-int gf_string2time (const char *str, uint32_t *n);
-
-int gf_lockfd (int fd);
-int gf_unlockfd (int fd);
-
-int get_checksum_for_file (int fd, uint32_t *checksum);
-int log_base2 (unsigned long x);
-
-int gf_system (const char *command);
-int get_checksum_for_path (char *path, uint32_t *checksum);
-
-char *strtail (char *str, const char *pattern);
-#endif /* _COMMON_UTILS_H */
-
diff --git a/libglusterfs/src/compat-errno.c b/libglusterfs/src/compat-errno.c
index 7b39b0a8da5..df57e243239 100644
--- a/libglusterfs/src/compat-errno.c
+++ b/libglusterfs/src/compat-errno.c
@@ -1,938 +1,955 @@
 /*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
 #include <stdint.h>
 
-#include "compat-errno.h"
-
+#include "glusterfs/compat-errno.h"
 
-static int32_t gf_error_to_errno_array[1024]; 
+static int32_t gf_error_to_errno_array[1024];
 static int32_t gf_errno_to_error_array[1024];
 
 static int32_t gf_compat_errno_init_done;
 
 #ifdef GF_SOLARIS_HOST_OS
-static void 
-init_compat_errno_arrays ()
+static void
+init_compat_errno_arrays()
 {
-/*  	ENOMSG	35	/ * No message of desired type		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-  gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-/*  	EIDRM	36	/ * Identifier removed			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-  gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-
-/*  	ECHRNG	37	/ * Channel number out of range		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_CHRNG] = ECHRNG;
-  gf_errno_to_error_array[ECHRNG] = GF_ERROR_CODE_CHRNG;
-
-/*  	EL2NSYNC 38	/ * Level 2 not synchronized		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_L2NSYNC] = EL2NSYNC;
-  gf_errno_to_error_array[EL2NSYNC] = GF_ERROR_CODE_L2NSYNC;
-
-/*  	EL3HLT	39	/ * Level 3 halted			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_L3HLT] = EL3HLT;
-  gf_errno_to_error_array[EL3HLT] = GF_ERROR_CODE_L3HLT;
-
-/*  	EL3RST	40	/ * Level 3 reset			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_L3RST] = EL3RST;
-  gf_errno_to_error_array[EL3RST] = GF_ERROR_CODE_L3RST;
-
-/*  	ELNRNG	41	/ * Link number out of range		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LNRNG] = ELNRNG;
-  gf_errno_to_error_array[ELNRNG] = GF_ERROR_CODE_LNRNG;
-
-/*  	EUNATCH 42	/ * Protocol driver not attached		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_UNATCH] = EUNATCH;
-  gf_errno_to_error_array[EUNATCH] = GF_ERROR_CODE_UNATCH;
-
-/*  	ENOCSI	43	/ * No CSI structure available		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOCSI] = ENOCSI;
-  gf_errno_to_error_array[ENOCSI] = GF_ERROR_CODE_NOCSI;
-
-/*  	EL2HLT	44	/ * Level 2 halted			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_L2HLT] = EL2HLT;
-  gf_errno_to_error_array[EL2HLT] = GF_ERROR_CODE_L2HLT;
-
-/*  	EDEADLK	45	/ * Deadlock condition.			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-  gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-
-/*  	ENOLCK	46	/ * No record locks available.		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-  gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-/*  	ECANCELED 47	/ * Operation canceled			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-  gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-/*  	ENOTSUP	48	/ * Operation not supported		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTSUPP] = ENOTSUP;
-  gf_errno_to_error_array[ENOTSUP] = GF_ERROR_CODE_NOTSUPP;
-
-/* Filesystem Quotas */
-/*  	EDQUOT	49	/ * Disc quota exceeded			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-  gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-/* Convergent Error Returns */
-/*  	EBADE	50	/ * invalid exchange			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADE] = EBADE;
-  gf_errno_to_error_array[EBADE] = GF_ERROR_CODE_BADE;
-/*  	EBADR	51	/ * invalid request descriptor		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADR] = EBADR;
-  gf_errno_to_error_array[EBADR] = GF_ERROR_CODE_BADR;
-/*  	EXFULL	52	/ * exchange full			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_XFULL] = EXFULL;
-  gf_errno_to_error_array[EXFULL] = GF_ERROR_CODE_XFULL;
-/*  	ENOANO	53	/ * no anode				*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOANO] = ENOANO;
-  gf_errno_to_error_array[ENOANO] = GF_ERROR_CODE_NOANO;
-/*  	EBADRQC	54	/ * invalid request code			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADRQC] = EBADRQC;
-  gf_errno_to_error_array[EBADRQC] = GF_ERROR_CODE_BADRQC;
-/*  	EBADSLT	55	/ * invalid slot				*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADSLT] = EBADSLT;
-  gf_errno_to_error_array[EBADSLT] = GF_ERROR_CODE_BADSLT;
-/*  	EDEADLOCK 56	/ * file locking deadlock error		*/
-/* This is same as EDEADLK on linux */
-  gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLOCK;
-  gf_errno_to_error_array[EDEADLOCK] = GF_ERROR_CODE_DEADLK;
-
-/*  	EBFONT	57	/ * bad font file fmt			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BFONT] = EBFONT;
-  gf_errno_to_error_array[EBFONT] = GF_ERROR_CODE_BFONT;
-
-/* Interprocess Robust Locks */
-/*  	EOWNERDEAD	58	/ * process died with the lock */
-  gf_error_to_errno_array[GF_ERROR_CODE_OWNERDEAD] = EOWNERDEAD;
-  gf_errno_to_error_array[EOWNERDEAD] = GF_ERROR_CODE_OWNERDEAD;
-/*  	ENOTRECOVERABLE	59	/ * lock is not recoverable */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTRECOVERABLE] = ENOTRECOVERABLE;
-  gf_errno_to_error_array[ENOTRECOVERABLE] = GF_ERROR_CODE_NOTRECOVERABLE;
-
-/* stream problems */
-/*  	ENOSTR	60	/ * Device not a stream			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
-  gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
-/*  	ENODATA	61	/ * no data (for no delay io)		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
-  gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
-/*  	ETIME	62	/ * timer expired			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
-  gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
-/*  	ENOSR	63	/ * out of streams resources		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
-  gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
-
-/*  	ENONET	64	/ * Machine is not on the network	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NONET] = ENONET;
-  gf_errno_to_error_array[ENONET] = GF_ERROR_CODE_NONET;
-/*  	ENOPKG	65	/ * Package not installed		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOPKG] = ENOPKG;
-  gf_errno_to_error_array[ENOPKG] = GF_ERROR_CODE_NOPKG;
-/*  	EREMOTE	66	/ * The object is remote			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-  gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-/*  	ENOLINK	67	/ * the link has been severed		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-  gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
-/*  	EADV	68	/ * advertise error			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_ADV] = EADV;
-  gf_errno_to_error_array[EADV] = GF_ERROR_CODE_ADV;
-/*  	ESRMNT	69	/ * srmount error			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_SRMNT] = ESRMNT;
-  gf_errno_to_error_array[ESRMNT] = GF_ERROR_CODE_SRMNT;
-
-/*  	ECOMM	70	/ * Communication error on send		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_COMM] = ECOMM;
-  gf_errno_to_error_array[ECOMM] = GF_ERROR_CODE_COMM;
-/*  	EPROTO	71	/ * Protocol error			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-  gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
-
-/* Interprocess Robust Locks */
-/*  	ELOCKUNMAPPED	72	/ * locked lock was unmapped */
-  gf_error_to_errno_array[GF_ERROR_CODE_LOCKUNMAPPED] = ELOCKUNMAPPED;
-  gf_errno_to_error_array[ELOCKUNMAPPED] = GF_ERROR_CODE_LOCKUNMAPPED;
-
-/*  	ENOTACTIVE 73	/ * Facility is not active		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTACTIVE] = ENOTACTIVE;
-  gf_errno_to_error_array[ENOTACTIVE] = GF_ERROR_CODE_NOTACTIVE;
-/*  	EMULTIHOP 74	/ * multihop attempted			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-  gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
-/*  	EBADMSG 77	/ * trying to read unreadable message	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-  gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
-/*  	ENAMETOOLONG 78	/ * path name is too long		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-  gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-/*  	EOVERFLOW 79	/ * value too large to be stored in data type */
-  gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-  gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-/*  	ENOTUNIQ 80	/ * given log. name not unique		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTUNIQ] = ENOTUNIQ;
-  gf_errno_to_error_array[ENOTUNIQ] = GF_ERROR_CODE_NOTUNIQ;
-/*  	EBADFD	81	/ * f.d. invalid for this operation	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_BADFD] = EBADFD;
-  gf_errno_to_error_array[EBADFD] = GF_ERROR_CODE_BADFD;
-/*  	EREMCHG	82	/ * Remote address changed		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_REMCHG] = EREMCHG;
-  gf_errno_to_error_array[EREMCHG] = GF_ERROR_CODE_REMCHG;
-
-/* shared library problems */
-/*  	ELIBACC	83	/ * Can't access a needed shared lib.	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LIBACC] = ELIBACC;
-  gf_errno_to_error_array[ELIBACC] = GF_ERROR_CODE_LIBACC;
-/*  	ELIBBAD	84	/ * Accessing a corrupted shared lib.	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LIBBAD] = ELIBBAD;
-  gf_errno_to_error_array[ELIBBAD] = GF_ERROR_CODE_LIBBAD;
-/*  	ELIBSCN	85	/ * .lib section in a.out corrupted.	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LIBSCN] = ELIBSCN;
-  gf_errno_to_error_array[ELIBSCN] = GF_ERROR_CODE_LIBSCN;
-/*  	ELIBMAX	86	/ * Attempting to link in too many libs.	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LIBMAX] = ELIBMAX;
-  gf_errno_to_error_array[ELIBMAX] = GF_ERROR_CODE_LIBMAX;
-/*  	ELIBEXEC 87	/ * Attempting to exec a shared library.	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LIBEXEC] = ELIBEXEC;
-  gf_errno_to_error_array[ELIBEXEC] = GF_ERROR_CODE_LIBEXEC;
-/*  	EILSEQ	88	/ * Illegal byte sequence.		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-  gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-/*  	ENOSYS	89	/ * Unsupported file system operation	*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-  gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-/*  	ELOOP	90	/ * Symbolic link loop			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-  gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-/*  	ERESTART 91	/ * Restartable system call		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_RESTART] = ERESTART;
-  gf_errno_to_error_array[ERESTART] = GF_ERROR_CODE_RESTART;
-/*  	ESTRPIPE 92	/ * if pipe/FIFO, don't sleep in stream head */
-  gf_error_to_errno_array[GF_ERROR_CODE_STRPIPE] = ESTRPIPE;
-  gf_errno_to_error_array[ESTRPIPE] = GF_ERROR_CODE_STRPIPE;
-/*  	ENOTEMPTY 93	/ * directory not empty			*/
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-  gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-/*  	EUSERS	94	/ * Too many users (for UFS)		*/
-  gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-  gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-/* BSD Networking Software */
-	/* argument errors */
-/*  	ENOTSOCK	95	/ * Socket operation on non-socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-  gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-/*  	EDESTADDRREQ	96	/ * Destination address required */
-  gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-  gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-/*  	EMSGSIZE	97	/ * Message too long */
-  gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-  gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-/*  	EPROTOTYPE	98	/ * Protocol wrong type for socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-  gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-/*  	ENOPROTOOPT	99	/ * Protocol not available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-  gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-/*  	EPROTONOSUPPORT	120	/ * Protocol not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-  gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-/*  	ESOCKTNOSUPPORT	121	/ * Socket type not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-  gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-/*  	EOPNOTSUPP	122	/ * Operation not supported on socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-  gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-/*  	EPFNOSUPPORT	123	/ * Protocol family not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-  gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-/*  	EAFNOSUPPORT	124	/ * Address family not supported by */
-				/* protocol family */
-  gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-  gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
-/*  	EADDRINUSE	125	/ * Address already in use */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-  gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
-/*  	EADDRNOTAVAIL	126	/ * Can't assign requested address */
-                                /* operational errors */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-  gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
-/*  	ENETDOWN	127	/ * Network is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-  gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
-/*  	ENETUNREACH	128	/ * Network is unreachable */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-  gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
-/*  	ENETRESET	129	/ * Network dropped connection because */
-				/* of reset */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-  gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
-/*  	ECONNABORTED	130	/ * Software caused connection abort */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-  gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
-/*  	ECONNRESET	131	/ * Connection reset by peer */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-  gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-/*  	ENOBUFS		132	/ * No buffer space available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-  gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-/*  	EISCONN		133	/ * Socket is already connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-  gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-/*  	ENOTCONN	134	/ * Socket is not connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-  gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-/* XENIX has 135 - 142 */
-/*  	ESHUTDOWN	143	/ * Can't send after socket shutdown */
-  gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-  gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-/*  	ETOOMANYREFS	144	/ * Too many references: can't splice */
-  gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-  gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-/*  	ETIMEDOUT	145	/ * Connection timed out */
-  gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-  gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-/*  	ECONNREFUSED	146	/ * Connection refused */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-  gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-/*  	EHOSTDOWN	147	/ * Host is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-  gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-/*  	EHOSTUNREACH	148	/ * No route to host */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-  gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-/*  	EALREADY	149	/ * operation already in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-  gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-/*  	EINPROGRESS	150	/ * operation now in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-  gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-/* SUN Network File System */
-/*  	ESTALE		151	/ * Stale NFS file handle */
-  gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-  gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-  return ;
+    /*      ENOMSG  35      / * No message of desired type          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
+
+    /*      EIDRM   36      / * Identifier removed                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
+
+    /*      ECHRNG  37      / * Channel number out of range         */
+    gf_error_to_errno_array[GF_ERROR_CODE_CHRNG] = ECHRNG;
+    gf_errno_to_error_array[ECHRNG] = GF_ERROR_CODE_CHRNG;
+
+    /*      EL2NSYNC 38     / * Level 2 not synchronized            */
+    gf_error_to_errno_array[GF_ERROR_CODE_L2NSYNC] = EL2NSYNC;
+    gf_errno_to_error_array[EL2NSYNC] = GF_ERROR_CODE_L2NSYNC;
+
+    /*      EL3HLT  39      / * Level 3 halted                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_L3HLT] = EL3HLT;
+    gf_errno_to_error_array[EL3HLT] = GF_ERROR_CODE_L3HLT;
+
+    /*      EL3RST  40      / * Level 3 reset                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_L3RST] = EL3RST;
+    gf_errno_to_error_array[EL3RST] = GF_ERROR_CODE_L3RST;
+
+    /*      ELNRNG  41      / * Link number out of range            */
+    gf_error_to_errno_array[GF_ERROR_CODE_LNRNG] = ELNRNG;
+    gf_errno_to_error_array[ELNRNG] = GF_ERROR_CODE_LNRNG;
+
+    /*      EUNATCH 42      / * Protocol driver not attached                */
+    gf_error_to_errno_array[GF_ERROR_CODE_UNATCH] = EUNATCH;
+    gf_errno_to_error_array[EUNATCH] = GF_ERROR_CODE_UNATCH;
+
+    /*      ENOCSI  43      / * No CSI structure available          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOCSI] = ENOCSI;
+    gf_errno_to_error_array[ENOCSI] = GF_ERROR_CODE_NOCSI;
+
+    /*      EL2HLT  44      / * Level 2 halted                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_L2HLT] = EL2HLT;
+    gf_errno_to_error_array[EL2HLT] = GF_ERROR_CODE_L2HLT;
+
+    /*      EDEADLK 45      / * Deadlock condition.                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*      ENOLCK  46      / * No record locks available.          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
+
+    /*      ECANCELED 47    / * Operation canceled                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
+
+    /*      ENOTSUP 48      / * Operation not supported             */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSUPP] = ENOTSUP;
+    gf_errno_to_error_array[ENOTSUP] = GF_ERROR_CODE_NOTSUPP;
+
+    /* Filesystem Quotas */
+    /*      EDQUOT  49      / * Disc quota exceeded                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /* Convergent Error Returns */
+    /*      EBADE   50      / * invalid exchange                    */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADE] = EBADE;
+    gf_errno_to_error_array[EBADE] = GF_ERROR_CODE_BADE;
+    /*      EBADR   51      / * invalid request descriptor          */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADR] = EBADR;
+    gf_errno_to_error_array[EBADR] = GF_ERROR_CODE_BADR;
+    /*      EXFULL  52      / * exchange full                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_XFULL] = EXFULL;
+    gf_errno_to_error_array[EXFULL] = GF_ERROR_CODE_XFULL;
+    /*      ENOANO  53      / * no anode                            */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOANO] = ENOANO;
+    gf_errno_to_error_array[ENOANO] = GF_ERROR_CODE_NOANO;
+    /*      EBADRQC 54      / * invalid request code                        */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRQC] = EBADRQC;
+    gf_errno_to_error_array[EBADRQC] = GF_ERROR_CODE_BADRQC;
+    /*      EBADSLT 55      / * invalid slot                                */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADSLT] = EBADSLT;
+    gf_errno_to_error_array[EBADSLT] = GF_ERROR_CODE_BADSLT;
+    /*      EDEADLOCK 56    / * file locking deadlock error         */
+    /* This is same as EDEADLK on linux */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLOCK;
+    gf_errno_to_error_array[EDEADLOCK] = GF_ERROR_CODE_DEADLK;
+
+    /*      EBFONT  57      / * bad font file fmt                   */
+    gf_error_to_errno_array[GF_ERROR_CODE_BFONT] = EBFONT;
+    gf_errno_to_error_array[EBFONT] = GF_ERROR_CODE_BFONT;
+
+    /* Interprocess Robust Locks */
+    /*      EOWNERDEAD      58      / * process died with the lock */
+    gf_error_to_errno_array[GF_ERROR_CODE_OWNERDEAD] = EOWNERDEAD;
+    gf_errno_to_error_array[EOWNERDEAD] = GF_ERROR_CODE_OWNERDEAD;
+    /*      ENOTRECOVERABLE 59      / * lock is not recoverable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTRECOVERABLE] = ENOTRECOVERABLE;
+    gf_errno_to_error_array[ENOTRECOVERABLE] = GF_ERROR_CODE_NOTRECOVERABLE;
+
+    /* stream problems */
+    /*      ENOSTR  60      / * Device not a stream                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
+    gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
+    /*      ENODATA 61      / * no data (for no delay io)           */
+    gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
+    gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
+    /*      ETIME   62      / * timer expired                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
+    gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
+    /*      ENOSR   63      / * out of streams resources            */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
+    gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
+
+    /*      ENONET  64      / * Machine is not on the network       */
+    gf_error_to_errno_array[GF_ERROR_CODE_NONET] = ENONET;
+    gf_errno_to_error_array[ENONET] = GF_ERROR_CODE_NONET;
+    /*      ENOPKG  65      / * Package not installed               */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPKG] = ENOPKG;
+    gf_errno_to_error_array[ENOPKG] = GF_ERROR_CODE_NOPKG;
+    /*      EREMOTE 66      / * The object is remote                        */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+    /*      ENOLINK 67      / * the link has been severed           */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+    /*      EADV    68      / * advertise error                     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADV] = EADV;
+    gf_errno_to_error_array[EADV] = GF_ERROR_CODE_ADV;
+    /*      ESRMNT  69      / * srmount error                       */
+    gf_error_to_errno_array[GF_ERROR_CODE_SRMNT] = ESRMNT;
+    gf_errno_to_error_array[ESRMNT] = GF_ERROR_CODE_SRMNT;
+
+    /*      ECOMM   70      / * Communication error on send         */
+    gf_error_to_errno_array[GF_ERROR_CODE_COMM] = ECOMM;
+    gf_errno_to_error_array[ECOMM] = GF_ERROR_CODE_COMM;
+    /*      EPROTO  71      / * Protocol error                      */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+
+    /* Interprocess Robust Locks */
+    /*      ELOCKUNMAPPED   72      / * locked lock was unmapped */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOCKUNMAPPED] = ELOCKUNMAPPED;
+    gf_errno_to_error_array[ELOCKUNMAPPED] = GF_ERROR_CODE_LOCKUNMAPPED;
+
+    /*      ENOTACTIVE 73   / * Facility is not active              */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTACTIVE] = ENOTACTIVE;
+    gf_errno_to_error_array[ENOTACTIVE] = GF_ERROR_CODE_NOTACTIVE;
+    /*      EMULTIHOP 74    / * multihop attempted                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
+    /*      EBADMSG 77      / * trying to read unreadable message   */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+    /*      ENAMETOOLONG 78 / * path name is too long               */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+    /*      EOVERFLOW 79    / * value too large to be stored in data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
+    /*      ENOTUNIQ 80     / * given log. name not unique          */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTUNIQ] = ENOTUNIQ;
+    gf_errno_to_error_array[ENOTUNIQ] = GF_ERROR_CODE_NOTUNIQ;
+    /*      EBADFD  81      / * f.d. invalid for this operation     */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADFD] = EBADFD;
+    gf_errno_to_error_array[EBADFD] = GF_ERROR_CODE_BADFD;
+    /*      EREMCHG 82      / * Remote address changed              */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMCHG] = EREMCHG;
+    gf_errno_to_error_array[EREMCHG] = GF_ERROR_CODE_REMCHG;
+
+    /* shared library problems */
+    /*      ELIBACC 83      / * Can't access a needed shared lib.   */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBACC] = ELIBACC;
+    gf_errno_to_error_array[ELIBACC] = GF_ERROR_CODE_LIBACC;
+    /*      ELIBBAD 84      / * Accessing a corrupted shared lib.   */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBBAD] = ELIBBAD;
+    gf_errno_to_error_array[ELIBBAD] = GF_ERROR_CODE_LIBBAD;
+    /*      ELIBSCN 85      / * .lib section in a.out corrupted.    */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBSCN] = ELIBSCN;
+    gf_errno_to_error_array[ELIBSCN] = GF_ERROR_CODE_LIBSCN;
+    /*      ELIBMAX 86      / * Attempting to link in too many libs.        */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBMAX] = ELIBMAX;
+    gf_errno_to_error_array[ELIBMAX] = GF_ERROR_CODE_LIBMAX;
+    /*      ELIBEXEC 87     / * Attempting to exec a shared library.        */
+    gf_error_to_errno_array[GF_ERROR_CODE_LIBEXEC] = ELIBEXEC;
+    gf_errno_to_error_array[ELIBEXEC] = GF_ERROR_CODE_LIBEXEC;
+    /*      EILSEQ  88      / * Illegal byte sequence.              */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
+    /*      ENOSYS  89      / * Unsupported file system operation   */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
+    /*      ELOOP   90      / * Symbolic link loop                  */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+    /*      ERESTART 91     / * Restartable system call             */
+    gf_error_to_errno_array[GF_ERROR_CODE_RESTART] = ERESTART;
+    gf_errno_to_error_array[ERESTART] = GF_ERROR_CODE_RESTART;
+    /*      ESTRPIPE 92     / * if pipe/FIFO, don't sleep in stream head */
+    gf_error_to_errno_array[GF_ERROR_CODE_STRPIPE] = ESTRPIPE;
+    gf_errno_to_error_array[ESTRPIPE] = GF_ERROR_CODE_STRPIPE;
+    /*      ENOTEMPTY 93    / * directory not empty                 */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+    /*      EUSERS  94      / * Too many users (for UFS)            */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /* BSD Networking Software */
+    /* argument errors */
+    /*      ENOTSOCK        95      / * Socket operation on non-socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+    /*      EDESTADDRREQ    96      / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+    /*      EMSGSIZE        97      / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+    /*      EPROTOTYPE      98      / * Protocol wrong type for socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+    /*      ENOPROTOOPT     99      / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+    /*      EPROTONOSUPPORT 120     / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+    /*      ESOCKTNOSUPPORT 121     / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      122     / * Operation not supported on socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+    /*      EPFNOSUPPORT    123     / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+    /*      EAFNOSUPPORT    124     / * Address family not supported by */
+    /* protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+    /*      EADDRINUSE      125     / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+    /*      EADDRNOTAVAIL   126     / * Can't assign requested address */
+    /* operational errors */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+    /*      ENETDOWN        127     / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+    /*      ENETUNREACH     128     / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+    /*      ENETRESET       129     / * Network dropped connection because */
+    /* of reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+    /*      ECONNABORTED    130     / * Software caused connection abort */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+    /*      ECONNRESET      131     / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+    /*      ENOBUFS         132     / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+    /*      EISCONN         133     / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+    /*      ENOTCONN        134     / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+    /* XENIX has 135 - 142 */
+    /*      ESHUTDOWN       143     / * Can't send after socket shutdown */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+    /*      ETOOMANYREFS    144     / * Too many references: can't splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+    /*      ETIMEDOUT       145     / * Connection timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    146     / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+    /*      EHOSTDOWN       147     / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+    /*      EHOSTUNREACH    148     / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+    /*      EALREADY        149     / * operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+    /*      EINPROGRESS     150     / * operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /* SUN Network File System */
+    /*      ESTALE          151     / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    return;
 }
 #endif /* GF_SOLARIS_HOST_OS */
 
 #ifdef GF_DARWIN_HOST_OS
-static void 
-init_compat_errno_arrays ()
+static void
+init_compat_errno_arrays()
 {
-  /*    EDEADLK         11              / * Resource deadlock would occur */
-  gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-  gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-
-  /*    EAGAIN          35              / * Try Again */
-  gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
-  gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
-  
-  /*	EINPROGRESS	36	        / * Operation now in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-  gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-  /*	EALREADY	37		/ * Operation already in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-  gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-  
-  /*	ENOTSOCK	38		/ * Socket operation on non-socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-  gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-  
-  /*	EDESTADDRREQ	39		/ * Destination address required */
-  gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-  gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-  
-  /*	EMSGSIZE	40		/ * Message too long */
-  gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-  gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-  
-  /*	EPROTOTYPE	41		/ * Protocol wrong type for socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-  gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-  
-  /*	ENOPROTOOPT	42		/ * Protocol not available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-  gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-  
-  /*	EPROTONOSUPPORT	43		/ * Protocol not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-  gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-  
-  /*	ESOCKTNOSUPPORT	44		/ * Socket type not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-  gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-  /*	EOPNOTSUPP	45		/ * Operation not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-  gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-  /*	EPFNOSUPPORT	46		/ * Protocol family not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-  gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-
-  /*	EAFNOSUPPORT	47		/ * Address family not supported by protocol family */
-  gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-  gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
-
-  /*	EADDRINUSE	48		/ * Address already in use */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-  gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
-
-  /*	EADDRNOTAVAIL	49		/ * Can't assign requested address */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-  gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
-
-  /*	ENETDOWN	50		/ * Network is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-  gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
-
-  /*	ENETUNREACH	51		/ * Network is unreachable */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-  gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
-
-  /*	ENETRESET	52		/ * Network dropped connection on reset */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-  gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
-
-  /*	ECONNABORTED	53		/ * Software caused connection abort */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-  gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
-
-  /*	ECONNRESET	54		/ * Connection reset by peer */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-  gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-  
-  /*	ENOBUFS		55		/ * No buffer space available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-  gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-
-  /*	EISCONN		56		/ * Socket is already connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-  gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-
-  /*	ENOTCONN	57		/ * Socket is not connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-  gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-
-  /*	ESHUTDOWN	58		/ * Can't send after socket shutdown */
-  gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-  gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-
-  /*	ETOOMANYREFS	59		/ * Too many references: can't splice */
-  gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-  gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-
-  /*	ETIMEDOUT	60		/ * Operation timed out */
-  gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-  gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-  /*	ECONNREFUSED	61		/ * Connection refused */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-  gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-
-  /*	ELOOP		62		/ * Too many levels of symbolic links */
-  gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-  gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-
-  /*	ENAMETOOLONG	63		/ * File name too long */
-  gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-  gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-
-  /*	EHOSTDOWN	64		/ * Host is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-  gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-
-  /*	EHOSTUNREACH	65		/ * No route to host */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-  gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-
-  /*	ENOTEMPTY	66		/ * Directory not empty */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-  gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-
-  /*	EPROCLIM	67		/ * Too many processes */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
-  gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
-
-  /*	EUSERS		68		/ * Too many users */
-  gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-  gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-  /*	EDQUOT		69		/ * Disc quota exceeded */
-  gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-  gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-  /*	ESTALE		70		/ * Stale NFS file handle */
-  gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-  gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-  /*	EREMOTE		71		/ * Too many levels of remote in path */
-  gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-  gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-
-  /*	EBADRPC		72		/ * RPC struct is bad */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
-  gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
-
-  /*	ERPCMISMATCH	73		/ * RPC version wrong */
-  gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
-  gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
-
-  /*	EPROGUNAVAIL	74		/ * RPC prog. not avail */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
-  gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
-
-  /*	EPROGMISMATCH	75		/ * Program version wrong */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
-  gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
-
-  /*	EPROCUNAVAIL	76		/ * Bad procedure for program */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
-  gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
-
-  /*	ENOLCK		77		/ * No locks available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-  gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-  /*	ENOSYS		78		/ * Function not implemented */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-  gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-
-  /*	EFTYPE		79		/ * Inappropriate file type or format */
-  gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
-  gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
-
-  /*	EAUTH		80		/ * Authentication error */
-  gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
-  gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
-
-  /*	ENEEDAUTH	81		/ * Need authenticator */
-  gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-  gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-/* Intelligent device errors */
-/*  	EPWROFF		82	/ * Device power is off */
-  gf_error_to_errno_array[GF_ERROR_CODE_PWROFF] = EPWROFF;
-  gf_errno_to_error_array[EPWROFF] = GF_ERROR_CODE_PWROFF;
-/*  	EDEVERR		83	/ * Device error, e.g. paper out */
-  gf_error_to_errno_array[GF_ERROR_CODE_DEVERR] = EDEVERR;
-  gf_errno_to_error_array[EDEVERR] = GF_ERROR_CODE_DEVERR;
- 
-  /*	EOVERFLOW	84		/ * Value too large to be stored in data type */
-  gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-  gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-
-/* Program loading errors */
-/*   EBADEXEC	85	/ * Bad executable */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADEXEC] = EBADEXEC;
-  gf_errno_to_error_array[EBADEXEC] = GF_ERROR_CODE_BADEXEC;
-
-/*   EBADARCH	86	/ * Bad CPU type in executable */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADARCH] = EBADARCH;
-  gf_errno_to_error_array[EBADARCH] = GF_ERROR_CODE_BADARCH;
-
-/*   ESHLIBVERS	87	/ * Shared library version mismatch */
-  gf_error_to_errno_array[GF_ERROR_CODE_SHLIBVERS] = ESHLIBVERS;
-  gf_errno_to_error_array[ESHLIBVERS] = GF_ERROR_CODE_SHLIBVERS;
-
-/*   EBADMACHO	88	/ * Malformed Macho file */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADMACHO] = EBADMACHO;
-  gf_errno_to_error_array[EBADMACHO] = GF_ERROR_CODE_BADMACHO;
-
-#if 0
-  /*    EDOOFUS		88		/ * Programming error */
-  gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
-  gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
+    /*    EDEADLK         11              / * Resource deadlock would occur */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*    EAGAIN          35              / * Try Again */
+    gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
+    gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
+
+    /*      EINPROGRESS     36              / * Operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /*      EALREADY        37              / * Operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+
+    /*      ENOTSOCK        38              / * Socket operation on non-socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+
+    /*      EDESTADDRREQ    39              / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+
+    /*      EMSGSIZE        40              / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+
+    /*      EPROTOTYPE      41              / * Protocol wrong type for socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+
+    /*      ENOPROTOOPT     42              / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+
+    /*      EPROTONOSUPPORT 43              / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+
+    /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      45              / * Operation not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*      EPFNOSUPPORT    46              / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+
+    /*      EAFNOSUPPORT    47              / * Address family not supported by
+     * protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+
+    /*      EADDRINUSE      48              / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+
+    /*      EADDRNOTAVAIL   49              / * Can't assign requested address
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+
+    /*      ENETDOWN        50              / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+
+    /*      ENETUNREACH     51              / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+
+    /*      ENETRESET       52              / * Network dropped connection on
+     * reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+
+    /*      ECONNABORTED    53              / * Software caused connection abort
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+
+    /*      ECONNRESET      54              / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+
+    /*      ENOBUFS         55              / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+
+    /*      EISCONN         56              / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+
+    /*      ENOTCONN        57              / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+
+    /*      ESHUTDOWN       58              / * Can't send after socket shutdown
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+
+    /*      ETOOMANYREFS    59              / * Too many references: can't
+     * splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+
+    /*      ETIMEDOUT       60              / * Operation timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    61              / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+
+    /*      ELOOP           62              / * Too many levels of symbolic
+     * links */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+
+    /*      ENAMETOOLONG    63              / * File name too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+
+    /*      EHOSTDOWN       64              / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+
+    /*      EHOSTUNREACH    65              / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+
+    /*      ENOTEMPTY       66              / * Directory not empty */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+
+    /*      EPROCLIM        67              / * Too many processes */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
+    gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
+
+    /*      EUSERS          68              / * Too many users */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /*      EDQUOT          69              / * Disc quota exceeded */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /*      ESTALE          70              / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    /*      EREMOTE         71              / * Too many levels of remote in
+     * path */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+
+    /*      EBADRPC         72              / * RPC struct is bad */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
+    gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
+
+    /*      ERPCMISMATCH    73              / * RPC version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
+    gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
+
+    /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
+    gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
+
+    /*      EPROGMISMATCH   75              / * Program version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
+    gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
+
+    /*      EPROCUNAVAIL    76              / * Bad procedure for program */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
+    gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
+
+    /*      ENOLCK          77              / * No locks available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
+
+    /*      ENOSYS          78              / * Function not implemented */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
+
+    /*      EFTYPE          79              / * Inappropriate file type or
+     * format */
+    gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
+    gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
+
+    /*      EAUTH           80              / * Authentication error */
+    gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
+    gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
+
+    /*      ENEEDAUTH       81              / * Need authenticator */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+    /* Intelligent device errors */
+    /*      EPWROFF         82      / * Device power is off */
+    gf_error_to_errno_array[GF_ERROR_CODE_PWROFF] = EPWROFF;
+    gf_errno_to_error_array[EPWROFF] = GF_ERROR_CODE_PWROFF;
+    /*      EDEVERR         83      / * Device error, e.g. paper out */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEVERR] = EDEVERR;
+    gf_errno_to_error_array[EDEVERR] = GF_ERROR_CODE_DEVERR;
+
+    /*      EOVERFLOW       84              / * Value too large to be stored in
+     * data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
+
+    /* Program loading errors */
+    /*   EBADEXEC   85      / * Bad executable */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADEXEC] = EBADEXEC;
+    gf_errno_to_error_array[EBADEXEC] = GF_ERROR_CODE_BADEXEC;
+
+    /*   EBADARCH   86      / * Bad CPU type in executable */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADARCH] = EBADARCH;
+    gf_errno_to_error_array[EBADARCH] = GF_ERROR_CODE_BADARCH;
+
+    /*   ESHLIBVERS 87      / * Shared library version mismatch */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHLIBVERS] = ESHLIBVERS;
+    gf_errno_to_error_array[ESHLIBVERS] = GF_ERROR_CODE_SHLIBVERS;
+
+    /*   EBADMACHO  88      / * Malformed Macho file */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMACHO] = EBADMACHO;
+    gf_errno_to_error_array[EBADMACHO] = GF_ERROR_CODE_BADMACHO;
+
+#ifdef EDOOFUS
+    /*    EDOOFUS           88              / * Programming error */
+    gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
+    gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
 #endif
 
-  /*  	ECANCELED	89		/ * Operation canceled */
-  gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-  gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-  /*   EIDRM		90		/ * Identifier removed */
-  gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-  gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-  /*   ENOMSG		91		/ * No message of desired type */   
-  gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-  gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-  /*   EILSEQ		92		/ * Illegal byte sequence */
-  gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-  gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-
-  /*   ENOATTR		93		/ * Attribute not found */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
-  gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
-
-  /*   EBADMSG		94		/ * Bad message */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-  gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
-
-  /*   EMULTIHOP	95		/ * Reserved */
-  gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-  gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
-
-  /*  	ENODATA		96		/ * No message available on STREAM */
-  gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-  gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-
-  /*   ENOLINK		97		/ * Reserved */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-  gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
-
-  /*   ENOSR		98		/ * No STREAM resources */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
-  gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
-
-  /*   ENOSTR		99		/ * Not a STREAM */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
-  gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
-
-/*  	EPROTO		100		/ * Protocol error */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-  gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
-/*   ETIME		101		/ * STREAM ioctl timeout */
-  gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
-  gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
-
-/* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */
-/*  	EOPNOTSUPP	102		/ * Operation not supported on socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-  gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-/*   ENOPOLICY	103		/ * No such policy registered */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOPOLICY] = ENOPOLICY;
-  gf_errno_to_error_array[ENOPOLICY] = GF_ERROR_CODE_NOPOLICY;
-
-  return ;
+    /*      ECANCELED       89              / * Operation canceled */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
+
+    /*   EIDRM              90              / * Identifier removed */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
+    /*   ENOMSG             91              / * No message of desired type */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
+
+    /*   EILSEQ             92              / * Illegal byte sequence */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
+
+    /*   ENOATTR            93              / * Attribute not found */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
+    gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
+
+    /*   EBADMSG            94              / * Bad message */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+
+    /*   EMULTIHOP  95              / * Reserved */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
+
+    /*      ENODATA         96              / * No message available on STREAM
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+
+    /*   ENOLINK            97              / * Reserved */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+
+    /*   ENOSR              98              / * No STREAM resources */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSR] = ENOSR;
+    gf_errno_to_error_array[ENOSR] = GF_ERROR_CODE_NOSR;
+
+    /*   ENOSTR             99              / * Not a STREAM */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSTR] = ENOSTR;
+    gf_errno_to_error_array[ENOSTR] = GF_ERROR_CODE_NOSTR;
+
+    /*      EPROTO          100             / * Protocol error */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+    /*   ETIME              101             / * STREAM ioctl timeout */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIME] = ETIME;
+    gf_errno_to_error_array[ETIME] = GF_ERROR_CODE_TIME;
+
+    /* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */
+    /*      EOPNOTSUPP      102             / * Operation not supported on
+     * socket */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*   ENOPOLICY  103             / * No such policy registered */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPOLICY] = ENOPOLICY;
+    gf_errno_to_error_array[ENOPOLICY] = GF_ERROR_CODE_NOPOLICY;
+
+    return;
 }
 #endif /* GF_DARWIN_HOST_OS */
 
 #ifdef GF_BSD_HOST_OS
-static void 
-init_compat_errno_arrays ()
+static void
+init_compat_errno_arrays()
 {
-  /* Quite a bit of things changed in FreeBSD - current */
-
-  /*    EAGAIN          35              / * Try Again */
-  gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
-  gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
-
-  /*    EDEADLK         11              / * Resource deadlock would occur */
-  gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
-  gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
-  
-  /*	EINPROGRESS	36	        / * Operation now in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
-  gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
-
-  /*	EALREADY	37		/ * Operation already in progress */
-  gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
-  gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
-  
-  /*	ENOTSOCK	38		/ * Socket operation on non-socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
-  gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
-  
-  /*	EDESTADDRREQ	39		/ * Destination address required */
-  gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
-  gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
-  
-  /*	EMSGSIZE	40		/ * Message too long */
-  gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
-  gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
-  
-  /*	EPROTOTYPE	41		/ * Protocol wrong type for socket */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
-  gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
-  
-  /*	ENOPROTOOPT	42		/ * Protocol not available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
-  gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
-  
-  /*	EPROTONOSUPPORT	43		/ * Protocol not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
-  gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
-  
-  /*	ESOCKTNOSUPPORT	44		/ * Socket type not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
-  gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
-
-  /*	EOPNOTSUPP	45		/ * Operation not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
-  gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
-
-  /*	EPFNOSUPPORT	46		/ * Protocol family not supported */
-  gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
-  gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
-
-  /*	EAFNOSUPPORT	47		/ * Address family not supported by protocol family */
-  gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
-  gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
-
-  /*	EADDRINUSE	48		/ * Address already in use */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
-  gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
-
-  /*	EADDRNOTAVAIL	49		/ * Can't assign requested address */
-  gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
-  gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
-
-  /*	ENETDOWN	50		/ * Network is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
-  gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
-
-  /*	ENETUNREACH	51		/ * Network is unreachable */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
-  gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
-
-  /*	ENETRESET	52		/ * Network dropped connection on reset */
-  gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
-  gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
-
-  /*	ECONNABORTED	53		/ * Software caused connection abort */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
-  gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
-
-  /*	ECONNRESET	54		/ * Connection reset by peer */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
-  gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
-  
-  /*	ENOBUFS		55		/ * No buffer space available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
-  gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
-
-  /*	EISCONN		56		/ * Socket is already connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
-  gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
-
-  /*	ENOTCONN	57		/ * Socket is not connected */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
-  gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
-
-  /*	ESHUTDOWN	58		/ * Can't send after socket shutdown */
-  gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
-  gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
-
-  /*	ETOOMANYREFS	59		/ * Too many references: can't splice */
-  gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
-  gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
-
-  /*	ETIMEDOUT	60		/ * Operation timed out */
-  gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
-  gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
-
-  /*	ECONNREFUSED	61		/ * Connection refused */
-  gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
-  gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
-
-  /*	ELOOP		62		/ * Too many levels of symbolic links */
-  gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
-  gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
-
-  /*	ENAMETOOLONG	63		/ * File name too long */
-  gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
-  gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
-
-  /*	EHOSTDOWN	64		/ * Host is down */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
-  gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
-
-  /*	EHOSTUNREACH	65		/ * No route to host */
-  gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
-  gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
-
-  /*	ENOTEMPTY	66		/ * Directory not empty */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
-  gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
-
-  /*	EPROCLIM	67		/ * Too many processes */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
-  gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
-
-  /*	EUSERS		68		/ * Too many users */
-  gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
-  gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
-
-  /*	EDQUOT		69		/ * Disc quota exceeded */
-  gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
-  gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
-
-  /*	ESTALE		70		/ * Stale NFS file handle */
-  gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
-  gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
-
-  /*	EREMOTE		71		/ * Too many levels of remote in path */
-  gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
-  gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
-
-  /*	EBADRPC		72		/ * RPC struct is bad */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
-  gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
-
-  /*	ERPCMISMATCH	73		/ * RPC version wrong */
-  gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
-  gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
-
-  /*	EPROGUNAVAIL	74		/ * RPC prog. not avail */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
-  gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
-
-  /*	EPROGMISMATCH	75		/ * Program version wrong */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
-  gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
-
-  /*	EPROCUNAVAIL	76		/ * Bad procedure for program */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
-  gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
-
-  /*	ENOLCK		77		/ * No locks available */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
-  gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
-
-  /*	ENOSYS		78		/ * Function not implemented */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
-  gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
-
-  /*	EFTYPE		79		/ * Inappropriate file type or format */
-  gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
-  gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
-
-  /*	EAUTH		80		/ * Authentication error */
-  gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
-  gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
-
-  /*	ENEEDAUTH	81		/ * Need authenticator */
-  gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
-  gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
-
-  /*	EIDRM		82		/ * Identifier removed */
-  gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
-  gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
-
-  /*	ENOMSG		83		/ * No message of desired type */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
-  gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
-
-  /*	EOVERFLOW	84		/ * Value too large to be stored in data type */
-  gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
-  gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
-
-  /*	ECANCELED	85		/ * Operation canceled */
-  gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
-  gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
-
-  /*	EILSEQ		86		/ * Illegal byte sequence */
-  gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
-  gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
-
-  /*	ENOATTR		87		/ * Attribute not found */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
-  gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
-  
-  /*    EDOOFUS		88		/ * Programming error */
-  gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
-  gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
+    /* Quite a bit of things changed in FreeBSD - current */
+
+    /*    EAGAIN          35              / * Try Again */
+    gf_error_to_errno_array[GF_ERROR_CODE_AGAIN] = EAGAIN;
+    gf_errno_to_error_array[EAGAIN] = GF_ERROR_CODE_AGAIN;
+
+    /*    EDEADLK         11              / * Resource deadlock would occur */
+    gf_error_to_errno_array[GF_ERROR_CODE_DEADLK] = EDEADLK;
+    gf_errno_to_error_array[EDEADLK] = GF_ERROR_CODE_DEADLK;
+
+    /*      EINPROGRESS     36              / * Operation now in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_INPROGRESS] = EINPROGRESS;
+    gf_errno_to_error_array[EINPROGRESS] = GF_ERROR_CODE_INPROGRESS;
+
+    /*      EALREADY        37              / * Operation already in progress */
+    gf_error_to_errno_array[GF_ERROR_CODE_ALREADY] = EALREADY;
+    gf_errno_to_error_array[EALREADY] = GF_ERROR_CODE_ALREADY;
+
+    /*      ENOTSOCK        38              / * Socket operation on non-socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTSOCK] = ENOTSOCK;
+    gf_errno_to_error_array[ENOTSOCK] = GF_ERROR_CODE_NOTSOCK;
+
+    /*      EDESTADDRREQ    39              / * Destination address required */
+    gf_error_to_errno_array[GF_ERROR_CODE_DESTADDRREQ] = EDESTADDRREQ;
+    gf_errno_to_error_array[EDESTADDRREQ] = GF_ERROR_CODE_DESTADDRREQ;
+
+    /*      EMSGSIZE        40              / * Message too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_MSGSIZE] = EMSGSIZE;
+    gf_errno_to_error_array[EMSGSIZE] = GF_ERROR_CODE_MSGSIZE;
+
+    /*      EPROTOTYPE      41              / * Protocol wrong type for socket
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTOTYPE] = EPROTOTYPE;
+    gf_errno_to_error_array[EPROTOTYPE] = GF_ERROR_CODE_PROTOTYPE;
+
+    /*      ENOPROTOOPT     42              / * Protocol not available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOPROTOOPT] = ENOPROTOOPT;
+    gf_errno_to_error_array[ENOPROTOOPT] = GF_ERROR_CODE_NOPROTOOPT;
+
+    /*      EPROTONOSUPPORT 43              / * Protocol not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTONOSUPPORT] = EPROTONOSUPPORT;
+    gf_errno_to_error_array[EPROTONOSUPPORT] = GF_ERROR_CODE_PROTONOSUPPORT;
+
+    /*      ESOCKTNOSUPPORT 44              / * Socket type not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_SOCKTNOSUPPORT] = ESOCKTNOSUPPORT;
+    gf_errno_to_error_array[ESOCKTNOSUPPORT] = GF_ERROR_CODE_SOCKTNOSUPPORT;
+
+    /*      EOPNOTSUPP      45              / * Operation not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_OPNOTSUPP] = EOPNOTSUPP;
+    gf_errno_to_error_array[EOPNOTSUPP] = GF_ERROR_CODE_OPNOTSUPP;
+
+    /*      EPFNOSUPPORT    46              / * Protocol family not supported */
+    gf_error_to_errno_array[GF_ERROR_CODE_PFNOSUPPORT] = EPFNOSUPPORT;
+    gf_errno_to_error_array[EPFNOSUPPORT] = GF_ERROR_CODE_PFNOSUPPORT;
+
+    /*      EAFNOSUPPORT    47              / * Address family not supported by
+     * protocol family */
+    gf_error_to_errno_array[GF_ERROR_CODE_AFNOSUPPORT] = EAFNOSUPPORT;
+    gf_errno_to_error_array[EAFNOSUPPORT] = GF_ERROR_CODE_AFNOSUPPORT;
+
+    /*      EADDRINUSE      48              / * Address already in use */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRINUSE] = EADDRINUSE;
+    gf_errno_to_error_array[EADDRINUSE] = GF_ERROR_CODE_ADDRINUSE;
+
+    /*      EADDRNOTAVAIL   49              / * Can't assign requested address
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_ADDRNOTAVAIL] = EADDRNOTAVAIL;
+    gf_errno_to_error_array[EADDRNOTAVAIL] = GF_ERROR_CODE_ADDRNOTAVAIL;
+
+    /*      ENETDOWN        50              / * Network is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETDOWN] = ENETDOWN;
+    gf_errno_to_error_array[ENETDOWN] = GF_ERROR_CODE_NETDOWN;
+
+    /*      ENETUNREACH     51              / * Network is unreachable */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETUNREACH] = ENETUNREACH;
+    gf_errno_to_error_array[ENETUNREACH] = GF_ERROR_CODE_NETUNREACH;
+
+    /*      ENETRESET       52              / * Network dropped connection on
+     * reset */
+    gf_error_to_errno_array[GF_ERROR_CODE_NETRESET] = ENETRESET;
+    gf_errno_to_error_array[ENETRESET] = GF_ERROR_CODE_NETRESET;
+
+    /*      ECONNABORTED    53              / * Software caused connection abort
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNABORTED] = ECONNABORTED;
+    gf_errno_to_error_array[ECONNABORTED] = GF_ERROR_CODE_CONNABORTED;
+
+    /*      ECONNRESET      54              / * Connection reset by peer */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNRESET] = ECONNRESET;
+    gf_errno_to_error_array[ECONNRESET] = GF_ERROR_CODE_CONNRESET;
+
+    /*      ENOBUFS         55              / * No buffer space available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOBUFS] = ENOBUFS;
+    gf_errno_to_error_array[ENOBUFS] = GF_ERROR_CODE_NOBUFS;
+
+    /*      EISCONN         56              / * Socket is already connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_ISCONN] = EISCONN;
+    gf_errno_to_error_array[EISCONN] = GF_ERROR_CODE_ISCONN;
+
+    /*      ENOTCONN        57              / * Socket is not connected */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTCONN] = ENOTCONN;
+    gf_errno_to_error_array[ENOTCONN] = GF_ERROR_CODE_NOTCONN;
+
+    /*      ESHUTDOWN       58              / * Can't send after socket shutdown
+     */
+    gf_error_to_errno_array[GF_ERROR_CODE_SHUTDOWN] = ESHUTDOWN;
+    gf_errno_to_error_array[ESHUTDOWN] = GF_ERROR_CODE_SHUTDOWN;
+
+    /*      ETOOMANYREFS    59              / * Too many references: can't
+     * splice */
+    gf_error_to_errno_array[GF_ERROR_CODE_TOOMANYREFS] = ETOOMANYREFS;
+    gf_errno_to_error_array[ETOOMANYREFS] = GF_ERROR_CODE_TOOMANYREFS;
+
+    /*      ETIMEDOUT       60              / * Operation timed out */
+    gf_error_to_errno_array[GF_ERROR_CODE_TIMEDOUT] = ETIMEDOUT;
+    gf_errno_to_error_array[ETIMEDOUT] = GF_ERROR_CODE_TIMEDOUT;
+
+    /*      ECONNREFUSED    61              / * Connection refused */
+    gf_error_to_errno_array[GF_ERROR_CODE_CONNREFUSED] = ECONNREFUSED;
+    gf_errno_to_error_array[ECONNREFUSED] = GF_ERROR_CODE_CONNREFUSED;
+
+    /*      ELOOP           62              / * Too many levels of symbolic
+     * links */
+    gf_error_to_errno_array[GF_ERROR_CODE_LOOP] = ELOOP;
+    gf_errno_to_error_array[ELOOP] = GF_ERROR_CODE_LOOP;
+
+    /*      ENAMETOOLONG    63              / * File name too long */
+    gf_error_to_errno_array[GF_ERROR_CODE_NAMETOOLONG] = ENAMETOOLONG;
+    gf_errno_to_error_array[ENAMETOOLONG] = GF_ERROR_CODE_NAMETOOLONG;
+
+    /*      EHOSTDOWN       64              / * Host is down */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTDOWN] = EHOSTDOWN;
+    gf_errno_to_error_array[EHOSTDOWN] = GF_ERROR_CODE_HOSTDOWN;
+
+    /*      EHOSTUNREACH    65              / * No route to host */
+    gf_error_to_errno_array[GF_ERROR_CODE_HOSTUNREACH] = EHOSTUNREACH;
+    gf_errno_to_error_array[EHOSTUNREACH] = GF_ERROR_CODE_HOSTUNREACH;
+
+    /*      ENOTEMPTY       66              / * Directory not empty */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOTEMPTY] = ENOTEMPTY;
+    gf_errno_to_error_array[ENOTEMPTY] = GF_ERROR_CODE_NOTEMPTY;
+
+    /*      EPROCLIM        67              / * Too many processes */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCLIM] = EPROCLIM;
+    gf_errno_to_error_array[EPROCLIM] = GF_ERROR_CODE_PROCLIM;
+
+    /*      EUSERS          68              / * Too many users */
+    gf_error_to_errno_array[GF_ERROR_CODE_USERS] = EUSERS;
+    gf_errno_to_error_array[EUSERS] = GF_ERROR_CODE_USERS;
+
+    /*      EDQUOT          69              / * Disc quota exceeded */
+    gf_error_to_errno_array[GF_ERROR_CODE_DQUOT] = EDQUOT;
+    gf_errno_to_error_array[EDQUOT] = GF_ERROR_CODE_DQUOT;
+
+    /*      ESTALE          70              / * Stale NFS file handle */
+    gf_error_to_errno_array[GF_ERROR_CODE_STALE] = ESTALE;
+    gf_errno_to_error_array[ESTALE] = GF_ERROR_CODE_STALE;
+
+    /*      EREMOTE         71              / * Too many levels of remote in
+     * path */
+    gf_error_to_errno_array[GF_ERROR_CODE_REMOTE] = EREMOTE;
+    gf_errno_to_error_array[EREMOTE] = GF_ERROR_CODE_REMOTE;
+
+    /*      EBADRPC         72              / * RPC struct is bad */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADRPC] = EBADRPC;
+    gf_errno_to_error_array[EBADRPC] = GF_ERROR_CODE_BADRPC;
+
+    /*      ERPCMISMATCH    73              / * RPC version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_RPCMISMATCH] = ERPCMISMATCH;
+    gf_errno_to_error_array[ERPCMISMATCH] = GF_ERROR_CODE_RPCMISMATCH;
+
+    /*      EPROGUNAVAIL    74              / * RPC prog. not avail */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGUNAVAIL] = EPROGUNAVAIL;
+    gf_errno_to_error_array[EPROGUNAVAIL] = GF_ERROR_CODE_PROGUNAVAIL;
+
+    /*      EPROGMISMATCH   75              / * Program version wrong */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROGMISMATCH] = EPROGMISMATCH;
+    gf_errno_to_error_array[EPROGMISMATCH] = GF_ERROR_CODE_PROGMISMATCH;
+
+    /*      EPROCUNAVAIL    76              / * Bad procedure for program */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROCUNAVAIL] = EPROCUNAVAIL;
+    gf_errno_to_error_array[EPROCUNAVAIL] = GF_ERROR_CODE_PROCUNAVAIL;
+
+    /*      ENOLCK          77              / * No locks available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLCK] = ENOLCK;
+    gf_errno_to_error_array[ENOLCK] = GF_ERROR_CODE_NOLCK;
+
+    /*      ENOSYS          78              / * Function not implemented */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOSYS] = ENOSYS;
+    gf_errno_to_error_array[ENOSYS] = GF_ERROR_CODE_NOSYS;
+
+    /*      EFTYPE          79              / * Inappropriate file type or
+     * format */
+    gf_error_to_errno_array[GF_ERROR_CODE_FTYPE] = EFTYPE;
+    gf_errno_to_error_array[EFTYPE] = GF_ERROR_CODE_FTYPE;
+
+    /*      EAUTH           80              / * Authentication error */
+    gf_error_to_errno_array[GF_ERROR_CODE_AUTH] = EAUTH;
+    gf_errno_to_error_array[EAUTH] = GF_ERROR_CODE_AUTH;
+
+    /*      ENEEDAUTH       81              / * Need authenticator */
+    gf_error_to_errno_array[GF_ERROR_CODE_NEEDAUTH] = ENEEDAUTH;
+    gf_errno_to_error_array[ENEEDAUTH] = GF_ERROR_CODE_NEEDAUTH;
+
+    /*      EIDRM           82              / * Identifier removed */
+    gf_error_to_errno_array[GF_ERROR_CODE_IDRM] = EIDRM;
+    gf_errno_to_error_array[EIDRM] = GF_ERROR_CODE_IDRM;
+
+    /*      ENOMSG          83              / * No message of desired type */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOMSG] = ENOMSG;
+    gf_errno_to_error_array[ENOMSG] = GF_ERROR_CODE_NOMSG;
+
+    /*      EOVERFLOW       84              / * Value too large to be stored in
+     * data type */
+    gf_error_to_errno_array[GF_ERROR_CODE_OVERFLOW] = EOVERFLOW;
+    gf_errno_to_error_array[EOVERFLOW] = GF_ERROR_CODE_OVERFLOW;
+
+    /*      ECANCELED       85              / * Operation canceled */
+    gf_error_to_errno_array[GF_ERROR_CODE_CANCELED] = ECANCELED;
+    gf_errno_to_error_array[ECANCELED] = GF_ERROR_CODE_CANCELED;
+
+    /*      EILSEQ          86              / * Illegal byte sequence */
+    gf_error_to_errno_array[GF_ERROR_CODE_ILSEQ] = EILSEQ;
+    gf_errno_to_error_array[EILSEQ] = GF_ERROR_CODE_ILSEQ;
+
+    /*      ENOATTR         87              / * Attribute not found */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOATTR] = ENOATTR;
+    gf_errno_to_error_array[ENOATTR] = GF_ERROR_CODE_NOATTR;
+
+#ifdef EDOOFUS
+    /*    EDOOFUS           88              / * Programming error */
+    gf_error_to_errno_array[GF_ERROR_CODE_DOOFUS] = EDOOFUS;
+    gf_errno_to_error_array[EDOOFUS] = GF_ERROR_CODE_DOOFUS;
+#endif
 
-  /*	EBADMSG		89		/ * Bad message */
-  gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
-  gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
+    /*      EBADMSG         89              / * Bad message */
+    gf_error_to_errno_array[GF_ERROR_CODE_BADMSG] = EBADMSG;
+    gf_errno_to_error_array[EBADMSG] = GF_ERROR_CODE_BADMSG;
 
-  /*	EMULTIHOP	90		/ * Multihop attempted */
-  gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
-  gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
+#ifdef __NetBSD__
+    /*      ENODATA         89              / * No message available */
+    gf_error_to_errno_array[GF_ERROR_CODE_NODATA] = ENODATA;
+    gf_errno_to_error_array[ENODATA] = GF_ERROR_CODE_NODATA;
+#endif
 
-  /*	ENOLINK		91		/ * Link has been severed */
-  gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
-  gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
+    /*      EMULTIHOP       90              / * Multihop attempted */
+    gf_error_to_errno_array[GF_ERROR_CODE_MULTIHOP] = EMULTIHOP;
+    gf_errno_to_error_array[EMULTIHOP] = GF_ERROR_CODE_MULTIHOP;
 
-  /*	EPROTO		92		/ * Protocol error */
-  gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
-  gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
+    /*      ENOLINK         91              / * Link has been severed */
+    gf_error_to_errno_array[GF_ERROR_CODE_NOLINK] = ENOLINK;
+    gf_errno_to_error_array[ENOLINK] = GF_ERROR_CODE_NOLINK;
 
+    /*      EPROTO          92              / * Protocol error */
+    gf_error_to_errno_array[GF_ERROR_CODE_PROTO] = EPROTO;
+    gf_errno_to_error_array[EPROTO] = GF_ERROR_CODE_PROTO;
 
-  return ;
+    return;
 }
 #endif /* GF_BSD_HOST_OS */
 
 #ifdef GF_LINUX_HOST_OS
-static void 
-init_compat_errno_arrays ()
+static void
+init_compat_errno_arrays()
 {
-  /* Things are fine. Everything should work seemlessly on GNU/Linux machines */
-  return ;
+    /* Things are fine. Everything should work seemlessly on GNU/Linux machines
+     */
+    return;
 }
 #endif /* GF_LINUX_HOST_OS */
 
-
 static void
-init_errno_arrays ()
+init_errno_arrays()
 {
-  int i;
-  for (i=0; i < GF_ERROR_CODE_UNKNOWN; i++) {
-    gf_errno_to_error_array[i] = i;
-    gf_error_to_errno_array[i] = i;
-  }
-  /* Now change the order if it needs to be. */
-  init_compat_errno_arrays();
-
-  return;
+    int i;
+    for (i = 0; i < GF_ERROR_CODE_UNKNOWN; i++) {
+        gf_errno_to_error_array[i] = i;
+        gf_error_to_errno_array[i] = i;
+    }
+    /* Now change the order if it needs to be. */
+    init_compat_errno_arrays();
+
+    return;
 }
 
-int32_t 
-gf_errno_to_error (int32_t op_errno)
+int32_t
+gf_errno_to_error(int32_t op_errno)
 {
-  if (!gf_compat_errno_init_done) {
-    init_errno_arrays ();
-    gf_compat_errno_init_done = 1;
-  }
+    if (!gf_compat_errno_init_done) {
+        init_errno_arrays();
+        gf_compat_errno_init_done = 1;
+    }
 
-  if ((op_errno > GF_ERROR_CODE_SUCCESS) && (op_errno < GF_ERROR_CODE_UNKNOWN))
-    return gf_errno_to_error_array[op_errno];
+    if ((op_errno > GF_ERROR_CODE_SUCCESS) &&
+        (op_errno < GF_ERROR_CODE_UNKNOWN))
+        return gf_errno_to_error_array[op_errno];
 
-  return op_errno;
+    return op_errno;
 }
 
-
-int32_t 
-gf_error_to_errno (int32_t error)
+int32_t
+gf_error_to_errno(int32_t error)
 {
-  if (!gf_compat_errno_init_done) {
-    init_errno_arrays ();
-    gf_compat_errno_init_done = 1;
-  }
+    if (!gf_compat_errno_init_done) {
+        init_errno_arrays();
+        gf_compat_errno_init_done = 1;
+    }
 
-  if ((error > GF_ERROR_CODE_SUCCESS) && (error < GF_ERROR_CODE_UNKNOWN))
-    return gf_error_to_errno_array[error];
+    if ((error > GF_ERROR_CODE_SUCCESS) && (error < GF_ERROR_CODE_UNKNOWN))
+        return gf_error_to_errno_array[error];
 
-  return error;
+    return error;
 }
-
diff --git a/libglusterfs/src/compat-errno.h b/libglusterfs/src/compat-errno.h
deleted file mode 100644
index bdd8becfdda..00000000000
--- a/libglusterfs/src/compat-errno.h
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __COMPAT_ERRNO_H__
-#define __COMPAT_ERRNO_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <errno.h>
-
-#define GF_ERROR_CODE_SUCCESS         0
-#define GF_ERROR_CODE_UNKNOWN         1024
-#define GF_ERRNO_UNKNOWN              1024
-
-#define GF_ERROR_CODE_PERM            1      /* Operation not permitted */
-#define GF_ERROR_CODE_NOENT           2      /* No such file or directory */
-#define GF_ERROR_CODE_SRCH            3      /* No such process */
-#define GF_ERROR_CODE_INTR            4      /* Interrupted system call */
-#define GF_ERROR_CODE_IO              5      /* I/O error */
-#define GF_ERROR_CODE_NXIO            6      /* No such device or address */
-#define GF_ERROR_CODE_2BIG            7      /* Argument list too long */
-#define GF_ERROR_CODE_NOEXEC          8      /* Exec format error */
-#define GF_ERROR_CODE_BADF            9      /* Bad file number */
-#define GF_ERROR_CODE_CHILD          10      /* No child processes */
-#define GF_ERROR_CODE_AGAIN          11      /* Try again */
-#define GF_ERROR_CODE_NOMEM          12      /* Out of memory */
-#define GF_ERROR_CODE_ACCES          13      /* Permission denied */
-#define GF_ERROR_CODE_FAULT          14      /* Bad address */
-#define GF_ERROR_CODE_NOTBLK         15      /* Block device required */
-#define GF_ERROR_CODE_BUSY           16      /* Device or resource busy */
-#define GF_ERROR_CODE_EXIST          17      /* File exists */
-#define GF_ERROR_CODE_XDEV           18      /* Cross-device link */
-#define GF_ERROR_CODE_NODEV          19      /* No such device */
-#define GF_ERROR_CODE_NOTDIR         20      /* Not a directory */
-#define GF_ERROR_CODE_ISDIR          21      /* Is a directory */
-#define GF_ERROR_CODE_INVAL          22      /* Invalid argument */
-#define GF_ERROR_CODE_NFILE          23      /* File table overflow */
-#define GF_ERROR_CODE_MFILE          24      /* Too many open files */
-#define GF_ERROR_CODE_NOTTY          25      /* Not a typewriter */
-#define GF_ERROR_CODE_TXTBSY         26      /* Text file busy */
-#define GF_ERROR_CODE_FBIG           27      /* File too large */
-#define GF_ERROR_CODE_NOSPC          28      /* No space left on device */
-#define GF_ERROR_CODE_SPIPE          29      /* Illegal seek */
-#define GF_ERROR_CODE_ROFS           30      /* Read-only file system */
-#define GF_ERROR_CODE_MLINK          31      /* Too many links */
-#define GF_ERROR_CODE_PIPE           32      /* Broken pipe */
-#define GF_ERROR_CODE_DOM            33      /* Math argument out of domain of func */
-#define GF_ERROR_CODE_RANGE          34      /* Math result not representable */
-#define GF_ERROR_CODE_DEADLK         35      /* Resource deadlock would occur */
-#define GF_ERROR_CODE_NAMETOOLONG    36      /* File name too long */
-#define GF_ERROR_CODE_NOLCK          37      /* No record locks available */
-#define GF_ERROR_CODE_NOSYS          38      /* Function not implemented */
-#define GF_ERROR_CODE_NOTEMPTY       39      /* Directory not empty */
-#define GF_ERROR_CODE_LOOP           40      /* Too many symbolic links encountered */
-
-#define GF_ERROR_CODE_NOMSG          42      /* No message of desired type */
-#define GF_ERROR_CODE_IDRM           43      /* Identifier removed */
-#define GF_ERROR_CODE_CHRNG          44      /* Channel number out of range */
-#define GF_ERROR_CODE_L2NSYNC        45      /* Level 2 not synchronized */
-#define GF_ERROR_CODE_L3HLT          46      /* Level 3 halted */
-#define GF_ERROR_CODE_L3RST          47      /* Level 3 reset */
-#define GF_ERROR_CODE_LNRNG          48      /* Link number out of range */
-#define GF_ERROR_CODE_UNATCH         49      /* Protocol driver not attached */
-#define GF_ERROR_CODE_NOCSI          50      /* No CSI structure available */
-#define GF_ERROR_CODE_L2HLT          51      /* Level 2 halted */
-#define GF_ERROR_CODE_BADE           52      /* Invalid exchange */
-#define GF_ERROR_CODE_BADR           53      /* Invalid request descriptor */
-#define GF_ERROR_CODE_XFULL          54      /* Exchange full */
-#define GF_ERROR_CODE_NOANO          55      /* No anode */
-#define GF_ERROR_CODE_BADRQC         56      /* Invalid request code */
-#define GF_ERROR_CODE_BADSLT         57      /* Invalid slot */
-#define GF_ERROR_CODE_BFONT          59      /* Bad font file format */
-#define GF_ERROR_CODE_NOSTR          60      /* Device not a stream */
-#define GF_ERROR_CODE_NODATA         61      /* No data available */
-#define GF_ERROR_CODE_TIME           62      /* Timer expired */
-#define GF_ERROR_CODE_NOSR           63      /* Out of streams resources */
-#define GF_ERROR_CODE_NONET          64      /* Machine is not on the network */
-#define GF_ERROR_CODE_NOPKG          65      /* Package not installed */
-#define GF_ERROR_CODE_REMOTE         66      /* Object is remote */
-#define GF_ERROR_CODE_NOLINK         67      /* Link has been severed */
-#define GF_ERROR_CODE_ADV            68      /* Advertise error */
-#define GF_ERROR_CODE_SRMNT          69      /* Srmount error */
-#define GF_ERROR_CODE_COMM           70      /* Communication error on send */
-#define GF_ERROR_CODE_PROTO          71      /* Protocol error */
-#define GF_ERROR_CODE_MULTIHOP       72      /* Multihop attempted */
-#define GF_ERROR_CODE_DOTDOT         73      /* RFS specific error */
-#define GF_ERROR_CODE_BADMSG         74      /* Not a data message */
-#define GF_ERROR_CODE_OVERFLOW       75      /* Value too large for defined data type */
-#define GF_ERROR_CODE_NOTUNIQ        76      /* Name not unique on network */
-#define GF_ERROR_CODE_BADFD          77      /* File descriptor in bad state */
-#define GF_ERROR_CODE_REMCHG         78      /* Remote address changed */
-#define GF_ERROR_CODE_LIBACC         79      /* Can not access a needed shared library */
-#define GF_ERROR_CODE_LIBBAD         80      /* Accessing a corrupted shared library */
-#define GF_ERROR_CODE_LIBSCN         81      /* .lib section in a.out corrupted */
-#define GF_ERROR_CODE_LIBMAX         82      /* Attempting to link in too many shared libraries */
-#define GF_ERROR_CODE_LIBEXEC        83      /* Cannot exec a shared library directly */
-#define GF_ERROR_CODE_ILSEQ          84      /* Illegal byte sequence */
-#define GF_ERROR_CODE_RESTART        85      /* Interrupted system call should be restarted */
-#define GF_ERROR_CODE_STRPIPE        86      /* Streams pipe error */
-#define GF_ERROR_CODE_USERS          87      /* Too many users */
-#define GF_ERROR_CODE_NOTSOCK        88      /* Socket operation on non-socket */
-#define GF_ERROR_CODE_DESTADDRREQ    89      /* Destination address required */
-#define GF_ERROR_CODE_MSGSIZE        90      /* Message too long */
-#define GF_ERROR_CODE_PROTOTYPE      91      /* Protocol wrong type for socket */
-#define GF_ERROR_CODE_NOPROTOOPT     92      /* Protocol not available */
-#define GF_ERROR_CODE_PROTONOSUPPORT 93      /* Protocol not supported */
-#define GF_ERROR_CODE_SOCKTNOSUPPORT 94      /* Socket type not supported */
-#define GF_ERROR_CODE_OPNOTSUPP      95      /* Operation not supported on transport endpoint */
-#define GF_ERROR_CODE_PFNOSUPPORT    96      /* Protocol family not supported */
-#define GF_ERROR_CODE_AFNOSUPPORT    97      /* Address family not supported by protocol */
-#define GF_ERROR_CODE_ADDRINUSE      98      /* Address already in use */
-#define GF_ERROR_CODE_ADDRNOTAVAIL   99      /* Cannot assign requested address */
-#define GF_ERROR_CODE_NETDOWN        100     /* Network is down */
-#define GF_ERROR_CODE_NETUNREACH     101     /* Network is unreachable */
-#define GF_ERROR_CODE_NETRESET       102     /* Network dropped connection because of reset */
-#define GF_ERROR_CODE_CONNABORTED    103     /* Software caused connection abort */
-#define GF_ERROR_CODE_CONNRESET      104     /* Connection reset by peer */
-#define GF_ERROR_CODE_NOBUFS         105     /* No buffer space available */
-#define GF_ERROR_CODE_ISCONN         106     /* Transport endpoint is already connected */
-#define GF_ERROR_CODE_NOTCONN        107     /* Transport endpoint is not connected */
-#define GF_ERROR_CODE_SHUTDOWN       108     /* Cannot send after transport endpoint shutdown */
-#define GF_ERROR_CODE_TOOMANYREFS    109     /* Too many references: cannot splice */
-#define GF_ERROR_CODE_TIMEDOUT       110     /* Connection timed out */
-#define GF_ERROR_CODE_CONNREFUSED    111     /* Connection refused */
-#define GF_ERROR_CODE_HOSTDOWN       112     /* Host is down */
-#define GF_ERROR_CODE_HOSTUNREACH    113     /* No route to host */
-#define GF_ERROR_CODE_ALREADY        114     /* Operation already in progress */
-#define GF_ERROR_CODE_INPROGRESS     115     /* Operation now in progress */
-#define GF_ERROR_CODE_ALREADY        114     /* Operation already in progress */
-#define GF_ERROR_CODE_INPROGRESS     115     /* Operation now in progress */
-#define GF_ERROR_CODE_STALE          116     /* Stale NFS file handle */
-#define GF_ERROR_CODE_UCLEAN         117     /* Structure needs cleaning */
-#define GF_ERROR_CODE_NOTNAM         118     /* Not a XENIX named type file */
-#define GF_ERROR_CODE_NAVAIL         119     /* No XENIX semaphores available */
-#define GF_ERROR_CODE_ISNAM          120     /* Is a named type file */
-#define GF_ERROR_CODE_REMOTEIO       121     /* Remote I/O error */
-#define GF_ERROR_CODE_DQUOT          122     /* Quota exceeded */
-#define GF_ERROR_CODE_NOMEDIUM       123     /* No medium found */
-#define GF_ERROR_CODE_MEDIUMTYPE     124     /* Wrong medium type */
-#define GF_ERROR_CODE_CANCELED       125     /* Operation Canceled */
-#define GF_ERROR_CODE_NOKEY          126     /* Required key not available */
-#define GF_ERROR_CODE_KEYEXPIRED     127     /* Key has expired */
-#define GF_ERROR_CODE_KEYREVOKED     128     /* Key has been revoked */
-#define GF_ERROR_CODE_KEYREJECTED    129     /* Key was rejected by service */
-
-/* for robust mutexes */
-#define GF_ERROR_CODE_OWNERDEAD      130     /* Owner died */
-#define GF_ERROR_CODE_NOTRECOVERABLE 131     /* State not recoverable */
-
-
-
-/* Should never be seen by user programs */
-#define GF_ERROR_CODE_RESTARTSYS     512
-#define GF_ERROR_CODE_RESTARTNOINTR  513
-#define GF_ERROR_CODE_RESTARTNOHAND  514     /* restart if no handler.. */
-#define GF_ERROR_CODE_NOIOCTLCMD     515     /* No ioctl command */
-#define GF_ERROR_CODE_RESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */
-
-/* Defined for the NFSv3 protocol */
-#define GF_ERROR_CODE_BADHANDLE      521     /* Illegal NFS file handle */
-#define GF_ERROR_CODE_NOTSYNC        522     /* Update synchronization mismatch */
-#define GF_ERROR_CODE_BADCOOKIE      523     /* Cookie is stale */
-#define GF_ERROR_CODE_NOTSUPP        524     /* Operation is not supported */
-#define GF_ERROR_CODE_TOOSMALL       525     /* Buffer or request is too small */
-#define GF_ERROR_CODE_SERVERFAULT    526     /* An untranslatable error occurred */
-#define GF_ERROR_CODE_BADTYPE        527     /* Type not supported by server */
-#define GF_ERROR_CODE_JUKEBOX        528     /* Request initiated, but will not complete before timeout */
-#define GF_ERROR_CODE_IOCBQUEUED     529     /* iocb queued, will get completion event */
-#define GF_ERROR_CODE_IOCBRETRY      530     /* iocb queued, will trigger a retry */
-
-/* Darwin OS X */
-#define GF_ERROR_CODE_NOPOLICY   701
-#define GF_ERROR_CODE_BADMACHO   702
-#define GF_ERROR_CODE_PWROFF     703
-#define GF_ERROR_CODE_DEVERR     704
-#define GF_ERROR_CODE_BADARCH    705
-#define GF_ERROR_CODE_BADEXEC    706
-#define GF_ERROR_CODE_SHLIBVERS  707
-
-
-
-/* Solaris */
-/*  	ENOTACTIVE 73	/ * Facility is not active		*/
-#define GF_ERROR_CODE_NOTACTIVE   801 
-/*  	ELOCKUNMAPPED	72	/ * locked lock was unmapped */
-#define GF_ERROR_CODE_LOCKUNMAPPED 802
-
-/* BSD system */
-#define GF_ERROR_CODE_PROCLIM	        901		/* Too many processes */
-#define GF_ERROR_CODE_BADRPC		902		/* RPC struct is bad */
-#define GF_ERROR_CODE_RPCMISMATCH	903		/* RPC version wrong */
-#define GF_ERROR_CODE_PROGUNAVAIL	904		/* RPC prog. not avail */
-#define GF_ERROR_CODE_PROGMISMATCH	905		/* Program version wrong */
-#define GF_ERROR_CODE_PROCUNAVAIL	905		/* Bad procedure for program */
-#define GF_ERROR_CODE_FTYPE		906		/* Inappropriate file type or format */
-#define GF_ERROR_CODE_AUTH		907		/* Authentication error */
-#define GF_ERROR_CODE_NEEDAUTH	        908		/* Need authenticator */
-#define GF_ERROR_CODE_DOOFUS		909		/* Programming error */
-
-#define GF_ERROR_CODE_NOATTR		GF_ERROR_CODE_NODATA		/* Attribute not found */
-
-/* Either one of enodata or enoattr will be there in system */
-#ifndef ENOATTR
-#define ENOATTR ENODATA
-#endif /* ENOATTR */
-
-#ifndef ENODATA
-#define ENODATA ENOATTR
-#endif /* ENODATA */
-
-#ifndef EBADFD
-#define EBADFD EBADRPC
-#endif /* EBADFD */
-
-/* These functions are defined for all the OS flags, but content will 
- * be different for each OS flag. 
- */
-int32_t gf_errno_to_error (int32_t op_errno);
-int32_t gf_error_to_errno (int32_t error);
-
-#endif /* __COMPAT_ERRNO_H__ */
diff --git a/libglusterfs/src/compat.c b/libglusterfs/src/compat.c
index 2102216317d..8a05a30a8fe 100644
--- a/libglusterfs/src/compat.c
+++ b/libglusterfs/src/compat.c
@@ -1,340 +1,618 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-#include <string.h>
 #include <stdlib.h>
+#include <unistd.h>
 #include <stdarg.h>
 #include <getopt.h>
 #include <sys/types.h>
 #include <dirent.h>
 
+#include "glusterfs/logging.h"
+#include "glusterfs/compat.h"
+#include "glusterfs/iatt.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/run.h"
+#include "glusterfs/libglusterfs-messages.h"
+
 #ifdef GF_SOLARIS_HOST_OS
-#include "logging.h"
-#endif /* GF_SOLARIS_HOST_OS */
+int
+solaris_fsetxattr(int fd, const char *key, const char *value, size_t size,
+                  int flags)
+{
+    int attrfd = -1;
+    int ret = 0;
 
-#include "compat.h"
-#include "common-utils.h"
+    attrfd = openat(fd, key, flags | O_CREAT | O_WRONLY | O_XATTR, 0777);
+    if (attrfd >= 0) {
+        ftruncate(attrfd, 0);
+        ret = write(attrfd, value, size);
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_ERROR, errno,
+                   LG_MSG_SET_ATTRIBUTE_FAILED,
+                   "Couldn't set "
+                   "extended attribute for %d",
+                   fd);
+        return -1;
+    }
 
-#ifdef GF_SOLARIS_HOST_OS
+    return 0;
+}
 
-int 
-solaris_fsetxattr(int fd, 
-		  const char* key, 
-		  const char *value, 
-		  size_t size, 
-		  int flags)
+int
+solaris_fgetxattr(int fd, const char *key, char *value, size_t size)
 {
-	int attrfd = -1;
-	int ret = 0;
-	
-	attrfd = openat (fd, key, flags|O_CREAT|O_WRONLY|O_XATTR, 0777);
-	if (attrfd >= 0) {
-		ftruncate (attrfd, 0);
-		ret = write (attrfd, value, size);
-		close (attrfd);
-	} else {
-		if (errno != ENOENT)
-			gf_log ("libglusterfs", GF_LOG_ERROR, 
-				"Couldn't set extended attribute for %d (%d)", 
-				fd, errno);
-		return -1;
-	}
-	
-	return 0;
-}
+    int attrfd = -1;
+    int ret = 0;
 
+    attrfd = openat(fd, key, O_RDONLY | O_XATTR);
+    if (attrfd >= 0) {
+        if (size == 0) {
+            struct stat buf;
+            fstat(attrfd, &buf);
+            ret = buf.st_size;
+        } else {
+            ret = read(attrfd, value, size);
+        }
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_INFO, errno,
+                   LG_MSG_READ_ATTRIBUTE_FAILED,
+                   "Couldn't read "
+                   "extended attribute for the file %d",
+                   fd);
+        if (errno == ENOENT)
+            errno = ENODATA;
+        return -1;
+    }
 
-int 
-solaris_fgetxattr(int fd, 
-		  const char* key, 
-		  char *value, 
-		  size_t size)
-{
-	int attrfd = -1;
-	int ret = 0;
-	
-	attrfd = openat (fd, key, O_RDONLY|O_XATTR);
-	if (attrfd >= 0) {
-		if (size == 0) {
-			struct stat buf;
-			fstat (attrfd, &buf);
-			ret = buf.st_size;
-		} else {
-			ret = read (attrfd, value, size);
-		}
-		close (attrfd);
-	} else {
-		if (errno == ENOENT)
-			errno = ENODATA;
-		if (errno != ENOENT)
-			gf_log ("libglusterfs", GF_LOG_DEBUG, 
-				"Couldn't read extended attribute for the file %d (%d)", 
-				fd, errno);
-		return -1;
-	}
-	
-	return ret;
+    return ret;
 }
 
+/* Solaris does not support xattr for symlinks and dev files. Since gfid and
+   other trusted attributes are stored as xattrs, we need to provide support for
+   them. A mapped regular file is stored in the /.glusterfs_xattr_inode of the
+   export dir. All xattr ops related to the special files are redirected to this
+   map file.
+*/
 
-int 
-solaris_setxattr(const char *path, 
-		 const char* key, 
-		 const char *value, 
-		 size_t size, 
-		 int flags)
+int
+make_export_path(const char *real_path, char **path)
 {
-	int attrfd = -1;
-	int ret = 0;
-	
-	attrfd = attropen (path, key, flags|O_CREAT|O_WRONLY, 0777);
-	if (attrfd >= 0) {
-		ftruncate (attrfd, 0);
-		ret = write (attrfd, value, size);
-		close (attrfd);
-	} else {
-		if (errno != ENOENT)
-			gf_log ("libglusterfs", GF_LOG_ERROR, 
-				"Couldn't set extended attribute for %s (%d)", 
-				path, errno);
-		return -1;
-	}
-	
-	return 0;
+    int ret = -1;
+    char *tmp = NULL;
+    char *export_path = NULL;
+    char *dup = NULL;
+    char *ptr = NULL;
+    char *freeptr = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    export_path = GF_CALLOC(1, sizeof(char) * PATH_MAX, 0);
+    if (!export_path)
+        goto out;
+
+    dup = gf_strdup(real_path);
+    if (!dup)
+        goto out;
+
+    freeptr = dup;
+    ret = solaris_getxattr("/", GFID_XATTR_KEY, gfid, 16);
+    /* Return value of getxattr */
+    if (ret == 16) {
+        if (__is_root_gfid(gfid)) {
+            strcat(export_path, "/");
+            ret = 0;
+            goto done;
+        }
+    }
+
+    do {
+        ptr = strtok_r(dup, "/", &tmp);
+        if (!ptr)
+            break;
+        strcat(export_path, dup);
+        ret = solaris_getxattr(export_path, GFID_XATTR_KEY, gfid, 16);
+        if (ret == 16) {
+            if (__is_root_gfid(gfid)) {
+                ret = 0;
+                goto done;
+            }
+        }
+        strcat(export_path, "/");
+        dup = tmp;
+    } while (ptr);
+
+    goto out;
+
+done:
+    if (!ret) {
+        *path = export_path;
+    }
+out:
+    GF_FREE(freeptr);
+    if (ret && export_path)
+        GF_FREE(export_path);
+
+    return ret;
 }
+int
+solaris_xattr_resolve_path(const char *real_path, char **path)
+{
+    int ret = -1;
+    char *export_path = NULL;
+    char xattr_path[PATH_MAX] = {
+        0,
+    };
+    struct stat lstatbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct stat statbuf = {
+        0,
+    };
+
+    ret = lstat(real_path, &lstatbuf);
+    if (ret != 0)
+        return ret;
+    iatt_from_stat(&stbuf, &lstatbuf);
+    if (IA_ISREG(stbuf.ia_type) || IA_ISDIR(stbuf.ia_type))
+        return -1;
+
+    ret = make_export_path(real_path, &export_path);
+    if (!ret && export_path) {
+        strcat(export_path, "/" GF_SOLARIS_XATTR_DIR);
+        if (lstat(export_path, &statbuf)) {
+            ret = mkdir(export_path, 0755);
+            if (ret && (errno != EEXIST)) {
+                gf_msg_debug(THIS->name, 0,
+                             "mkdir failed,"
+                             " errno: %d",
+                             errno);
+                goto out;
+            }
+        }
+
+        snprintf(xattr_path, PATH_MAX, "%s%s%lu", export_path, "/",
+                 stbuf.ia_ino);
 
+        ret = lstat(xattr_path, &statbuf);
+
+        if (ret) {
+            ret = mknod(xattr_path, S_IFREG | O_WRONLY, 0);
+            if (ret && (errno != EEXIST)) {
+                gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                       "Failed to "
+                       "create mapped file %s",
+                       xattr_path);
+                goto out;
+            }
+        }
+        *path = gf_strdup(xattr_path);
+    }
+out:
+    GF_FREE(export_path);
+    if (*path)
+        return 0;
+    else
+        return -1;
+}
 
 int
-solaris_listxattr(const char *path, 
-		  char *list, 
-		  size_t size)
+solaris_setxattr(const char *path, const char *key, const char *value,
+                 size_t size, int flags)
 {
-	int attrdirfd = -1;
-	ssize_t len = 0;
-	DIR *dirptr = NULL;
-	struct dirent *dent = NULL;
-	int newfd = -1;
-	
-	attrdirfd = attropen (path, ".", O_RDONLY, 0);
-	if (attrdirfd >= 0) {
-		newfd = dup(attrdirfd);
-		dirptr = fdopendir(newfd);
-		if (dirptr) {
-			while ((dent = readdir(dirptr))) {
-				size_t listlen = strlen(dent->d_name);
-				if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
-					/* we don't want "." and ".." here */
-					continue;
-				}
-				if (size == 0) {
-					/* return the current size of the list of extended attribute names*/
-					len += listlen + 1;
-				} else {
-					/* check size and copy entrie + nul into list. */
-					if ((len + listlen + 1) > size) {
-						errno = ERANGE;
-						len = -1;
-						break;
-					} else {
-						strncpy(list + len, dent->d_name, listlen);
-						len += listlen;
-						list[len] = '\0';
-						++len;
-					}
-				}
-			}
-			
-			if (closedir(dirptr) == -1) {
-				close (attrdirfd);
-				return -1;
-			}
-		} else {
-			close (attrdirfd);
-			return -1;
-		}
-		close (attrdirfd);
-	}
-	return len;
+    int attrfd = -1;
+    int ret = 0;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, key, flags | O_CREAT | O_WRONLY, 0777);
+    } else {
+        attrfd = attropen(path, key, flags | O_CREAT | O_WRONLY, 0777);
+    }
+    if (attrfd >= 0) {
+        ftruncate(attrfd, 0);
+        ret = write(attrfd, value, size);
+        close(attrfd);
+        ret = 0;
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_ERROR, errno,
+                   LG_MSG_SET_ATTRIBUTE_FAILED,
+                   "Couldn't set "
+                   "extended attribute for %s",
+                   path);
+        ret = -1;
+    }
+    GF_FREE(mapped_path);
+    return ret;
 }
 
+int
+solaris_listxattr(const char *path, char *list, size_t size)
+{
+    int attrdirfd = -1;
+    ssize_t len = 0;
+    DIR *dirptr = NULL;
+    struct dirent *dent = NULL;
+    int newfd = -1;
+    char *mapped_path = NULL;
+    int ret = -1;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrdirfd = attropen(mapped_path, ".", O_RDONLY, 0);
+    } else {
+        attrdirfd = attropen(path, ".", O_RDONLY, 0);
+    }
+    if (attrdirfd >= 0) {
+        newfd = dup(attrdirfd);
+        dirptr = fdopendir(newfd);
+        if (dirptr) {
+            while ((dent = readdir(dirptr))) {
+                size_t listlen = strlen(dent->d_name);
+                if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
+                    /* we don't want "." and ".." here */
+                    continue;
+                }
+                if (size == 0) {
+                    /* return the current size of the list
+                       of extended attribute names*/
+                    len += listlen + 1;
+                } else {
+                    /* check size and copy entry + null
+                       into list. */
+                    if ((len + listlen + 1) > size) {
+                        errno = ERANGE;
+                        len = -1;
+                        break;
+                    } else {
+                        strncpy(list + len, dent->d_name, listlen);
+                        len += listlen;
+                        list[len] = '\0';
+                        ++len;
+                    }
+                }
+            }
+
+            if (closedir(dirptr) == -1) {
+                close(attrdirfd);
+                len = -1;
+                goto out;
+            }
+        } else {
+            close(attrdirfd);
+            len = -1;
+            goto out;
+        }
+        close(attrdirfd);
+    }
+out:
+    GF_FREE(mapped_path);
+    return len;
+}
 
 int
-solaris_flistxattr(int fd,
-                   char *list, 
-                   size_t size)
+solaris_flistxattr(int fd, char *list, size_t size)
 {
-	int attrdirfd = -1;
-	ssize_t len = 0;
-	DIR *dirptr = NULL;
-	struct dirent *dent = NULL;
-	int newfd = -1;
-	
-	attrdirfd = openat (fd, ".", O_RDONLY, 0);
-	if (attrdirfd >= 0) {
-		newfd = dup(attrdirfd);
-		dirptr = fdopendir(newfd);
-		if (dirptr) {
-			while ((dent = readdir(dirptr))) {
-				size_t listlen = strlen(dent->d_name);
-				if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
-					/* we don't want "." and ".." here */
-					continue;
-				}
-				if (size == 0) {
-					/* return the current size of the list of extended attribute names*/
-					len += listlen + 1;
-				} else {
-					/* check size and copy entrie + nul into list. */
-					if ((len + listlen + 1) > size) {
-						errno = ERANGE;
-						len = -1;
-						break;
-					} else {
-						strncpy(list + len, dent->d_name, listlen);
-						len += listlen;
-						list[len] = '\0';
-						++len;
-					}
-				}
-			}
-			
-			if (closedir(dirptr) == -1) {
-				close (attrdirfd);
-				return -1;
-			}
-		} else {
-			close (attrdirfd);
-			return -1;
-		}
-		close (attrdirfd);
-	}
-	return len;
+    int attrdirfd = -1;
+    ssize_t len = 0;
+    DIR *dirptr = NULL;
+    struct dirent *dent = NULL;
+    int newfd = -1;
+
+    attrdirfd = openat(fd, ".", O_RDONLY, 0);
+    if (attrdirfd >= 0) {
+        newfd = dup(attrdirfd);
+        dirptr = fdopendir(newfd);
+        if (dirptr) {
+            while ((dent = readdir(dirptr))) {
+                size_t listlen = strlen(dent->d_name);
+                if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
+                    /* we don't want "." and ".." here */
+                    continue;
+                }
+                if (size == 0) {
+                    /* return the current size of the list
+                       of extended attribute names*/
+                    len += listlen + 1;
+                } else {
+                    /* check size and copy entry + null
+                       into list. */
+                    if ((len + listlen + 1) > size) {
+                        errno = ERANGE;
+                        len = -1;
+                        break;
+                    } else {
+                        strncpy(list + len, dent->d_name, listlen);
+                        len += listlen;
+                        list[len] = '\0';
+                        ++len;
+                    }
+                }
+            }
+
+            if (closedir(dirptr) == -1) {
+                close(attrdirfd);
+                return -1;
+            }
+        } else {
+            close(attrdirfd);
+            return -1;
+        }
+        close(attrdirfd);
+    }
+    return len;
 }
 
+int
+solaris_removexattr(const char *path, const char *key)
+{
+    int ret = -1;
+    int attrfd = -1;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, ".", O_RDONLY, 0);
+    } else {
+        attrfd = attropen(path, ".", O_RDONLY, 0);
+    }
+    if (attrfd >= 0) {
+        ret = unlinkat(attrfd, key, 0);
+        close(attrfd);
+    } else {
+        if (errno == ENOENT)
+            errno = ENODATA;
+        ret = -1;
+    }
+
+    GF_FREE(mapped_path);
+
+    return ret;
+}
 
-int 
-solaris_removexattr(const char *path, 
-		    const char* key)
+int
+solaris_getxattr(const char *path, const char *key, char *value, size_t size)
 {
-	int ret = -1;
-	int attrfd = attropen (path, ".", O_RDONLY, 0);
-	if (attrfd >= 0) {
-		ret = unlinkat (attrfd, key, 0);
-		close (attrfd);
-	} else {
-		if (errno == ENOENT)
-			errno = ENODATA;
-		return -1;
-	}
-	
-	return ret;
+    int attrfd = -1;
+    int ret = 0;
+    char *mapped_path = NULL;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+    if (!ret) {
+        attrfd = attropen(mapped_path, key, O_RDONLY, 0);
+    } else {
+        attrfd = attropen(path, key, O_RDONLY, 0);
+    }
+
+    if (attrfd >= 0) {
+        if (size == 0) {
+            struct stat buf;
+            fstat(attrfd, &buf);
+            ret = buf.st_size;
+        } else {
+            ret = read(attrfd, value, size);
+        }
+        close(attrfd);
+    } else {
+        if (errno != ENOENT)
+            gf_msg("libglusterfs", GF_LOG_INFO, errno,
+                   LG_MSG_READ_ATTRIBUTE_FAILED,
+                   "Couldn't read "
+                   "extended attribute for the file %s",
+                   path);
+        if (errno == ENOENT)
+            errno = ENODATA;
+        ret = -1;
+    }
+    GF_FREE(mapped_path);
+    return ret;
 }
 
-int 
-solaris_getxattr(const char *path, 
-		 const char* key, 
-		 char *value, 
-		 size_t size)
+char *
+strsep(char **str, const char *delims)
 {
-	int attrfd = -1;
-	int ret = 0;
-	
-	attrfd = attropen (path, key, O_RDONLY, 0);
-	if (attrfd >= 0) {
-		if (size == 0) {
-			struct stat buf;
-			fstat (attrfd, &buf);
-			ret = buf.st_size;
-		} else {
-			ret = read (attrfd, value, size);
-		}
-		close (attrfd);
-	} else {
-		if (errno == ENOENT)
-			errno = ENODATA;
-		if (errno != ENOENT)
-			gf_log ("libglusterfs", GF_LOG_DEBUG, 
-				"Couldn't read extended attribute for the file %s (%d)", 
-				path, errno);
-		return -1;
-	}
-	return ret;
+    char *token;
+
+    if (*str == NULL) {
+        /* No more tokens */
+        return NULL;
+    }
+
+    token = *str;
+    while (**str != '\0') {
+        if (strchr(delims, **str) != NULL) {
+            **str = '\0';
+            (*str)++;
+            return token;
+        }
+        (*str)++;
+    }
+    /* There is no other token */
+    *str = NULL;
+    return token;
 }
 
+/* Code comes from libiberty */
 
-char* strsep(char** str, const char* delims)
+int
+vasprintf(char **result, const char *format, va_list args)
 {
-	char* token;
-	
-	if (*str==NULL) {
-		/* No more tokens */
-		return NULL;
-	}
-	
-	token=*str;
-	while (**str!='\0') {
-		if (strchr(delims,**str)!=NULL) {
-			**str='\0';
-			(*str)++;
-			return token;
-		}
-		(*str)++;
-	}
-	/* There is no other token */
-	*str=NULL;
-	return token;
+    return gf_vasprintf(result, format, args);
 }
 
-/* Code comes from libiberty */
+int
+asprintf(char **buf, const char *fmt, ...)
+{
+    int status;
+    va_list ap;
+
+    va_start(ap, fmt);
+    status = vasprintf(buf, fmt, ap);
+    va_end(ap);
+    return status;
+}
 
 int
-vasprintf (char **result, const char *format, va_list args)
+solaris_unlink(const char *path)
 {
-  return gf_vasprintf(result, format, args);
+    char *mapped_path = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    ret = solaris_xattr_resolve_path(path, &mapped_path);
+
+    if (!ret && mapped_path) {
+        if (lstat(path, &stbuf)) {
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                   "Stat failed on "
+                   "mapped file %s",
+                   mapped_path);
+            goto out;
+        }
+        if (stbuf.st_nlink == 1) {
+            if (remove(mapped_path))
+                gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                       "Failed to "
+                       "remove mapped file %s",
+                       mapped_path);
+        }
+    }
+
+out:
+    GF_FREE(mapped_path);
+
+    return unlink(path);
 }
 
 int
-asprintf (char **buf, const char *fmt, ...)
+solaris_rename(const char *old_path, const char *new_path)
 {
-  int status;
-  va_list ap;
+    char *mapped_path = NULL;
+    int ret = -1;
+
+    ret = solaris_xattr_resolve_path(new_path, &mapped_path);
+
+    if (!ret && mapped_path) {
+        if (!remove(mapped_path))
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, LG_MSG_FILE_OP_FAILED,
+                   "Failed to remove "
+                   "mapped file %s.",
+                   mapped_path);
+        GF_FREE(mapped_path);
+    }
 
-  va_start (ap, fmt);
-  status = vasprintf (buf, fmt, ap);
-  va_end (ap);
-  return status;  
+    return rename(old_path, new_path);
+}
+
+char *
+mkdtemp(char *tempstring)
+{
+    char *new_string = NULL;
+    int ret = 0;
+
+    new_string = mkstemp(tempstring);
+    if (!new_string)
+        goto out;
+
+    ret = mkdir(new_string, 0700);
+    if (ret < 0)
+        new_string = NULL;
+
+out:
+    return new_string;
 }
 
 #endif /* GF_SOLARIS_HOST_OS */
 
+#ifdef GF_BSD_HOST_OS
+void
+gf_extattr_list_reshape(char *bsd_list, ssize_t size)
+{
+    /*
+     * the format of bsd_list is
+     *     <attr_len>attr<attr_len>attr...
+     * we try to reformat it as Linux's
+     *     attr<\0>attr<\0>...
+     * */
+    if (NULL == bsd_list || size <= 0)
+        return;
+
+    size_t i = 0, j;
+
+    while (i < size) {
+        size_t attr_len = bsd_list[i];
+
+        for (j = i; j < i + attr_len; ++j)
+            bsd_list[j] = bsd_list[j + 1];
+        bsd_list[j] = '\0';
+
+        i += attr_len + 1;
+        gf_msg_debug("syscall", 0, "syscall debug: %lu", attr_len);
+    }
+}
+#endif /* GF_BSD_HOST_OS */
+
 #ifndef HAVE_STRNLEN
-size_t 
-strnlen(const char *string, size_t maxlen)                   
+size_t
+strnlen(const char *string, size_t maxlen)
 {
-	int len = 0;
-	while ((len < maxlen) && string[len])
-		len++;
-	return len;
+    int len = 0;
+    while ((len < maxlen) && string[len])
+        len++;
+    return len;
 }
 #endif /* STRNLEN */
+
+int
+gf_umount_lazy(char *xlname, char *path, int rmdir_flag)
+{
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+#ifdef GF_LINUX_HOST_OS
+    runner_add_args(&runner, _PATH_UMOUNT, "-l", path, NULL);
+#else
+    if (rmdir_flag)
+        runner_add_args(&runner, SBIN_DIR "/umountd", "-r", path, NULL);
+    else
+        runner_add_args(&runner, SBIN_DIR "/umountd", path, NULL);
+#endif
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(xlname, GF_LOG_ERROR, errno, LG_MSG_UNMOUNT_FAILED,
+               "Lazy unmount of %s", path);
+    }
+
+#ifdef GF_LINUX_HOST_OS
+    if (!ret && rmdir_flag) {
+        ret = sys_rmdir(path);
+        if (ret)
+            gf_msg(xlname, GF_LOG_WARNING, errno, LG_MSG_DIR_OP_FAILED,
+                   "rmdir %s", path);
+    }
+#endif
+
+    return ret;
+}
diff --git a/libglusterfs/src/compat.h b/libglusterfs/src/compat.h
deleted file mode 100644
index 690b4f9e689..00000000000
--- a/libglusterfs/src/compat.h
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __COMPAT_H__
-#define __COMPAT_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include "dict.h"
-
-#ifndef LLONG_MAX
-#define LLONG_MAX __LONG_LONG_MAX__ /* compat with old gcc */
-#endif /* LLONG_MAX */
-
-
-#ifdef GF_LINUX_HOST_OS
-
-#define UNIX_PATH_MAX 108
-
-#include <sys/un.h>
-#include <linux/limits.h>
-#include <sys/xattr.h>
-#include <endian.h>
-
-
-#ifndef HAVE_LLISTXATTR
-
-/* This part is valid only incase of old glibc which doesn't support 
- * 'llistxattr()' system calls.
- */
-
-#define lremovexattr(path,key) removexattr(path,key)
-#define llistxattr(path,key,size)  listxattr(path,key,size)
-#define lgetxattr(path, key, value, size) getxattr(path,key,value,size)
-#define lsetxattr(path,key,value,size,flags) setxattr(path,key,value,size,flags)
-
-#endif /* HAVE_LLISTXATTR */
-#endif /* GF_LINUX_HOST_OS */
-
-#ifdef GF_BSD_HOST_OS 
-/* In case of FreeBSD */
-
-#define UNIX_PATH_MAX 104
-#include <sys/types.h>
-
-#include <sys/un.h>
-#include <sys/endian.h>
-#include <sys/extattr.h>
-#include <limits.h>
-
-#include <libgen.h>
-
-enum {
-        ATTR_CREATE = 1,
-#define XATTR_CREATE ATTR_CREATE
-        ATTR_REPLACE = 2
-#define XATTR_REPLACE ATTR_REPLACE
-};
-
-
-#ifndef sighandler_t
-#define sighandler_t sig_t
-#endif
-
-#ifndef ino64_t
-#define ino64_t ino_t
-#endif
-
-#ifndef EUCLEAN
-#define EUCLEAN 0
-#endif 
-
-#include <netinet/in.h>
-#ifndef s6_addr16
-#define s6_addr16 __u6_addr.__u6_addr16
-#endif
-#ifndef s6_addr32
-#define s6_addr32 __u6_addr.__u6_addr32
-#endif 
-
-/* Posix dictates NAME_MAX to be used */
-# ifndef NAME_MAX
-#  ifdef  MAXNAMLEN
-#   define NAME_MAX MAXNAMLEN
-#  else
-#   define NAME_MAX 255
-#  endif
-# endif
-        
-#define F_GETLK64       F_GETLK
-#define F_SETLK64       F_SETLK
-#define F_SETLKW64      F_SETLKW
-
-#endif /* GF_BSD_HOST_OS */
-
-#ifdef GF_DARWIN_HOST_OS
-
-#define UNIX_PATH_MAX 104
-#include <sys/types.h>
-
-#include <sys/un.h>
-#include <machine/endian.h>
-#include <sys/xattr.h>
-#include <limits.h>
-
-#include <libgen.h>
-
-
-#if __DARWIN_64_BIT_INO_T == 0
-#    error '64 bit ino_t is must for GlusterFS to work, Compile with "CFLAGS=-D__DARWIN_64_BIT_INO_T"'
-#endif /* __DARWIN_64_BIT_INO_T */
-
-
-#if __DARWIN_64_BIT_INO_T == 0
-#    error '64 bit ino_t is must for GlusterFS to work, Compile with "CFLAGS=-D__DARWIN_64_BIT_INO_T"'
-#endif /* __DARWIN_64_BIT_INO_T */
-
-#ifndef sighandler_t
-#define sighandler_t sig_t
-#endif
-
-#ifndef EUCLEAN
-#define EUCLEAN 0
-#endif
-
-#include <netinet/in.h>
-#ifndef s6_addr16
-#define s6_addr16 __u6_addr.__u6_addr16
-#endif 
-#ifndef s6_addr32
-#define s6_addr32 __u6_addr.__u6_addr32
-#endif 
-
-/* Posix dictates NAME_MAX to be used */
-# ifndef NAME_MAX
-#  ifdef  MAXNAMLEN
-#   define NAME_MAX MAXNAMLEN
-#  else
-#   define NAME_MAX 255
-#  endif
-# endif
-
-#define F_GETLK64       F_GETLK
-#define F_SETLK64       F_SETLK
-#define F_SETLKW64      F_SETLKW
-
-#ifndef FTW_CONTINUE
-  #define FTW_CONTINUE 0
-#endif
-
-int32_t gf_darwin_compat_listxattr (int len, dict_t *dict, int size);
-int32_t gf_darwin_compat_getxattr (const char *key, dict_t *dict);
-int32_t gf_darwin_compat_setxattr (dict_t *dict);
-
-#endif /* GF_DARWIN_HOST_OS */
-
-#ifdef GF_SOLARIS_HOST_OS
-
-#define UNIX_PATH_MAX 108
-#define EUCLEAN 117 
-
-#include <sys/un.h>
-#include <limits.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <sys/fcntl.h>
-#include <libgen.h>
-#include <sys/mkdev.h>
-
-#ifndef lchmod
-#define lchmod chmod
-#endif 
-
-#define lgetxattr(path, key, value, size) solaris_getxattr(path,key,value,size)
-enum {
-        ATTR_CREATE = 1,
-#define XATTR_CREATE ATTR_CREATE
-        ATTR_REPLACE = 2
-#define XATTR_REPLACE ATTR_REPLACE
-};
-
-/* This patch is not present in Solaris 10 and before */
-#ifndef dirfd
-#define dirfd(dirp)   ((dirp)->dd_fd)
-#endif
-
-/* Posix dictates NAME_MAX to be used */
-# ifndef NAME_MAX
-#  ifdef  MAXNAMLEN
-#   define NAME_MAX MAXNAMLEN
-#  else 
-#   define NAME_MAX 255
-#  endif
-# endif 
-
-#include <netinet/in.h>
-#ifndef s6_addr16
-#define S6_ADDR16(x)    ((uint16_t*) ((char*)&(x).s6_addr))
-#endif
-#ifndef s6_addr32
-#define s6_addr32       _S6_un._S6_u32
-#endif
-
-#define lutimes(filename,times)              utimes(filename,times)
-
-#ifndef SEEK_SET
-#define SEEK_SET 0
-#endif
-
-enum {
-        DT_UNKNOWN = 0,
-# define DT_UNKNOWN	DT_UNKNOWN
-        DT_FIFO = 1,
-# define DT_FIFO	DT_FIFO
-        DT_CHR = 2,
-# define DT_CHR		DT_CHR
-        DT_DIR = 4,
-# define DT_DIR		DT_DIR
-        DT_BLK = 6,
-# define DT_BLK		DT_BLK
-        DT_REG = 8,
-# define DT_REG		DT_REG
-        DT_LNK = 10,
-# define DT_LNK		DT_LNK
-        DT_SOCK = 12,
-# define DT_SOCK	DT_SOCK
-        DT_WHT = 14
-# define DT_WHT		DT_WHT
-};
-
-#ifndef _PATH_MOUNTED
- #define _PATH_MOUNTED "/etc/mtab"
-#endif
-
-#ifndef O_ASYNC
-  #ifdef FASYNC
-    #define O_ASYNC FASYNC
-  #else
-    #define O_ASYNC 0
-  #endif
-#endif
-
-#ifndef FTW_CONTINUE
-  #define FTW_CONTINUE 0
-#endif
-
-int asprintf(char **string_ptr, const char *format, ...); 
-
-int vasprintf (char **result, const char *format, va_list args);
-char* strsep(char** str, const char* delims);
-int solaris_listxattr(const char *path, char *list, size_t size);
-int solaris_removexattr(const char *path, const char* key);
-int solaris_getxattr(const char *path, const char* key, 
-                     char *value, size_t size);
-int solaris_setxattr(const char *path, const char* key, const char *value, 
-                     size_t size, int flags);
-int solaris_fgetxattr(int fd, const char* key,
-                      char *value, size_t size);
-int solaris_fsetxattr(int fd, const char* key, const char *value, 
-                      size_t size, int flags);
-int solaris_flistxattr(int fd, char *list, size_t size);
-
-#endif /* GF_SOLARIS_HOST_OS */
-
-#ifndef HAVE_ARGP
-#include "argp.h"
-#else
-#include <argp.h>
-#endif /* HAVE_ARGP */
-
-#ifndef HAVE_STRNLEN
-size_t strnlen(const char *string, size_t maxlen);                   
-#endif /* STRNLEN */
-
-#ifndef strdupa
-#define strdupa(s)                                                      \
-        (__extension__                                                  \
-         ({                                                             \
-                 __const char *__old = (s);                             \
-                 size_t __len = strlen (__old) + 1;                     \
-                 char *__new = (char *) __builtin_alloca (__len);       \
-                 (char *) memcpy (__new, __old, __len);                 \
-         }))
-#endif 
-
-#define GF_DIR_ALIGN(x) (((x) + sizeof (uint64_t) - 1) & ~(sizeof (uint64_t) - 1))
-
-#include <sys/types.h>
-#include <dirent.h>
-
-static inline int32_t
-dirent_size (struct dirent *entry)
-{
-#ifdef GF_BSD_HOST_OS
-        return GF_DIR_ALIGN (24 /* FIX MEEEE!!! */ + entry->d_namlen);
-#endif
-#ifdef GF_DARWIN_HOST_OS
-        return GF_DIR_ALIGN (24 /* FIX MEEEE!!! */ + entry->d_namlen);
-#endif
-#ifdef GF_LINUX_HOST_OS
-        return GF_DIR_ALIGN (24 /* FIX MEEEE!!! */ + entry->d_reclen);
-#endif
-#ifdef GF_SOLARIS_HOST_OS
-        return GF_DIR_ALIGN (24 /* FIX MEEEE!!! */ + entry->d_reclen);
-#endif
-}
-
-
-#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-/* Linux, Solaris, Cygwin */
-#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atim.tv_nsec)
-#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctim.tv_nsec)
-#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtim.tv_nsec)
-#define ST_ATIM_NSEC_SET(stbuf, val) ((stbuf)->st_atim.tv_nsec = (val))
-#define ST_MTIM_NSEC_SET(stbuf, val) ((stbuf)->st_mtim.tv_nsec = (val))
-#define ST_CTIM_NSEC_SET(stbuf, val) ((stbuf)->st_ctim.tv_nsec = (val))
-#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC)
-/* FreeBSD, NetBSD */
-#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atimespec.tv_nsec)
-#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctimespec.tv_nsec)
-#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtimespec.tv_nsec)
-#define ST_ATIM_NSEC_SET(stbuf, val) ((stbuf)->st_atimespec.tv_nsec = (val))
-#define ST_MTIM_NSEC_SET(stbuf, val) ((stbuf)->st_mtimespec.tv_nsec = (val))
-#define ST_CTIM_NSEC_SET(stbuf, val) ((stbuf)->st_ctimespec.tv_nsec = (val))
-#else
-#define ST_ATIM_NSEC(stbuf) (0)
-#define ST_CTIM_NSEC(stbuf) (0)
-#define ST_MTIM_NSEC(stbuf) (0)
-#define ST_ATIM_NSEC_SET(stbuf, val) do { } while (0);
-#define ST_MTIM_NSEC_SET(stbuf, val) do { } while (0);
-#define ST_CTIM_NSEC_SET(stbuf, val) do { } while (0);
-#endif
-
-#ifndef IXDR_GET_LONG
-#define IXDR_GET_LONG(buf) ((long)IXDR_GET_U_INT32(buf))
-#endif
-
-#ifndef IXDR_PUT_LONG
-#define IXDR_PUT_LONG(buf, v) ((long)IXDR_PUT_INT32(buf, (long)(v)))
-#endif
-
-#ifndef IXDR_GET_U_LONG
-#define IXDR_GET_U_LONG(buf)          ((u_long)IXDR_GET_LONG(buf))
-#endif
-
-#ifndef IXDR_PUT_U_LONG
-#define IXDR_PUT_U_LONG(buf, v)       IXDR_PUT_LONG(buf, (long)(v))
-#endif
-
-#endif /* __COMPAT_H__ */
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
new file mode 100644
index 00000000000..3d890b04ec9
--- /dev/null
+++ b/libglusterfs/src/ctx.c
@@ -0,0 +1,97 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <pthread.h>
+
+#include "glusterfs/globals.h"
+#include "glusterfs/glusterfs.h"
+#include "timer-wheel.h"
+
+glusterfs_ctx_t *
+glusterfs_ctx_new()
+{
+    glusterfs_ctx_t *ctx = NULL;
+
+    /* no GF_CALLOC here, gf_acct_mem_set_enable is not
+       yet decided at this point */
+    ctx = CALLOC(1, sizeof(*ctx));
+    if (!ctx) {
+        goto out;
+    }
+
+    ctx->mem_acct_enable = gf_global_mem_acct_enable_get();
+
+    INIT_LIST_HEAD(&ctx->graphs);
+    INIT_LIST_HEAD(&ctx->mempool_list);
+    INIT_LIST_HEAD(&ctx->volfile_list);
+
+    ctx->daemon_pipe[0] = -1;
+    ctx->daemon_pipe[1] = -1;
+
+    ctx->log.loglevel = DEFAULT_LOG_LEVEL;
+
+#if defined(RUN_WITH_MEMCHECK)
+    ctx->cmd_args.vgtool = _gf_memcheck;
+#elif defined(RUN_WITH_DRD)
+    ctx->cmd_args.vgtool = _gf_drd;
+#else
+    ctx->cmd_args.vgtool = _gf_none;
+#endif
+
+    /* lock is never destroyed! */
+    if (LOCK_INIT(&ctx->lock)) {
+        free(ctx);
+        ctx = NULL;
+        goto out;
+    }
+
+    GF_ATOMIC_INIT(ctx->stats.max_dict_pairs, 0);
+    GF_ATOMIC_INIT(ctx->stats.total_pairs_used, 0);
+    GF_ATOMIC_INIT(ctx->stats.total_dicts_used, 0);
+out:
+    return ctx;
+}
+
+static void
+glusterfs_ctx_tw_destroy(struct gf_ctx_tw *ctx_tw)
+{
+    if (ctx_tw->timer_wheel)
+        gf_tw_cleanup_timers(ctx_tw->timer_wheel);
+
+    GF_FREE(ctx_tw);
+}
+
+struct tvec_base *
+glusterfs_ctx_tw_get(glusterfs_ctx_t *ctx)
+{
+    struct gf_ctx_tw *ctx_tw = NULL;
+
+    LOCK(&ctx->lock);
+    {
+        if (ctx->tw) {
+            ctx_tw = GF_REF_GET(ctx->tw);
+        } else {
+            ctx_tw = GF_CALLOC(1, sizeof(struct gf_ctx_tw),
+                               gf_common_mt_tw_ctx);
+            ctx_tw->timer_wheel = gf_tw_init_timers();
+            GF_REF_INIT(ctx_tw, glusterfs_ctx_tw_destroy);
+            ctx->tw = ctx_tw;
+        }
+    }
+    UNLOCK(&ctx->lock);
+
+    return ctx_tw->timer_wheel;
+}
+
+void
+glusterfs_ctx_tw_put(glusterfs_ctx_t *ctx)
+{
+    GF_REF_PUT(ctx->tw);
+}
diff --git a/libglusterfs/src/daemon.c b/libglusterfs/src/daemon.c
new file mode 100644
index 00000000000..0a3e5438325
--- /dev/null
+++ b/libglusterfs/src/daemon.c
@@ -0,0 +1,65 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <stdio.h>
+#include "glusterfs/daemon.h"
+
+int
+os_daemon_return(int nochdir, int noclose)
+{
+    pid_t pid = -1;
+    int ret = -1;
+    FILE *ptr = NULL;
+
+    ret = fork();
+    if (ret)
+        return ret;
+
+    pid = setsid();
+
+    if (pid == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    if (!nochdir)
+        ret = chdir("/");
+
+    if (!noclose) {
+        ptr = freopen(DEVNULLPATH, "r", stdin);
+        if (!ptr)
+            goto out;
+
+        ptr = freopen(DEVNULLPATH, "w", stdout);
+        if (!ptr)
+            goto out;
+
+        ptr = freopen(DEVNULLPATH, "w", stderr);
+        if (!ptr)
+            goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+os_daemon(int nochdir, int noclose)
+{
+    int ret = -1;
+
+    ret = os_daemon_return(nochdir, noclose);
+    if (ret <= 0)
+        return ret;
+
+    _exit(0);
+}
diff --git a/libglusterfs/src/default-args.c b/libglusterfs/src/default-args.c
new file mode 100644
index 00000000000..a0ba1cfb299
--- /dev/null
+++ b/libglusterfs/src/default-args.c
@@ -0,0 +1,1651 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs/defaults.h"
+
+int
+args_lookup_store(default_args_t *args, loc_t *loc, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_lookup_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, inode_t *inode, struct iatt *buf,
+                      dict_t *xdata, struct iatt *postparent)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_stat_store(default_args_t *args, loc_t *loc, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_stat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    struct iatt *buf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fstat_store(default_args_t *args, fd_t *fd, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fstat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *buf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (buf)
+        args->stat = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_truncate_store(default_args_t *args, loc_t *loc, off_t off, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_truncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_ftruncate_store(default_args_t *args, fd_t *fd, off_t off, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_ftruncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_access_store(default_args_t *args, loc_t *loc, int32_t mask, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->mask = mask;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_access_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_readlink_store(default_args_t *args, loc_t *loc, size_t size,
+                    dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->size = size;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_readlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, const char *path, struct iatt *stbuf,
+                        dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (path)
+        args->buf = gf_strdup(path);
+    if (stbuf)
+        args->stat = *stbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_mknod_store(default_args_t *args, loc_t *loc, mode_t mode, dev_t rdev,
+                 mode_t umask, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->rdev = rdev;
+    args->umask = umask;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_mknod_cbk_store(default_args_cbk_t *args, int op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_mkdir_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+                 dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->umask = umask;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_mkdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_unlink_store(default_args_t *args, loc_t *loc, int xflag, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->xflag = xflag;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_unlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_rmdir_store(default_args_t *args, loc_t *loc, int flags, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_rmdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_symlink_store(default_args_t *args, const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata)
+{
+    args->linkname = gf_strdup(linkname);
+    args->umask = umask;
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_symlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, inode_t *inode, struct iatt *buf,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_rename_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                  dict_t *xdata)
+{
+    loc_copy(&args->loc, oldloc);
+    loc_copy(&args->loc2, newloc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_rename_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *buf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (buf)
+        args->stat = *buf;
+    if (preoldparent)
+        args->preparent = *preoldparent;
+    if (postoldparent)
+        args->postparent = *postoldparent;
+    if (prenewparent)
+        args->preparent2 = *prenewparent;
+    if (postnewparent)
+        args->postparent2 = *postnewparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_link_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                dict_t *xdata)
+{
+    loc_copy(&args->loc, oldloc);
+    loc_copy(&args->loc2, newloc);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_link_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_create_store(default_args_t *args, loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    args->mode = mode;
+    args->umask = umask;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_create_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, fd_t *fd, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (buf)
+        args->stat = *buf;
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_open_store(default_args_t *args, loc_t *loc, int32_t flags, fd_t *fd,
+                dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->flags = flags;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_open_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    fd_t *fd, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_readv_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                 uint32_t flags, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
+    args->flags = flags;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_readv_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iovec *vector, int32_t count, struct iatt *stbuf,
+                     struct iobref *iobref, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        args->vector = iov_dup(vector, count);
+        args->count = count;
+        args->stat = *stbuf;
+        args->iobref = iobref_ref(iobref);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_writev_store(default_args_t *args, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t off, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->vector = iov_dup(vector, count);
+    args->count = count;
+    args->offset = off;
+    args->flags = flags;
+    args->iobref = iobref_ref(iobref);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_writev_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0)
+        args->poststat = *postbuf;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_put_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+               uint32_t flags, struct iovec *vector, int32_t count, off_t off,
+               struct iobref *iobref, dict_t *xattr, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+    args->umask = umask;
+    args->flags = flags;
+    args->vector = iov_dup(vector, count);
+    args->count = count;
+    args->offset = off;
+    args->iobref = iobref_ref(iobref);
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_put_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0)
+        args->stat = *buf;
+    if (inode)
+        args->inode = inode_ref(inode);
+    if (preparent)
+        args->preparent = *preparent;
+    if (postparent)
+        args->postparent = *postparent;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+int
+args_flush_store(default_args_t *args, fd_t *fd, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_flush_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fsync_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                 dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->datasync = datasync;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fsync_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (prebuf)
+        args->prestat = *prebuf;
+    if (postbuf)
+        args->poststat = *postbuf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_opendir_store(default_args_t *args, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_opendir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (fd)
+        args->fd = fd_ref(fd);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fsyncdir_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                    dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->datasync = datasync;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+int
+args_fsyncdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_statfs_store(default_args_t *args, loc_t *loc, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_statfs_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->statvfs = *buf;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_setxattr_store(default_args_t *args, loc_t *loc, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    /* TODO */
+    if (dict)
+        args->xattr = dict_ref(dict);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_setxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_getxattr_store(default_args_t *args, loc_t *loc, const char *name,
+                    dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+
+    if (name)
+        args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_getxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (dict)
+        args->xattr = dict_ref(dict);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fsetxattr_store(default_args_t *args, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+
+    if (dict)
+        args->xattr = dict_ref(dict);
+    args->flags = flags;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fsetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fgetxattr_store(default_args_t *args, fd_t *fd, const char *name,
+                     dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+
+    if (name)
+        args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fgetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (dict)
+        args->xattr = dict_ref(dict);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_removexattr_store(default_args_t *args, loc_t *loc, const char *name,
+                       dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_removexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fremovexattr_store(default_args_t *args, fd_t *fd, const char *name,
+                        dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+    args->name = gf_strdup(name);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fremovexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                            int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_lk_store(default_args_t *args, fd_t *fd, int32_t cmd,
+              struct gf_flock *lock, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->cmd = cmd;
+    args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_lk_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                  struct gf_flock *lock, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_inodelk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    if (volume)
+        args->volume = gf_strdup(volume);
+
+    loc_copy(&args->loc, loc);
+    args->cmd = cmd;
+    args->lock = *lock;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_inodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_finodelk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    if (volume)
+        args->volume = gf_strdup(volume);
+
+    args->cmd = cmd;
+    args->lock = *lock;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_finodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_entrylk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   const char *name, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
+{
+    if (volume)
+        args->volume = gf_strdup(volume);
+
+    loc_copy(&args->loc, loc);
+
+    args->entrylkcmd = cmd;
+    args->entrylktype = type;
+
+    if (name)
+        args->name = gf_strdup(name);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_entrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fentrylk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    const char *name, entrylk_cmd cmd, entrylk_type type,
+                    dict_t *xdata)
+{
+    if (volume)
+        args->volume = gf_strdup(volume);
+
+    if (fd)
+        args->fd = fd_ref(fd);
+    args->entrylkcmd = cmd;
+    args->entrylktype = type;
+    if (name)
+        args->name = gf_strdup(name);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fentrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_readdirp_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                    dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_readdirp_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+    gf_dirent_t *stub_entry = NULL, *entry = NULL;
+
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            stub_entry = gf_dirent_for_name(entry->d_name);
+            if (!stub_entry)
+                goto out;
+            stub_entry->d_off = entry->d_off;
+            stub_entry->d_ino = entry->d_ino;
+            stub_entry->d_stat = entry->d_stat;
+            stub_entry->d_type = entry->d_type;
+            if (entry->inode)
+                stub_entry->inode = inode_ref(entry->inode);
+            if (entry->dict)
+                stub_entry->dict = dict_ref(entry->dict);
+            list_add_tail(&stub_entry->list, &args->entries.list);
+        }
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+out:
+    return 0;
+}
+
+int
+args_readdir_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                   dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+    args->size = size;
+    args->offset = off;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_readdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+    gf_dirent_t *stub_entry = NULL, *entry = NULL;
+
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            stub_entry = gf_dirent_for_name(entry->d_name);
+            if (!stub_entry)
+                goto out;
+            stub_entry->d_off = entry->d_off;
+            stub_entry->d_ino = entry->d_ino;
+            stub_entry->d_type = entry->d_type;
+            list_add_tail(&stub_entry->list, &args->entries.list);
+        }
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+out:
+    return 0;
+}
+
+int
+args_rchecksum_store(default_args_t *args, fd_t *fd, off_t offset, int32_t len,
+                     dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+    args->offset = offset;
+    args->size = len;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_rchecksum_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, uint32_t weak_checksum,
+                         uint8_t *strong_checksum, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        args->weak_checksum = weak_checksum;
+        args->strong_checksum = gf_memdup(strong_checksum,
+                                          SHA256_DIGEST_LENGTH);
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_xattrop_store(default_args_t *args, loc_t *loc, gf_xattrop_flags_t optype,
+                   dict_t *xattr, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+
+    args->optype = optype;
+    args->xattr = dict_ref(xattr);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_xattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fxattrop_store(default_args_t *args, fd_t *fd, gf_xattrop_flags_t optype,
+                    dict_t *xattr, dict_t *xdata)
+{
+    args->fd = fd_ref(fd);
+
+    args->optype = optype;
+    args->xattr = dict_ref(xattr);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fxattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xattr)
+        args->xattr = dict_ref(xattr);
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_setattr_store(default_args_t *args, loc_t *loc, struct iatt *stbuf,
+                   int32_t valid, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+
+    if (stbuf)
+        args->stat = *stbuf;
+
+    args->valid = valid;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_setattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fsetattr_store(default_args_t *args, fd_t *fd, struct iatt *stbuf,
+                    int32_t valid, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    if (stbuf)
+        args->stat = *stbuf;
+
+    args->valid = valid;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+int
+args_fsetattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_fallocate_store(default_args_t *args, fd_t *fd, int32_t mode, off_t offset,
+                     size_t len, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    args->flags = mode;
+    args->offset = offset;
+    args->size = len;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_fallocate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_discard_store(default_args_t *args, fd_t *fd, off_t offset, size_t len,
+                   dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    args->offset = offset;
+    args->size = len;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_discard_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_zerofill_store(default_args_t *args, fd_t *fd, off_t offset, off_t len,
+                    dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    args->offset = offset;
+    args->size = len;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_zerofill_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (statpre)
+        args->prestat = *statpre;
+    if (statpost)
+        args->poststat = *statpost;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_ipc_store(default_args_t *args, int32_t op, dict_t *xdata)
+{
+    args->cmd = op;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_ipc_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_seek_store(default_args_t *args, fd_t *fd, off_t offset,
+                gf_seek_what_t what, dict_t *xdata)
+{
+    if (fd)
+        args->fd = fd_ref(fd);
+
+    args->offset = offset;
+    args->what = what;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_seek_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    off_t offset, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    args->offset = offset;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_getactivelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, lock_migration_info_t *locklist,
+                           dict_t *xdata)
+{
+    lock_migration_info_t *stub_entry = NULL, *entry = NULL;
+    int ret = 0;
+
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    /*op_ret needs to carry the number of locks present in the list*/
+    if (op_ret > 0) {
+        list_for_each_entry(entry, &locklist->list, list)
+        {
+            stub_entry = GF_CALLOC(1, sizeof(*stub_entry), gf_common_mt_char);
+            if (!stub_entry) {
+                ret = -1;
+                goto out;
+            }
+
+            INIT_LIST_HEAD(&stub_entry->list);
+            stub_entry->flock = entry->flock;
+
+            stub_entry->lk_flags = entry->lk_flags;
+
+            stub_entry->client_uid = gf_strdup(entry->client_uid);
+            if (!stub_entry->client_uid) {
+                GF_FREE(stub_entry);
+                ret = -1;
+                goto out;
+            }
+
+            list_add_tail(&stub_entry->list, &args->locklist.list);
+        }
+    }
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+out:
+    return ret;
+}
+
+int
+args_setactivelk_store(default_args_t *args, loc_t *loc,
+                       lock_migration_info_t *locklist, dict_t *xdata)
+{
+    lock_migration_info_t *stub_entry = NULL, *entry = NULL;
+    int ret = 0;
+
+    list_for_each_entry(entry, &locklist->list, list)
+    {
+        stub_entry = GF_CALLOC(1, sizeof(*stub_entry), gf_common_mt_lock_mig);
+        if (!stub_entry) {
+            ret = -1;
+            goto out;
+        }
+
+        INIT_LIST_HEAD(&stub_entry->list);
+        stub_entry->flock = entry->flock;
+
+        stub_entry->lk_flags = entry->lk_flags;
+
+        stub_entry->client_uid = gf_strdup(entry->client_uid);
+        if (!stub_entry->client_uid) {
+            GF_FREE(stub_entry);
+            ret = -1;
+            goto out;
+        }
+
+        list_add_tail(&stub_entry->list, &args->locklist.list);
+    }
+
+    loc_copy(&args->loc, loc);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+out:
+    return ret;
+}
+
+void
+args_lease_store(default_args_t *args, loc_t *loc, struct gf_lease *lease,
+                 dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->lease = *lease;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return;
+}
+
+void
+args_lease_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct gf_lease *lease, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret == 0)
+        args->lease = *lease;
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+}
+
+int
+args_icreate_store(default_args_t *args, loc_t *loc, mode_t mode, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+    args->mode = mode;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_namelink_store(default_args_t *args, loc_t *loc, dict_t *xdata)
+{
+    loc_copy(&args->loc, loc);
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+    return 0;
+}
+
+int
+args_copy_file_range_store(default_args_t *args, fd_t *fd_in, off64_t off_in,
+                           fd_t *fd_out, off64_t off_out, size_t len,
+                           uint32_t flags, dict_t *xdata)
+{
+    if (fd_in)
+        args->fd = fd_ref(fd_in);
+    if (fd_out)
+        args->fd_dst = fd_ref(fd_out);
+    args->size = len;
+    args->off_in = off_in;
+    args->off_out = off_out;
+    args->flags = flags;
+
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+int
+args_copy_file_range_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                               int32_t op_errno, struct iatt *stbuf,
+                               struct iatt *prebuf_dst,
+                               struct iatt *postbuf_dst, dict_t *xdata)
+{
+    args->op_ret = op_ret;
+    args->op_errno = op_errno;
+    if (op_ret >= 0) {
+        if (postbuf_dst)
+            args->poststat = *postbuf_dst;
+        if (prebuf_dst)
+            args->prestat = *prebuf_dst;
+        if (stbuf)
+            args->stat = *stbuf;
+    }
+    if (xdata)
+        args->xdata = dict_ref(xdata);
+
+    return 0;
+}
+
+void
+args_cbk_wipe(default_args_cbk_t *args_cbk)
+{
+    if (!args_cbk)
+        return;
+    if (args_cbk->inode)
+        inode_unref(args_cbk->inode);
+
+    GF_FREE((char *)args_cbk->buf);
+
+    GF_FREE(args_cbk->vector);
+
+    if (args_cbk->iobref)
+        iobref_unref(args_cbk->iobref);
+
+    if (args_cbk->fd)
+        fd_unref(args_cbk->fd);
+
+    if (args_cbk->xattr)
+        dict_unref(args_cbk->xattr);
+
+    GF_FREE(args_cbk->strong_checksum);
+
+    if (args_cbk->xdata)
+        dict_unref(args_cbk->xdata);
+
+    if (!list_empty(&args_cbk->entries.list))
+        gf_dirent_free(&args_cbk->entries);
+}
+
+void
+args_wipe(default_args_t *args)
+{
+    if (!args)
+        return;
+
+    loc_wipe(&args->loc);
+
+    loc_wipe(&args->loc2);
+
+    if (args->fd)
+        fd_unref(args->fd);
+
+    GF_FREE((char *)args->linkname);
+
+    GF_FREE(args->vector);
+
+    if (args->iobref)
+        iobref_unref(args->iobref);
+
+    if (args->xattr)
+        dict_unref(args->xattr);
+
+    if (args->xdata)
+        dict_unref(args->xdata);
+
+    GF_FREE((char *)args->name);
+
+    GF_FREE((char *)args->volume);
+}
+
+void
+args_cbk_init(default_args_cbk_t *args_cbk)
+{
+    INIT_LIST_HEAD(&args_cbk->entries);
+}
diff --git a/libglusterfs/src/defaults-tmpl.c b/libglusterfs/src/defaults-tmpl.c
new file mode 100644
index 00000000000..3cf707f42aa
--- /dev/null
+++ b/libglusterfs/src/defaults-tmpl.c
@@ -0,0 +1,247 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* libglusterfs/src/defaults.c:
+   This file contains functions, which are used to fill the 'fops', 'cbk'
+   structures in the xlator structures, if they are not written. Here, all the
+   function calls are plainly forwarded to the first child of the xlator, and
+   all the *_cbk function does plain STACK_UNWIND of the frame, and returns.
+
+   This function also implements *_resume () functions, which does same
+   operation as a fop().
+
+   All the functions are plain enough to understand.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/defaults.h"
+
+#pragma generate
+
+struct xlator_fops _default_fops = {
+    .create = default_create,
+    .open = default_open,
+    .stat = default_stat,
+    .readlink = default_readlink,
+    .mknod = default_mknod,
+    .mkdir = default_mkdir,
+    .unlink = default_unlink,
+    .rmdir = default_rmdir,
+    .symlink = default_symlink,
+    .rename = default_rename,
+    .link = default_link,
+    .truncate = default_truncate,
+    .readv = default_readv,
+    .writev = default_writev,
+    .statfs = default_statfs,
+    .flush = default_flush,
+    .fsync = default_fsync,
+    .setxattr = default_setxattr,
+    .getxattr = default_getxattr,
+    .fsetxattr = default_fsetxattr,
+    .fgetxattr = default_fgetxattr,
+    .removexattr = default_removexattr,
+    .fremovexattr = default_fremovexattr,
+    .opendir = default_opendir,
+    .readdir = default_readdir,
+    .readdirp = default_readdirp,
+    .fsyncdir = default_fsyncdir,
+    .access = default_access,
+    .ftruncate = default_ftruncate,
+    .fstat = default_fstat,
+    .lk = default_lk,
+    .inodelk = default_inodelk,
+    .finodelk = default_finodelk,
+    .entrylk = default_entrylk,
+    .fentrylk = default_fentrylk,
+    .lookup = default_lookup,
+    .rchecksum = default_rchecksum,
+    .xattrop = default_xattrop,
+    .fxattrop = default_fxattrop,
+    .setattr = default_setattr,
+    .fsetattr = default_fsetattr,
+    .fallocate = default_fallocate,
+    .discard = default_discard,
+    .zerofill = default_zerofill,
+    .ipc = default_ipc,
+    .seek = default_seek,
+
+    .getspec = default_getspec,
+    .getactivelk = default_getactivelk,
+    .setactivelk = default_setactivelk,
+    .put = default_put,
+    .icreate = default_icreate,
+    .namelink = default_namelink,
+    .copy_file_range = default_copy_file_range,
+};
+struct xlator_fops *default_fops = &_default_fops;
+
+/*
+ * Remaining functions don't follow the fop calling conventions, so they're
+ * not generated.
+ */
+
+int32_t
+default_forget(xlator_t *this, inode_t *inode)
+{
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement forget_cbk");
+    return 0;
+}
+
+int32_t
+default_releasedir(xlator_t *this, fd_t *fd)
+{
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement releasedir_cbk");
+    return 0;
+}
+
+int32_t
+default_release(xlator_t *this, fd_t *fd)
+{
+    gf_log_callingfn(this->name, GF_LOG_DEBUG,
+                     "xlator does not "
+                     "implement release_cbk");
+    return 0;
+}
+
+/* notify */
+int
+default_notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    GF_UNUSED int ret = 0;
+    xlator_t *victim = data;
+
+    glusterfs_graph_t *graph = NULL;
+
+    GF_VALIDATE_OR_GOTO("notify", this, out);
+    graph = this->graph;
+    GF_VALIDATE_OR_GOTO(this->name, graph, out);
+
+    switch (event) {
+        case GF_EVENT_PARENT_UP:
+        case GF_EVENT_PARENT_DOWN: {
+            xlator_list_t *list = this->children;
+
+            while (list) {
+                if (victim && victim->cleanup_starting)
+                    xlator_notify(list->xlator, event, victim);
+                else
+                    xlator_notify(list->xlator, event, this);
+                list = list->next;
+            }
+        } break;
+        case GF_EVENT_CHILD_CONNECTING:
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_CHILD_UP:
+        case GF_EVENT_AUTH_FAILED: {
+            xlator_list_t *parent = this->parents;
+
+            /*
+             * Handle case of CHILD_* & AUTH_FAILED event specially, send
+             * it to fuse.
+             */
+            if (!parent && this->ctx && this->ctx->master) {
+                xlator_notify(this->ctx->master, event, this->graph, NULL);
+            }
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, this, NULL);
+                parent = parent->next;
+            }
+
+            if (event == GF_EVENT_CHILD_DOWN &&
+                !(this->ctx && this->ctx->master) && (graph->top == this)) {
+                /* Make sure this is not a daemon with master xlator */
+                pthread_mutex_lock(&graph->mutex);
+                {
+                    if (graph->parent_down ==
+                        graph_total_client_xlator(graph)) {
+                        graph->used = 0;
+                        pthread_cond_broadcast(&graph->child_down_cond);
+                    }
+                }
+                pthread_mutex_unlock(&graph->mutex);
+            }
+        } break;
+        case GF_EVENT_UPCALL: {
+            xlator_list_t *parent = this->parents;
+
+            if (!parent && this->ctx && this->ctx->master)
+                xlator_notify(this->ctx->master, event, data, NULL);
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, data, NULL);
+                parent = parent->next;
+            }
+        } break;
+        case GF_EVENT_CHILD_PING: {
+            xlator_list_t *parent = this->parents;
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    XLATOR_NOTIFY(ret, parent->xlator, event, this, data);
+                parent = parent->next;
+            }
+        } break;
+        case GF_EVENT_CLEANUP: {
+            xlator_list_t *list = this->children;
+
+            while (list) {
+                xlator_notify(list->xlator, event, this);
+                list = list->next;
+            }
+        } break;
+
+        default: {
+            xlator_list_t *parent = this->parents;
+
+            while (parent) {
+                if (parent->xlator->init_succeeded)
+                    xlator_notify(parent->xlator, event, this, NULL);
+                parent = parent->next;
+            }
+        }
+            /*
+             * Apparently our picky-about-everything else coding standard allows
+             * adjacent same-indendation-level close braces.  Clearly it has
+             * nothing to do with readability.
+             */
+    }
+out:
+    return 0;
+}
+
+int32_t
+default_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_common_mt_end);
+
+    return ret;
+}
+
+void
+default_fini(xlator_t *this)
+{
+    if (this && this->private)
+        GF_FREE(this->private);
+}
diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c
deleted file mode 100644
index 718afeb9ac2..00000000000
--- a/libglusterfs/src/defaults.c
+++ /dev/null
@@ -1,1260 +0,0 @@
-/*
-  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/* libglusterfs/src/defaults.c:
-   This file contains functions, which are used to fill the 'fops', 'cbk'
-   structures in the xlator structures, if they are not written. Here, all the
-   function calls are plainly forwared to the first child of the xlator, and
-   all the *_cbk function does plain STACK_UNWIND of the frame, and returns.
-
-   This function also implements *_resume () functions, which does same
-   operation as a fop().
-
-   All the functions are plain enough to understand.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-
-/* _CBK function section */
-
-int32_t
-default_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
-        return 0;
-}
-
-int32_t
-default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-
-int32_t
-default_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-        return 0;
-}
-
-int32_t
-default_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
-{
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-        return 0;
-}
-
-int32_t
-default_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
-        return 0;
-}
-
-int32_t
-default_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
-{
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf);
-        return 0;
-}
-
-
-int32_t
-default_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
-        return 0;
-}
-
-int32_t
-default_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
-        return 0;
-}
-
-int32_t
-default_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
-        return 0;
-}
-
-int32_t
-default_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
-        return 0;
-}
-
-
-int32_t
-default_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
-
-
-int32_t
-default_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
-        return 0;
-}
-
-
-int32_t
-default_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
-
-
-int32_t
-default_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
-
-int32_t
-default_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-        return 0;
-}
-
-int32_t
-default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
-{
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
-        return 0;
-}
-
-
-int32_t
-default_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
-{
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
-}
-
-
-int32_t
-default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-
-int32_t
-default_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
-}
-
-int32_t
-default_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-int32_t
-default_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
-        return 0;
-}
-
-int32_t
-default_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
-        return 0;
-}
-
-int32_t
-default_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-
-int32_t
-default_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-
-int32_t
-default_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-default_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-int32_t
-default_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-int32_t
-default_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-default_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-int32_t
-default_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
-        return 0;
-}
-
-int32_t
-default_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-default_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-        return 0;
-}
-
-int32_t
-default_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-        return 0;
-}
-
-int32_t
-default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-default_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum)
-{
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
-        return 0;
-}
-
-
-int32_t
-default_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
-        return 0;
-}
-
-
-int32_t
-default_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
-        return 0;
-}
-
-int32_t
-default_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
-{
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
-        return 0;
-}
-
-int32_t
-default_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost)
-{
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
-        return 0;
-}
-
-int32_t
-default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, char *spec_data)
-{
-        STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
-        return 0;
-}
-
-/* RESUME */
-
-int32_t
-default_fgetxattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          const char *name)
-{
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
-        return 0;
-}
-
-int32_t
-default_fsetxattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          dict_t *dict, int32_t flags)
-{
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
-        return 0;
-}
-
-int32_t
-default_setxattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         dict_t *dict, int32_t flags)
-{
-        STACK_WIND (frame, default_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
-        return 0;
-}
-
-int32_t
-default_statfs_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc);
-        return 0;
-}
-
-int32_t
-default_fsyncdir_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         int32_t flags)
-{
-        STACK_WIND (frame, default_fsyncdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir, fd, flags);
-        return 0;
-}
-
-int32_t
-default_opendir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        fd_t *fd)
-{
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
-        return 0;
-}
-
-int32_t
-default_fstat_resume (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
-        return 0;
-}
-
-int32_t
-default_fsync_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      int32_t flags)
-{
-        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
-        return 0;
-}
-
-int32_t
-default_flush_resume (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
-        return 0;
-}
-
-int32_t
-default_writev_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       struct iovec *vector, int32_t count, off_t off,
-                       struct iobref *iobref)
-{
-        STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    iobref);
-        return 0;
-}
-
-int32_t
-default_readv_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      size_t size, off_t offset)
-{
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset);
-        return 0;
-}
-
-
-int32_t
-default_open_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, fd_t *fd, int32_t wbflags)
-{
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
-}
-
-int32_t
-default_create_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
-{
-        STACK_WIND (frame, default_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
-        return 0;
-}
-
-int32_t
-default_link_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc)
-{
-        STACK_WIND (frame, default_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
-        return 0;
-}
-
-int32_t
-default_rename_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                       loc_t *newloc)
-{
-        STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
-        return 0;
-}
-
-
-int
-default_symlink_resume (call_frame_t *frame, xlator_t *this,
-                        const char *linkpath, loc_t *loc, dict_t *params)
-{
-        STACK_WIND (frame, default_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
-        return 0;
-}
-
-int32_t
-default_rmdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      int flags)
-{
-        STACK_WIND (frame, default_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
-        return 0;
-}
-
-int32_t
-default_unlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
-        return 0;
-}
-
-int
-default_mkdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      mode_t mode, dict_t *params)
-{
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
-        return 0;
-}
-
-
-int
-default_mknod_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      mode_t mode, dev_t rdev, dict_t *parms)
-{
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
-        return 0;
-}
-
-int32_t
-default_readlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         size_t size)
-{
-        STACK_WIND (frame, default_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
-        return 0;
-}
-
-
-int32_t
-default_access_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       int32_t mask)
-{
-        STACK_WIND (frame, default_access_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
-        return 0;
-}
-
-int32_t
-default_ftruncate_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset)
-{
-        STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
-}
-
-int32_t
-default_getxattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name)
-{
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
-        return 0;
-}
-
-
-int32_t
-default_xattrop_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        gf_xattrop_flags_t flags, dict_t *dict)
-{
-        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
-        return 0;
-}
-
-int32_t
-default_fxattrop_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         gf_xattrop_flags_t flags, dict_t *dict)
-{
-        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
-        return 0;
-}
-
-int32_t
-default_removexattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            const char *name)
-{
-        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
-        return 0;
-}
-
-int32_t
-default_lk_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   int32_t cmd, struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-        return 0;
-}
-
-
-int32_t
-default_inodelk_resume (call_frame_t *frame, xlator_t *this,
-                        const char *volume, loc_t *loc, int32_t cmd,
-                        struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_inodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    volume, loc, cmd, lock);
-        return 0;
-}
-
-int32_t
-default_finodelk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, fd_t *fd, int32_t cmd,
-                         struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_finodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->finodelk,
-                    volume, fd, cmd, lock);
-        return 0;
-}
-
-int32_t
-default_entrylk_resume (call_frame_t *frame, xlator_t *this,
-                        const char *volume, loc_t *loc, const char *basename,
-                        entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, default_entrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
-        return 0;
-}
-
-int32_t
-default_fentrylk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, fd_t *fd, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, default_fentrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
-        return 0;
-}
-
-int32_t
-default_rchecksum_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset, int32_t len)
-{
-        STACK_WIND (frame, default_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
-        return 0;
-}
-
-
-int32_t
-default_readdir_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        size_t size, off_t off)
-{
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off);
-        return 0;
-}
-
-
-int32_t
-default_readdirp_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         size_t size, off_t off)
-{
-        STACK_WIND (frame, default_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size, off);
-        return 0;
-}
-
-int32_t
-default_setattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
-        return 0;
-}
-
-int32_t
-default_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         off_t offset)
-{
-        STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-        return 0;
-}
-
-int32_t
-default_stat_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-        return 0;
-}
-
-int32_t
-default_lookup_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xattr_req)
-{
-        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        return 0;
-}
-
-int32_t
-default_fsetattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
-        return 0;
-}
-
-/* FOPS */
-
-int32_t
-default_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name)
-{
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
-        return 0;
-}
-
-int32_t
-default_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags)
-{
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
-        return 0;
-}
-
-int32_t
-default_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags)
-{
-        STACK_WIND (frame, default_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
-        return 0;
-}
-
-int32_t
-default_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc);
-        return 0;
-}
-
-int32_t
-default_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        STACK_WIND (frame, default_fsyncdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir, fd, flags);
-        return 0;
-}
-
-int32_t
-default_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
-        return 0;
-}
-
-int32_t
-default_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
-        return 0;
-}
-
-int32_t
-default_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
-        return 0;
-}
-
-int32_t
-default_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
-        return 0;
-}
-
-int32_t
-default_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off,
-                struct iobref *iobref)
-{
-        STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    iobref);
-        return 0;
-}
-
-int32_t
-default_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset)
-{
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset);
-        return 0;
-}
-
-
-int32_t
-default_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              fd_t *fd, int32_t wbflags)
-{
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
-}
-
-int32_t
-default_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, fd_t *fd, dict_t *params)
-{
-        STACK_WIND (frame, default_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
-        return 0;
-}
-
-int32_t
-default_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        STACK_WIND (frame, default_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
-        return 0;
-}
-
-int32_t
-default_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc)
-{
-        STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
-        return 0;
-}
-
-
-int
-default_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                 loc_t *loc, dict_t *params)
-{
-        STACK_WIND (frame, default_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
-        return 0;
-}
-
-int32_t
-default_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        STACK_WIND (frame, default_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
-        return 0;
-}
-
-int32_t
-default_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
-        return 0;
-}
-
-int
-default_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dict_t *params)
-{
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
-        return 0;
-}
-
-
-int
-default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dev_t rdev, dict_t *parms)
-{
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
-        return 0;
-}
-
-int32_t
-default_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
-{
-        STACK_WIND (frame, default_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
-        return 0;
-}
-
-
-int32_t
-default_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        STACK_WIND (frame, default_access_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
-        return 0;
-}
-
-int32_t
-default_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
-{
-        STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
-}
-
-int32_t
-default_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name)
-{
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
-        return 0;
-}
-
-
-int32_t
-default_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 gf_xattrop_flags_t flags, dict_t *dict)
-{
-        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
-        return 0;
-}
-
-int32_t
-default_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  gf_xattrop_flags_t flags, dict_t *dict)
-{
-        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
-        return 0;
-}
-
-int32_t
-default_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name)
-{
-        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
-        return 0;
-}
-
-int32_t
-default_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            int32_t cmd, struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-        return 0;
-}
-
-
-int32_t
-default_inodelk (call_frame_t *frame, xlator_t *this,
-                 const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_inodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    volume, loc, cmd, lock);
-        return 0;
-}
-
-int32_t
-default_finodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
-{
-        STACK_WIND (frame, default_finodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->finodelk,
-                    volume, fd, cmd, lock);
-        return 0;
-}
-
-int32_t
-default_entrylk (call_frame_t *frame, xlator_t *this,
-                 const char *volume, loc_t *loc, const char *basename,
-                 entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, default_entrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
-        return 0;
-}
-
-int32_t
-default_fentrylk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, fd_t *fd, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, default_fentrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
-        return 0;
-}
-
-int32_t
-default_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   int32_t len)
-{
-        STACK_WIND (frame, default_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
-        return 0;
-}
-
-
-int32_t
-default_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 size_t size, off_t off)
-{
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off);
-        return 0;
-}
-
-
-int32_t
-default_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  size_t size, off_t off)
-{
-        STACK_WIND (frame, default_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size, off);
-        return 0;
-}
-
-int32_t
-default_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
-        return 0;
-}
-
-int32_t
-default_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-        return 0;
-}
-
-int32_t
-default_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-        return 0;
-}
-
-int32_t
-default_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *xattr_req)
-{
-        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        return 0;
-}
-
-int32_t
-default_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
-        return 0;
-}
-
-
-int32_t
-default_forget (xlator_t *this, inode_t *inode)
-{
-        return 0;
-}
-
-
-int32_t
-default_releasedir (xlator_t *this, fd_t *fd)
-{
-        return 0;
-}
-
-int32_t
-default_release (xlator_t *this, fd_t *fd)
-{
-        return 0;
-}
-
-/* End of FOP/_CBK/_RESUME section */
-
-
-/* Management operations */
-
-int32_t
-default_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-                 int32_t flags)
-{
-        STACK_WIND (frame, default_getspec_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getspec, key, flags);
-        return 0;
-}
-
-/* notify */
-int
-default_notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        switch (event)
-        {
-        case GF_EVENT_PARENT_UP:
-        {
-                xlator_list_t *list = this->children;
-
-                while (list) {
-                        xlator_notify (list->xlator, event, this);
-                        list = list->next;
-                }
-        }
-        break;
-        case GF_EVENT_CHILD_CONNECTING:
-        case GF_EVENT_CHILD_UP:
-        {
-                xlator_list_t *parent = this->parents;
-                /* Handle the case of CHILD_UP specially, send it to fuse */
-                if (!parent && this->ctx && this->ctx->master)
-                        xlator_notify (this->ctx->master, event, this->graph, NULL);
-
-                while (parent) {
-                        if (parent->xlator->init_succeeded)
-                                xlator_notify (parent->xlator, event,
-                                               this, NULL);
-                        parent = parent->next;
-                }
-        }
-        break;
-        case GF_EVENT_CHILD_DOWN:
-        default:
-        {
-                xlator_list_t *parent = this->parents;
-                while (parent) {
-                        if (parent->xlator->init_succeeded)
-                                xlator_notify (parent->xlator, event,
-                                               this, NULL);
-                        parent = parent->next;
-                }
-        }
-        }
-
-        return 0;
-}
-
-int32_t
-default_mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        ret = xlator_mem_acct_init (this, gf_common_mt_end);
-
-        return ret;
-}
diff --git a/libglusterfs/src/defaults.h b/libglusterfs/src/defaults.h
deleted file mode 100644
index 6e451602b68..00000000000
--- a/libglusterfs/src/defaults.h
+++ /dev/null
@@ -1,679 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-/* libglusterfs/src/defaults.h:
-       This file contains definition of default fops and mops functions.
-*/
-
-#ifndef _DEFAULTS_H
-#define _DEFAULTS_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-
-/* Management Operations */
-
-int32_t default_getspec (call_frame_t *frame,
-                         xlator_t *this,
-                         const char *key,
-                         int32_t flag);
-
-int32_t default_rchecksum (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd, off_t offset,
-                           int32_t len);
-
-/* FileSystem operations */
-int32_t default_lookup (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        dict_t *xattr_req);
-
-int32_t default_stat (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc);
-
-int32_t default_fstat (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd);
-
-int32_t default_truncate (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          off_t offset);
-
-int32_t default_ftruncate (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           off_t offset);
-
-int32_t default_access (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        int32_t mask);
-
-int32_t default_readlink (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          size_t size);
-
-int32_t default_mknod (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
-
-int32_t default_mkdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dict_t *params);
-
-int32_t default_unlink (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc);
-
-int32_t default_rmdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, int flags);
-
-int32_t default_symlink (call_frame_t *frame, xlator_t *this,
-                         const char *linkpath, loc_t *loc, dict_t *params);
-
-int32_t default_rename (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *oldloc,
-                        loc_t *newloc);
-
-int32_t default_link (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *oldloc,
-                      loc_t *newloc);
-
-int32_t default_create (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int32_t flags, mode_t mode,
-                        fd_t *fd, dict_t *params);
-
-int32_t default_open (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc,
-                      int32_t flags, fd_t *fd,
-                      int32_t wbflags);
-
-int32_t default_readv (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       size_t size,
-                       off_t offset);
-
-int32_t default_writev (call_frame_t *frame,
-                        xlator_t *this,
-                        fd_t *fd,
-                        struct iovec *vector,
-                        int32_t count,
-                        off_t offset,
-                        struct iobref *iobref);
-
-int32_t default_flush (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd);
-
-int32_t default_fsync (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       int32_t datasync);
-
-int32_t default_opendir (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc, fd_t *fd);
-
-int32_t default_fsyncdir (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          int32_t datasync);
-
-int32_t default_statfs (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc);
-
-int32_t default_setxattr (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          dict_t *dict,
-                          int32_t flags);
-
-int32_t default_getxattr (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          const char *name);
-
-int32_t default_fsetxattr (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           dict_t *dict,
-                           int32_t flags);
-
-int32_t default_fgetxattr (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           const char *name);
-
-int32_t default_removexattr (call_frame_t *frame,
-                             xlator_t *this,
-                             loc_t *loc,
-                             const char *name);
-
-int32_t default_lk (call_frame_t *frame,
-                    xlator_t *this,
-                    fd_t *fd,
-                    int32_t cmd,
-                    struct gf_flock *flock);
-
-int32_t default_inodelk (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock);
-
-int32_t default_finodelk (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock);
-
-int32_t default_entrylk (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type);
-
-int32_t default_fentrylk (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
-
-int32_t default_readdir (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off);
-
-int32_t default_readdirp (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off);
-
-int32_t default_xattrop (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         gf_xattrop_flags_t flags,
-                         dict_t *dict);
-
-int32_t default_fxattrop (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          gf_xattrop_flags_t flags,
-                          dict_t *dict);
-
-int32_t default_notify (xlator_t *this,
-                        int32_t event,
-                        void *data,
-                        ...);
-
-int32_t default_forget (xlator_t *this,
-                        inode_t *inode);
-
-int32_t default_release (xlator_t *this,
-                         fd_t *fd);
-
-int32_t default_releasedir (xlator_t *this,
-                            fd_t *fd);
-
-int32_t default_setattr (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         struct iatt *stbuf,
-                         int32_t valid);
-
-int32_t default_fsetattr (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          struct iatt *stbuf,
-                          int32_t valid);
-
-/* Resume */
-int32_t default_getspec (call_frame_t *frame,
-                         xlator_t *this,
-                         const char *key,
-                         int32_t flag);
-
-int32_t default_rchecksum (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd, off_t offset,
-                           int32_t len);
-
-/* FileSystem operations */
-int32_t default_lookup_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        dict_t *xattr_req);
-
-int32_t default_stat_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc);
-
-int32_t default_fstat_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd);
-
-int32_t default_truncate_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          off_t offset);
-
-int32_t default_ftruncate_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           off_t offset);
-
-int32_t default_access_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        int32_t mask);
-
-int32_t default_readlink_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          size_t size);
-
-int32_t default_mknod_resume (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
-
-int32_t default_mkdir_resume (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dict_t *params);
-
-int32_t default_unlink_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc);
-
-int32_t default_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                              loc_t *loc, int flags);
-
-int32_t default_symlink_resume (call_frame_t *frame, xlator_t *this,
-                         const char *linkpath, loc_t *loc, dict_t *params);
-
-int32_t default_rename_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *oldloc,
-                        loc_t *newloc);
-
-int32_t default_link_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *oldloc,
-                      loc_t *newloc);
-
-int32_t default_create_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int32_t flags, mode_t mode,
-                        fd_t *fd, dict_t *params);
-
-int32_t default_open_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc,
-                      int32_t flags, fd_t *fd,
-                      int32_t wbflags);
-
-int32_t default_readv_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       size_t size,
-                       off_t offset);
-
-int32_t default_writev_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        fd_t *fd,
-                        struct iovec *vector,
-                        int32_t count,
-                        off_t offset,
-                        struct iobref *iobref);
-
-int32_t default_flush_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd);
-
-int32_t default_fsync_resume (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t *fd,
-                       int32_t datasync);
-
-int32_t default_opendir_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc, fd_t *fd);
-
-int32_t default_fsyncdir_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          int32_t datasync);
-
-int32_t default_statfs_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc);
-
-int32_t default_setxattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          dict_t *dict,
-                          int32_t flags);
-
-int32_t default_getxattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          loc_t *loc,
-                          const char *name);
-
-int32_t default_fsetxattr_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           dict_t *dict,
-                           int32_t flags);
-
-int32_t default_fgetxattr_resume (call_frame_t *frame,
-                           xlator_t *this,
-                           fd_t *fd,
-                           const char *name);
-
-int32_t default_removexattr_resume (call_frame_t *frame,
-                             xlator_t *this,
-                             loc_t *loc,
-                             const char *name);
-
-int32_t default_lk_resume (call_frame_t *frame,
-                    xlator_t *this,
-                    fd_t *fd,
-                    int32_t cmd,
-                    struct gf_flock *flock);
-
-int32_t default_inodelk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock);
-
-int32_t default_finodelk_resume (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock);
-
-int32_t default_entrylk_resume (call_frame_t *frame, xlator_t *this,
-                         const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type);
-
-int32_t default_fentrylk_resume (call_frame_t *frame, xlator_t *this,
-                          const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
-
-int32_t default_readdir_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off);
-
-int32_t default_readdirp_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          size_t size, off_t off);
-
-int32_t default_xattrop_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         gf_xattrop_flags_t flags,
-                         dict_t *dict);
-
-int32_t default_fxattrop_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          gf_xattrop_flags_t flags,
-                          dict_t *dict);
-int32_t default_rchecksum_resume (call_frame_t *frame,
-                                  xlator_t *this,
-                                  fd_t *fd, off_t offset,
-                                  int32_t len);
-
-int32_t default_notify (xlator_t *this,
-                        int32_t event,
-                        void *data,
-                        ...);
-
-int32_t default_forget (xlator_t *this,
-                        inode_t *inode);
-
-int32_t default_release (xlator_t *this,
-                         fd_t *fd);
-
-int32_t default_releasedir (xlator_t *this,
-                            fd_t *fd);
-
-int32_t default_setattr_resume (call_frame_t *frame,
-                         xlator_t *this,
-                         loc_t *loc,
-                         struct iatt *stbuf,
-                         int32_t valid);
-
-int32_t default_fsetattr_resume (call_frame_t *frame,
-                          xlator_t *this,
-                          fd_t *fd,
-                          struct iatt *stbuf,
-                          int32_t valid);
-
-/* _cbk */
-
-int32_t
-default_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent);
-
-int32_t
-default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf);
-
-
-int32_t
-default_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf);
-
-int32_t
-default_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf);
-
-int32_t
-default_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno);
-
-int32_t
-default_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf);
-
-
-int32_t
-default_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent);
-
-int32_t
-default_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent);
-
-int32_t
-default_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent);
-
-int32_t
-default_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent);
-
-
-int32_t
-default_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent);
-
-
-int32_t
-default_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent);
-
-
-int32_t
-default_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent);
-
-
-int32_t
-default_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent);
-
-int32_t
-default_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd);
-
-int32_t
-default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref);
-
-
-int32_t
-default_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf);
-
-
-int32_t
-default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno);
-
-
-
-int32_t
-default_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf);
-
-int32_t
-default_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf);
-
-int32_t
-default_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd);
-
-int32_t
-default_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
-
-int32_t
-default_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf);
-
-
-int32_t
-default_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
-
-
-int32_t
-default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno);
-
-
-
-int32_t
-default_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict);
-
-
-int32_t
-default_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict);
-
-int32_t
-default_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict);
-
-int32_t
-default_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict);
-
-
-int32_t
-default_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno);
-
-int32_t
-default_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
-
-int32_t
-default_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno);
-
-
-int32_t
-default_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
-
-int32_t
-default_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno);
-
-int32_t
-default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
-
-
-int32_t
-default_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum);
-
-
-int32_t
-default_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries);
-
-
-int32_t
-default_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries);
-
-int32_t
-default_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost);
-
-int32_t
-default_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost);
-
-int32_t
-default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, char *spec_data);
-
-int32_t
-default_mem_acct_init (xlator_t *this);
-#endif /* _DEFAULTS_H */
diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index b5da61144d3..1d9be9217a6 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
@@ -22,1282 +13,1475 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <inttypes.h>
+#include <limits.h>
+#include <fnmatch.h>
+
+#include "glusterfs/dict.h"
+#define XXH_INLINE_ALL
+#include "xxhash.h"
+#include "glusterfs/compat.h"
+#include "glusterfs/compat-errno.h"
+#include "glusterfs/byte-order.h"
+#include "glusterfs/statedump.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+struct dict_cmp {
+    dict_t *dict;
+    gf_boolean_t (*value_ignore)(char *k);
+};
+
+#define VALIDATE_DATA_AND_LOG(data, type, key, ret_val)                        \
+    do {                                                                       \
+        if (!data || !data->data) {                                            \
+            gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG, \
+                             "data is NULL");                                  \
+            return ret_val;                                                    \
+        }                                                                      \
+        /* Not of the asked type, or old version */                            \
+        if ((data->data_type != type) &&                                       \
+            (data->data_type != GF_DATA_TYPE_STR_OLD)) {                       \
+            gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG, \
+                             "key %s, %s type asked, has %s type", key,        \
+                             data_type_name[type],                             \
+                             data_type_name[data->data_type]);                 \
+        }                                                                      \
+    } while (0)
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "common-utils.h"
-#include "dict.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "compat.h"
-#include "byte-order.h"
-
-data_pair_t *
-get_new_data_pair ()
-{
-	data_pair_t *data_pair_ptr = NULL;
+static data_t *
+get_new_data()
+{
+    data_t *data = mem_get(THIS->ctx->dict_data_pool);
+
+    if (!data)
+        return NULL;
+
+    GF_ATOMIC_INIT(data->refcount, 0);
+    data->is_static = _gf_false;
+
+    return data;
+}
+
+static dict_t *
+get_new_dict_full(int size_hint)
+{
+    dict_t *dict = mem_get0(THIS->ctx->dict_pool);
+
+    if (!dict) {
+        return NULL;
+    }
+
+    dict->hash_size = size_hint;
+    if (size_hint == 1) {
+        /*
+         * This is the only case we ever see currently.  If we ever
+         * need to support resizing the hash table, the resize function
+         * will have to take into account the possibility that
+         * "members" is not separately allocated (i.e. don't just call
+         * realloc() blindly.
+         */
+        dict->members = &dict->members_internal;
+    } else {
+        /*
+         * We actually need to allocate space for size_hint *pointers*
+         * but we actually allocate space for one *structure*.  Since
+         * a data_pair_t consists of five pointers, we're wasting four
+         * pointers' worth for N=1, and will overrun what we allocated
+         * for N>5.  If anybody ever starts using size_hint, we'll need
+         * to fix this.
+         */
+        GF_ASSERT(size_hint <= (sizeof(data_pair_t) / sizeof(data_pair_t *)));
+        dict->members = mem_get0(THIS->ctx->dict_pair_pool);
+        if (!dict->members) {
+            mem_put(dict);
+            return NULL;
+        }
+    }
 
-	data_pair_ptr = (data_pair_t *) GF_CALLOC (1, sizeof (data_pair_t),
-                                                   gf_common_mt_data_pair_t);
-        if (!data_pair_ptr)
-                gf_log ("dict", GF_LOG_ERROR, "memory alloc failed");
+    dict->free_pair.key = NULL;
+    dict->totkvlen = 0;
+    LOCK_INIT(&dict->lock);
 
-	return data_pair_ptr;
+    return dict;
 }
 
-data_t *
-get_new_data ()
+dict_t *
+dict_new(void)
 {
-	data_t *data = NULL;
+    dict_t *dict = get_new_dict_full(1);
 
-	data = (data_t *) GF_CALLOC (1, sizeof (data_t), gf_common_mt_data_t);
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"calloc () returned NULL");
-		return NULL;
-	}
+    if (dict)
+        dict_ref(dict);
 
-	LOCK_INIT (&data->lock);
-	return data;
+    return dict;
 }
 
-dict_t *
-get_new_dict_full (int size_hint)
+int32_t
+is_data_equal(data_t *one, data_t *two)
 {
-	dict_t *dict = GF_CALLOC (1, sizeof (dict_t), gf_common_mt_dict_t);
-
-	if (!dict) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"calloc () returned NULL");
-		return NULL;
-	}
-
-	dict->hash_size = size_hint;
-	dict->members = GF_CALLOC (size_hint, sizeof (data_pair_t *),
-                                   gf_common_mt_data_pair_t);
+    struct iatt *iatt1, *iatt2;
+    struct mdata_iatt *mdata_iatt1, *mdata_iatt2;
 
-	if (!dict->members) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"calloc () returned NULL");
-		return NULL;
-	}
+    if (!one || !two || !one->data || !two->data) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "input arguments are provided "
+                         "with value data_t as NULL");
+        return -1;
+    }
 
-	LOCK_INIT (&dict->lock);
+    if (one == two)
+        return 1;
 
-	return dict;
-}
+    if (one->data == two->data)
+        return 1;
 
-dict_t *
-get_new_dict (void)
-{
-	return get_new_dict_full (1);
-}
+    if (one->data_type != two->data_type) {
+        return 0;
+    }
 
-dict_t *
-dict_new (void)
-{
-	dict_t *dict = NULL;
-	
-	dict = get_new_dict_full(1);
-	
-	if (dict)
-		dict_ref (dict);
-	
-	return dict;
-}
+    if (one->data_type == GF_DATA_TYPE_IATT) {
+        if ((one->len < sizeof(struct iatt)) ||
+            (two->len < sizeof(struct iatt))) {
+            return 0;
+        }
 
+        iatt1 = (struct iatt *)one->data;
+        iatt2 = (struct iatt *)two->data;
 
-int32_t 
-is_data_equal (data_t *one,
-	       data_t *two)
-{
-	if (!one || !two || !one->data || !two->data)
-		return 1;
+        /* Two iatt structs are considered equal if main fields are
+         * equal, even if times differ.
+         * TODO: maybe when ctime if fully operational we could
+         *       enforce time matching. */
+        if (iatt1->ia_ino != iatt2->ia_ino) {
+            return 0;
+        }
+        if (iatt1->ia_type != iatt2->ia_type) {
+            return 0;
+        }
+        if ((iatt1->ia_type == IA_IFBLK) || (iatt1->ia_type == IA_IFCHR)) {
+            if (iatt1->ia_rdev != iatt2->ia_rdev) {
+                return 0;
+            }
+        }
+        if (gf_uuid_compare(iatt1->ia_gfid, iatt2->ia_gfid) != 0) {
+            return 0;
+        }
 
-	if (one == two)
-		return 1;
+        /* TODO: ia_uid, ia_gid, ia_prot and ia_size can be changed
+         *       with some commands. Here we don't have enough
+         *       information to decide if they should match or not. */
+        /*
+                        if ((iatt1->ia_uid != iatt2->ia_uid) ||
+                            (iatt1->ia_gid != iatt2->ia_gid) ||
+                            (st_mode_from_ia(iatt1->ia_prot, iatt1->ia_type) !=
+                                    st_mode_from_ia(iatt2->ia_prot,
+           iatt2->ia_type))) { return 0;
+                        }
+                        if (iatt1->ia_type == IA_IFREG) {
+                                if (iatt1->ia_size != iatt2->ia_size) {
+                                        return 0;
+                                }
+                        }
+        */
+        return 1;
+    }
+    if (one->data_type == GF_DATA_TYPE_MDATA) {
+        if ((one->len < sizeof(struct mdata_iatt)) ||
+            (two->len < sizeof(struct mdata_iatt))) {
+            return 0;
+        }
+        mdata_iatt1 = (struct mdata_iatt *)one->data;
+        mdata_iatt2 = (struct mdata_iatt *)two->data;
+
+        if (mdata_iatt1->ia_atime != mdata_iatt2->ia_atime ||
+            mdata_iatt1->ia_mtime != mdata_iatt2->ia_mtime ||
+            mdata_iatt1->ia_ctime != mdata_iatt2->ia_ctime ||
+            mdata_iatt1->ia_atime_nsec != mdata_iatt2->ia_atime_nsec ||
+            mdata_iatt1->ia_mtime_nsec != mdata_iatt2->ia_mtime_nsec ||
+            mdata_iatt1->ia_ctime_nsec != mdata_iatt2->ia_ctime_nsec) {
+            return 0;
+        }
+        return 1;
+    }
 
-	if (one->len != two->len)
-		return 0;
+    if (one->len != two->len)
+        return 0;
 
-	if (one->data == two->data)
-		return 1;
+    if (memcmp(one->data, two->data, one->len) == 0)
+        return 1;
 
-	if (memcmp (one->data, two->data, one->len) == 0)
-		return 1;
+    return 0;
+}
 
-	return 0;
+static int
+key_value_cmp(dict_t *one, char *key1, data_t *value1, void *data)
+{
+    struct dict_cmp *cmp = data;
+    dict_t *two = cmp->dict;
+    data_t *value2 = dict_get(two, key1);
+
+    if (value2) {
+        if (cmp->value_ignore && cmp->value_ignore(key1))
+            return 0;
+
+        if (is_data_equal(value1, value2) == 1)
+            return 0;
+    }
+
+    if (value2 == NULL) {
+        gf_msg_debug(THIS->name, 0, "'%s' found only on one dict", key1);
+    } else {
+        gf_msg_debug(THIS->name, 0,
+                     "'%s' is different in two dicts "
+                     "(%u, %u)",
+                     key1, value1->len, value2->len);
+    }
+
+    return -1;
+}
+
+/* If both dicts are NULL then equal. If one of the dicts is NULL but the
+ * other has only ignorable keys then also they are equal. If both dicts are
+ * non-null then check if for each non-ignorable key, values are same or
+ * not.  value_ignore function is used to skip comparing values for the keys
+ * which must be present in both the dictionaries but the value could be
+ * different.
+ */
+gf_boolean_t
+are_dicts_equal(dict_t *one, dict_t *two,
+                gf_boolean_t (*match)(dict_t *d, char *k, data_t *v,
+                                      void *data),
+                gf_boolean_t (*value_ignore)(char *k))
+{
+    int num_matches1 = 0;
+    int num_matches2 = 0;
+    struct dict_cmp cmp = {0};
+
+    if (one == two)
+        return _gf_true;
+
+    if (!match)
+        match = dict_match_everything;
+
+    if ((one == NULL) || (two == NULL)) {
+        num_matches1 = dict_foreach_match(one ? one : two, match, NULL,
+                                          dict_null_foreach_fn, NULL);
+        goto done;
+    }
+
+    cmp.dict = two;
+    cmp.value_ignore = value_ignore;
+    num_matches1 = dict_foreach_match(one, match, NULL, key_value_cmp, &cmp);
+
+    if (num_matches1 == -1)
+        return _gf_false;
+
+    if ((num_matches1 == one->count) && (one->count == two->count))
+        return _gf_true;
+
+    num_matches2 = dict_foreach_match(two, match, NULL, dict_null_foreach_fn,
+                                      NULL);
+done:
+    /* If the number of matches is same in 'two' then for all the
+     * valid-keys that exist in 'one' the value matched and no extra valid
+     * keys exist in 'two' alone. Otherwise there exists at least one extra
+     * valid-key in 'two' which doesn't exist in 'one' */
+    if (num_matches1 == num_matches2)
+        return _gf_true;
+    return _gf_false;
 }
 
 void
-data_destroy (data_t *data)
+data_destroy(data_t *data)
 {
-	if (data) {
-		LOCK_DESTROY (&data->lock);
+    if (data) {
+        if (!data->is_static)
+            GF_FREE(data->data);
 
-		if (!data->is_static) {
-			if (data->data) {
-                                if (data->is_stdalloc)
-                                        free (data->data);
-				else
-                                        GF_FREE (data->data);
-                        }
-			if (data->vec)
-				GF_FREE (data->vec);
-		}
-
-		data->len = 0xbabababa;
-		if (!data->is_const)
-			GF_FREE (data);
-	}
+        data->len = 0xbabababa;
+        mem_put(data);
+    }
 }
 
 data_t *
-data_copy (data_t *old)
-{
-	if (!old) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@old is NULL");
-		return NULL;
-	}
-
-	data_t *newdata = (data_t *) GF_CALLOC (1, sizeof (*newdata),
-                                                gf_common_mt_data_t);
-
-	if (!newdata) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@newdata - NULL returned by CALLOC");
-		return NULL;
-	}
-
-	if (old) {
-		newdata->len = old->len;
-		if (old->data) {
-			newdata->data = memdup (old->data, old->len);
-			if (!newdata->data)
-				goto err_out;
-		}
-		if (old->vec) {
-			newdata->vec = memdup (old->vec, old->len * (sizeof (void *) +
-								     sizeof (size_t)));
-			if (!newdata->vec)
-				goto err_out;
-		}
-	}
-
-        LOCK_INIT (&newdata->lock);
-	return newdata;
-
- err_out:
-
-	if (newdata->data)
-		FREE (newdata->data);
-	if (newdata->vec)
-		FREE (newdata->vec);
-	FREE (newdata);
-
-	gf_log ("dict", GF_LOG_CRITICAL,
-		"@newdata->data || @newdata->vec got NULL from CALLOC()");
-	return NULL;
+data_copy(data_t *old)
+{
+    if (!old) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, 0, LG_MSG_NULL_PTR,
+                         "old is NULL");
+        return NULL;
+    }
+
+    data_t *newdata = mem_get0(THIS->ctx->dict_data_pool);
+    if (!newdata) {
+        return NULL;
+    }
+
+    newdata->len = old->len;
+    if (old->data) {
+        newdata->data = gf_memdup(old->data, old->len);
+        if (!newdata->data)
+            goto err_out;
+    }
+    newdata->data_type = old->data_type;
+
+    return newdata;
+
+err_out:
+    mem_put(newdata);
+
+    return NULL;
 }
 
+/* Always need to be called under lock
+ * Always this and key variables are not null -
+ * checked by callers.
+ */
 static data_pair_t *
-_dict_lookup (dict_t *this, char *key)
+dict_lookup_common(const dict_t *this, const char *key, const uint32_t hash)
 {
-	if (!this || !key) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@this=%p @key=%p", this, key);
-		return NULL;
-	}
+    int hashval = 0;
+    data_pair_t *pair;
 
-	int hashval = SuperFastHash (key, strlen (key)) % this->hash_size;
-	data_pair_t *pair;
-   
-	for (pair = this->members[hashval]; pair != NULL; pair = pair->hash_next) {
-		if (pair->key && !strcmp (pair->key, key))
-			return pair;
-	}
-  
-	return NULL;
-}
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1)
+        hashval = hash % this->hash_size;
 
+    for (pair = this->members[hashval]; pair != NULL; pair = pair->hash_next) {
+        if (pair->key && (hash == pair->key_hash) && !strcmp(pair->key, key))
+            return pair;
+    }
 
-static int32_t
-_dict_set (dict_t *this, 
-	   char *key, 
-	   data_t *value)
-{
-	int hashval;
-	data_pair_t *pair;
-	char key_free = 0;
-	int tmp = 0;
-        int ret = 0;
-
-	if (!key) {
-		ret = gf_asprintf (&key, "ref:%p", value);
-                if (-1 == ret) {
-                        gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                        return -1;
-                }
-		key_free = 1;
-	}
-
-	tmp = SuperFastHash (key, strlen (key));
-	hashval = (tmp % this->hash_size);
-	pair = _dict_lookup (this, key);
-
-	if (pair) {
-		data_t *unref_data = pair->value;
-		pair->value = data_ref (value);
-		data_unref (unref_data);
-		if (key_free)
-			GF_FREE (key);
-		/* Indicates duplicate key */
-		return 0;
-	}
-	pair = (data_pair_t *) GF_CALLOC (1, sizeof (*pair),
-                                        gf_common_mt_data_pair_t);
-	if (!pair) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@pair - NULL returned by CALLOC");
-		return -1;
-	}
-
-	pair->key = (char *) GF_CALLOC (1, strlen (key) + 1,
-                                        gf_common_mt_char);
-	if (!pair->key) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@pair->key - NULL returned by CALLOC");
-		FREE (pair);
-
-                if (key_free)
-                        GF_FREE (key);
-		return -1;
-	}
-
-	strcpy (pair->key, key);
-	pair->value = data_ref (value);
- 
-	pair->hash_next = this->members[hashval];
-	this->members[hashval] = pair;
- 
-	pair->next = this->members_list;
-	pair->prev = NULL;
-	if (this->members_list)
-		this->members_list->prev = pair;
-	this->members_list = pair;
-	this->count++;
-  
-	if (key_free)
-		GF_FREE (key);
-	return 0;
+    return NULL;
 }
 
 int32_t
-dict_set (dict_t *this,
-	  char *key,
-	  data_t *value)
+dict_lookup(dict_t *this, char *key, data_t **data)
 {
-	int32_t ret;
+    if (!this || !key || !data) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !key || "
+                         "!data");
+        return -1;
+    }
 
-	if (!this || !value) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"@this=%p @value=%p, key=%s", this, value, key);
-		return -1;
-	}
+    data_pair_t *tmp = NULL;
 
-	LOCK (&this->lock);
+    uint32_t hash = (uint32_t)XXH64(key, strlen(key), 0);
 
-	ret = _dict_set (this, key, value);
+    LOCK(&this->lock);
+    {
+        tmp = dict_lookup_common(this, key, hash);
+    }
+    UNLOCK(&this->lock);
 
-	UNLOCK (&this->lock);
+    if (!tmp)
+        return -1;
 
-	return ret;
+    *data = tmp->value;
+    return 0;
+}
+
+static int32_t
+dict_set_lk(dict_t *this, char *key, const int key_len, data_t *value,
+            const uint32_t hash, gf_boolean_t replace)
+{
+    int hashval = 0;
+    data_pair_t *pair;
+    int key_free = 0;
+    uint32_t key_hash;
+    int keylen;
+
+    if (!key) {
+        keylen = gf_asprintf(&key, "ref:%p", value);
+        if (-1 == keylen) {
+            return -1;
+        }
+        key_free = 1;
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    } else {
+        keylen = key_len;
+        key_hash = hash;
+    }
+
+    /* Search for a existing key if 'replace' is asked for */
+    if (replace) {
+        pair = dict_lookup_common(this, key, key_hash);
+        if (pair) {
+            data_t *unref_data = pair->value;
+            pair->value = data_ref(value);
+            this->totkvlen += (value->len - unref_data->len);
+            data_unref(unref_data);
+            if (key_free)
+                GF_FREE(key);
+            /* Indicates duplicate key */
+            return 0;
+        }
+    }
+
+    if (this->free_pair.key) { /* the free_pair is used */
+        pair = mem_get(THIS->ctx->dict_pair_pool);
+        if (!pair) {
+            if (key_free)
+                GF_FREE(key);
+            return -1;
+        }
+    } else { /* assign the pair to the free pair */
+        pair = &this->free_pair;
+    }
+
+    if (key_free) {
+        /* It's ours.  Use it. */
+        pair->key = key;
+        key_free = 0;
+    } else {
+        pair->key = (char *)GF_MALLOC(keylen + 1, gf_common_mt_char);
+        if (!pair->key) {
+            if (pair != &this->free_pair) {
+                mem_put(pair);
+            }
+            return -1;
+        }
+        strcpy(pair->key, key);
+    }
+    pair->key_hash = key_hash;
+    pair->value = data_ref(value);
+    this->totkvlen += (keylen + 1 + value->len);
+
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1) {
+        hashval = (key_hash % this->hash_size);
+    }
+    pair->hash_next = this->members[hashval];
+    this->members[hashval] = pair;
+
+    pair->next = this->members_list;
+    pair->prev = NULL;
+    if (this->members_list)
+        this->members_list->prev = pair;
+    this->members_list = pair;
+    this->count++;
+
+    if (key_free)
+        GF_FREE(key);
+
+    if (this->max_count < this->count)
+        this->max_count = this->count;
+    return 0;
 }
 
+int32_t
+dict_set(dict_t *this, char *key, data_t *value)
+{
+    if (key)
+        return dict_setn(this, key, strlen(key), value);
+    else
+        return dict_setn(this, NULL, 0, value);
+}
 
-data_t *
-dict_get (dict_t *this,
-  	  char *key)
+int32_t
+dict_setn(dict_t *this, char *key, const int keylen, data_t *value)
 {
-	data_pair_t *pair;
+    int32_t ret;
+    uint32_t key_hash = 0;
 
-	if (!this || !key) {
-		gf_log_callingfn ("dict", GF_LOG_DEBUG,
-                                  "@this=%p key=%s", this, (key) ? key : "()");
-		return NULL;
-	}
+    if (!this || !value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !value for "
+                         "key=%s",
+                         key);
+        return -1;
+    }
 
-	LOCK (&this->lock);
+    if (key) {
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    }
 
-	pair = _dict_lookup (this, key);
+    LOCK(&this->lock);
 
-	UNLOCK (&this->lock);
+    ret = dict_set_lk(this, key, keylen, value, key_hash, 1);
 
-	if (pair)
-		return pair->value;
-  
-	return NULL;
+    UNLOCK(&this->lock);
+
+    return ret;
 }
 
-void
-dict_del (dict_t *this,
-  	  char *key)
-{
-	if (!this || !key) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p @key=%p", this, key);
-		return;
-	}
-
-	LOCK (&this->lock);
-
-	int hashval = SuperFastHash (key, strlen (key)) % this->hash_size;
-	data_pair_t *pair = this->members[hashval];
-	data_pair_t *prev = NULL;
-  
-	while (pair) {
-		if (strcmp (pair->key, key) == 0) {
-			if (prev)
-				prev->hash_next = pair->hash_next;
-			else
-				this->members[hashval] = pair->hash_next;
-  
-			data_unref (pair->value);
-  
-			if (pair->prev)
-				pair->prev->next = pair->next;
-			else
-				this->members_list = pair->next;
-
-			if (pair->next)
-				pair->next->prev = pair->prev;
-  
-			GF_FREE (pair->key);
-			GF_FREE (pair);
-			this->count--;
-			break;
-		}
- 
-		prev = pair;
-		pair = pair->hash_next;
-	}
-
-	UNLOCK (&this->lock);
-
-	return;
+int32_t
+dict_add(dict_t *this, char *key, data_t *value)
+{
+    if (key)
+        return dict_addn(this, key, strlen(key), value);
+    else
+        return dict_addn(this, NULL, 0, value);
 }
 
-void
-dict_destroy (dict_t *this)
+int32_t
+dict_addn(dict_t *this, char *key, const int keylen, data_t *value)
 {
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return;
-	}
+    int32_t ret;
+    uint32_t key_hash = 0;
 
-	data_pair_t *pair = this->members_list;
-	data_pair_t *prev = this->members_list;
+    if (!this || !value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || !value for key=%s", key);
+        return -1;
+    }
 
-	LOCK_DESTROY (&this->lock);
+    if (key) {
+        key_hash = (uint32_t)XXH64(key, keylen, 0);
+    }
 
-	while (prev) {
-		pair = pair->next;
-		data_unref (prev->value);
-		GF_FREE (prev->key);
-		GF_FREE (prev);
-		prev = pair;
-	}
+    LOCK(&this->lock);
 
-	GF_FREE (this->members);
+    ret = dict_set_lk(this, key, keylen, value, key_hash, 0);
 
-	if (this->extra_free)
-		GF_FREE (this->extra_free);
-        if (this->extra_stdfree)
-                free (this->extra_stdfree);
+    UNLOCK(&this->lock);
+
+    return ret;
+}
 
-	if (!this->is_static)
-		GF_FREE (this);
+data_t *
+dict_get(dict_t *this, char *key)
+{
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", (key) ? key : "()");
+        return NULL;
+    }
 
-	return;
+    return dict_getn(this, key, strlen(key));
 }
 
-void
-dict_unref (dict_t *this)
+data_t *
+dict_getn(dict_t *this, char *key, const int keylen)
 {
-	int32_t ref;
+    data_pair_t *pair;
+    uint32_t hash;
 
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return;
-	}
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", (key) ? key : "()");
+        return NULL;
+    }
 
-	LOCK (&this->lock);
+    hash = (uint32_t)XXH64(key, keylen, 0);
 
-	this->refcount--;
-	ref = this->refcount;
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
+    }
+    UNLOCK(&this->lock);
 
-	UNLOCK (&this->lock);
+    if (pair)
+        return pair->value;
 
-	if (!ref)
-		dict_destroy (this);
+    return NULL;
 }
 
-dict_t *
-dict_ref (dict_t *this)
+int
+dict_key_count(dict_t *this)
 {
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return NULL;
-	}
-
-	LOCK (&this->lock);
+    int ret = -1;
 
-	this->refcount++;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict passed is NULL");
+        return ret;
+    }
 
-	UNLOCK (&this->lock);
+    LOCK(&this->lock);
+    {
+        ret = this->count;
+    }
+    UNLOCK(&this->lock);
 
-	return this;
+    return ret;
 }
 
 void
-data_unref (data_t *this)
+dict_del(dict_t *this, char *key)
 {
-	int32_t ref;
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", key);
+        return;
+    }
 
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return;
-	}
+    return dict_deln(this, key, strlen(key));
+}
 
-	LOCK (&this->lock);
+void
+dict_deln(dict_t *this, char *key, const int keylen)
+{
+    int hashval = 0;
+    uint32_t hash;
+
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!this || key=%s", key);
+        return;
+    }
+
+    hash = (uint32_t)XXH64(key, keylen, 0);
+
+    LOCK(&this->lock);
+
+    /* If the divisor is 1, the modulo is always 0,
+     * in such case avoid hash calculation.
+     */
+    if (this->hash_size != 1)
+        hashval = hash % this->hash_size;
+
+    data_pair_t *pair = this->members[hashval];
+    data_pair_t *prev = NULL;
+
+    while (pair) {
+        if ((hash == pair->key_hash) && strcmp(pair->key, key) == 0) {
+            if (prev)
+                prev->hash_next = pair->hash_next;
+            else
+                this->members[hashval] = pair->hash_next;
+
+            this->totkvlen -= pair->value->len;
+            data_unref(pair->value);
+
+            if (pair->prev)
+                pair->prev->next = pair->next;
+            else
+                this->members_list = pair->next;
+
+            if (pair->next)
+                pair->next->prev = pair->prev;
+
+            this->totkvlen -= (strlen(pair->key) + 1);
+            GF_FREE(pair->key);
+            if (pair == &this->free_pair) {
+                this->free_pair.key = NULL;
+            } else {
+                mem_put(pair);
+            }
+            this->count--;
+            break;
+        }
 
-	this->refcount--;
-	ref = this->refcount;
+        prev = pair;
+        pair = pair->hash_next;
+    }
 
-	UNLOCK (&this->lock);
+    UNLOCK(&this->lock);
 
-	if (!ref)
-		data_destroy (this);
+    return;
 }
 
-data_t *
-data_ref (data_t *this)
-{
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return NULL;
-	}
-
-	LOCK (&this->lock);
-
-	this->refcount++;
-
-	UNLOCK (&this->lock);
+void
+dict_destroy(dict_t *this)
+{
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return;
+    }
+
+    data_pair_t *pair = this->members_list;
+    data_pair_t *prev = this->members_list;
+    glusterfs_ctx_t *ctx = NULL;
+    uint64_t current_max = 0;
+    uint32_t total_pairs = 0;
+
+    LOCK_DESTROY(&this->lock);
+
+    while (prev) {
+        pair = pair->next;
+        data_unref(prev->value);
+        GF_FREE(prev->key);
+        if (prev != &this->free_pair) {
+            mem_put(prev);
+        } else {
+            this->free_pair.key = NULL;
+        }
+        total_pairs++;
+        prev = pair;
+    }
 
-	return this;
-}
+    this->totkvlen = 0;
+    if (this->members != &this->members_internal) {
+        mem_put(this->members);
+    }
 
-/*
-  Serialization format:
-  ----
-  Count:8
-  Key_len:8:Value_len:8
-  Key
-  Value
-  .
-  .
-  .
-*/
+    free(this->extra_stdfree);
 
-int32_t 
-dict_serialized_length_old (dict_t *this)
-{
+    /* update 'ctx->stats.dict.details' using max_count */
+    ctx = THIS->ctx;
 
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p", this);
-		return -1;
-	}
+    /* NOTE: below logic is not totaly race proof */
+    /* thread0 and thread1 gets current_max as 10 */
+    /* thread0 has 'this->max_count as 11 */
+    /* thread1 has 'this->max_count as 20 */
+    /* thread1 goes ahead and sets the max_dict_pairs to 20 */
+    /* thread0 then goes and sets it to 11 */
+    /* As it is for information purpose only, no functionality will be
+       broken by this, but a point to consider about ATOMIC macros. */
+    current_max = GF_ATOMIC_GET(ctx->stats.max_dict_pairs);
+    if (current_max < this->max_count)
+        GF_ATOMIC_INIT(ctx->stats.max_dict_pairs, this->max_count);
 
-	int32_t len = 9; /* count + \n */
-	int32_t count = this->count;
-	data_pair_t *pair = this->members_list;
+    GF_ATOMIC_ADD(ctx->stats.total_pairs_used, total_pairs);
+    GF_ATOMIC_INC(ctx->stats.total_dicts_used);
 
-	while (count) {
-		len += 18;
-		len += strlen (pair->key) + 1;
-		if (pair->value->vec) {
-			int i;
-			for (i=0; i<pair->value->len; i++) {
-				len += pair->value->vec[i].iov_len;
-			}
-		} else {
-			len += pair->value->len;
-		}
-		pair = pair->next;
-		count--;
-	}
+    mem_put(this);
 
-	return len;
+    return;
 }
 
-int32_t 
-dict_serialize_old (dict_t *this, char *buf)
+void
+dict_unref(dict_t *this)
 {
-	if (!this || !buf) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"@this=%p @buf=%p", this, buf);
-		return -1;
-	}
+    uint64_t ref = 0;
 
-	data_pair_t *pair = this->members_list;
-	int32_t count = this->count;
-	uint64_t dcount = this->count;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return;
+    }
 
-	// FIXME: magic numbers
+    ref = GF_ATOMIC_DEC(this->refcount);
 
-	sprintf (buf, "%08"PRIx64"\n", dcount);
-	buf += 9;
-	while (count) {
-		uint64_t keylen = strlen (pair->key) + 1;
-		uint64_t vallen = pair->value->len;
-
-		sprintf (buf, "%08"PRIx64":%08"PRIx64"\n", keylen, vallen);
-		buf += 18;
-		memcpy (buf, pair->key, keylen);
-		buf += keylen;
-		memcpy (buf, pair->value->data, pair->value->len);
-		buf += pair->value->len;
-		pair = pair->next;
-		count--;
-	}
-	return (0);
+    if (!ref)
+        dict_destroy(this);
 }
 
-
 dict_t *
-dict_unserialize_old (char *buf, int32_t size, dict_t **fill)
-{
-	int32_t ret = 0;
-	int32_t cnt = 0;
-
-	if (!buf || !fill || !(*fill)) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"@buf=%p @fill=%p @*fill=%p",
-                        buf, fill, (fill) ? (*fill) : NULL);
-		return NULL;
-	}
-
-	uint64_t count;
-	ret = sscanf (buf, "%"SCNx64"\n", &count);
-	(*fill)->count = 0;
-
-	if (!ret){
-		gf_log ("dict",
-			GF_LOG_ERROR,
-			"sscanf on buf failed");
-		goto err;
-	}
-	buf += 9;
-  
-	if (count == 0) {
-		gf_log ("dict",
-			GF_LOG_ERROR,
-			"count == 0");
-		goto err;
-	}
-
-	for (cnt = 0; cnt < count; cnt++) {
-		data_t *value = NULL;
-		char *key = NULL;
-		uint64_t key_len, value_len;
-    
-		ret = sscanf (buf, "%"SCNx64":%"SCNx64"\n", &key_len, &value_len);
-		if (ret != 2) {
-			gf_log ("dict",
-				GF_LOG_ERROR,
-				"sscanf for key_len and value_len failed");
-			goto err;
-		}
-		buf += 18;
-
-		key = buf;
-		buf += key_len;
-    
-		value = get_new_data ();
-		value->len = value_len;
-		value->data = buf;
-		value->is_static = 1;
-		buf += value_len;
-
-		dict_set (*fill, key, value);
-	}
-
-	goto ret;
-
-err:
-	GF_FREE (*fill);
-	*fill = NULL; 
+dict_ref(dict_t *this)
+{
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_DEBUG, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return NULL;
+    }
 
-ret:
-	return *fill;
+    GF_ATOMIC_INC(this->refcount);
+    return this;
 }
 
-
-int32_t
-dict_iovec_len (dict_t *this)
+void
+data_unref(data_t *this)
 {
-	if (!this) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@this=%p", this);
-		return -1;
-	}
+    uint64_t ref;
 
-	int32_t len = 0;
-	data_pair_t *pair = this->members_list;
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "data is NULL");
+        return;
+    }
 
-	len++; /* initial header */
-	while (pair) {
-		len++; /* pair header */
-		len++; /* key */
+    ref = GF_ATOMIC_DEC(this->refcount);
 
-		if (pair->value->vec)
-			len += pair->value->len;
-		else
-			len++;
-		pair = pair->next;
-	}
-
-	return len;
-}
-
-int32_t
-dict_to_iovec (dict_t *this,
-	       struct iovec *vec,
-	       int32_t count)
-{
-	if (!this || !vec) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@this=%p @vec=%p", this, vec);
-		return -1;
-	}
-
-	int32_t i = 0;
-	data_pair_t *pair = this->members_list;
-
-	vec[0].iov_len = 9;
-	if (vec[0].iov_base)
-		sprintf (vec[0].iov_base,
-			 "%08"PRIx64"\n",
-			 (int64_t)this->count);
-	i++;
-
-	while (pair) {
-		int64_t keylen = strlen (pair->key) + 1;
-		int64_t vallen = 0;
-
-		if (pair->value->vec) {
-			int i;
-
-			for (i=0; i<pair->value->len; i++) {
-				vallen += pair->value->vec[i].iov_len;
-			}
-		} else {
-			vallen = pair->value->len;
-		}
-
-		vec[i].iov_len = 18;
-		if (vec[i].iov_base)
-			sprintf (vec[i].iov_base,
-				 "%08"PRIx64":%08"PRIx64"\n",
-				 keylen,
-				 vallen);
-		i++;
-
-		vec[i].iov_len = keylen;
-		vec[i].iov_base = pair->key;
-		i++;
-
-		if (pair->value->vec) {
-			int k;
-
-			for (k=0; k<pair->value->len; k++) {
-				vec[i].iov_len = pair->value->vec[k].iov_len;
-				vec[i].iov_base = pair->value->vec[k].iov_base;
-				i++;
-			}
-		} else {
-			vec[i].iov_len = pair->value->len;
-			vec[i].iov_base = pair->value->data;
-			i++;
-		}
-
-		pair = pair->next;
-	}
-
-	return 0;
+    if (!ref)
+        data_destroy(this);
 }
 
 data_t *
-int_to_data (int64_t value)
+data_ref(data_t *this)
 {
-        int     ret = 0;
-	data_t *data = get_new_data ();
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "data is NULL");
+        return NULL;
+    }
 
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-
-	ret = gf_asprintf (&data->data, "%"PRId64, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
-	data->len = strlen (data->data) + 1;
+    GF_ATOMIC_INC(this->refcount);
 
-	return data;
+    return this;
 }
 
 data_t *
-data_from_int64 (int64_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRId64, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
-	data->len = strlen (data->data) + 1;
+int_to_data(int64_t value)
+{
+    data_t *data = get_new_data();
+
+    if (!data) {
+        return NULL;
+    }
+
+    data->len = gf_asprintf(&data->data, "%" PRId64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-	return data;
+    return data;
 }
 
 data_t *
-data_from_int32 (int32_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRId32, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_int64(int64_t value)
+{
+    data_t *data = get_new_data();
 
-	data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-	return data;
+    return data;
 }
 
 data_t *
-data_from_int16 (int16_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRId16, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_int32(int32_t value)
+{
+    data_t *data = get_new_data();
 
-	data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId32, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	return data;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
+
+    return data;
 }
 
 data_t *
-data_from_int8 (int8_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%d", value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_int16(int16_t value)
+{
+    data_t *data = get_new_data();
+
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRId16, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
 
-	return data;
+    return data;
 }
 
 data_t *
-data_from_uint64 (uint64_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRIu64, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_int8(int8_t value)
+{
+    data_t *data = get_new_data();
 
-	data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%d", value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	return data;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_INT;
+
+    return data;
 }
 
-static data_t *
-data_from_double (double value)
+data_t *
+data_from_uint64(uint64_t value)
 {
-	data_t *data = NULL;
-	int     ret  = 0;
-
-	data = get_new_data ();
+    data_t *data = get_new_data();
 
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu64, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	ret = gf_asprintf (&data->data, "%f", value);
-	if (ret == -1) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - allocation failed by ASPRINTF");
-		return NULL;
-	}
-	data->len = strlen (data->data) + 1;
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
-	return data;
+    return data;
 }
 
-
 data_t *
-data_from_uint32 (uint32_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRIu32, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_double(double value)
+{
+    data_t *data = get_new_data();
 
-	data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
 
-	return data;
-}
+    data->len = gf_asprintf(&data->data, "%f", value);
+    if (data->len == -1) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_DOUBLE;
 
+    return data;
+}
 
 data_t *
-data_from_uint16 (uint16_t value)
-{
-        int     ret = 0;
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	ret = gf_asprintf (&data->data, "%"PRIu16, value);
-        if (-1 == ret) {
-                gf_log ("dict", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
+data_from_uint32(uint32_t value)
+{
+    data_t *data = get_new_data();
 
-	data->len = strlen (data->data) + 1;
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu32, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	return data;
-}
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
+    return data;
+}
 
 data_t *
-data_from_ptr (void *value)
+data_from_uint16(uint16_t value)
 {
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
+    data_t *data = get_new_data();
 
-	data_t *data = get_new_data ();
+    if (!data) {
+        return NULL;
+    }
+    data->len = gf_asprintf(&data->data, "%" PRIu16, value);
+    if (-1 == data->len) {
+        gf_msg_debug("dict", 0, "asprintf failed");
+        data_destroy(data);
+        return NULL;
+    }
 
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
+    data->len++; /* account for terminating NULL */
+    data->data_type = GF_DATA_TYPE_UINT;
 
-	data->data = value;
-	return data;
+    return data;
 }
 
-data_t *
-data_from_static_ptr (void *value)
+static data_t *
+data_from_ptr_common(void *value, gf_boolean_t is_static)
 {
-/*
-  this is valid to set 0 as value..
+    /* it is valid to set 0/NULL as a value, no need to check *value */
 
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
-*/
-	data_t *data = get_new_data ();
+    data_t *data = get_new_data();
+    if (!data) {
+        return NULL;
+    }
 
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
+    data->data = value;
+    data->len = 0;
+    data->is_static = is_static;
 
-	data->is_static = 1;
-	data->data = value;
-
-	return data;
+    data->data_type = GF_DATA_TYPE_PTR;
+    return data;
 }
 
 data_t *
-str_to_data (char *value)
+str_to_data(char *value)
 {
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
-	data_t *data = get_new_data ();
-
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data - NULL returned by CALLOC");
-		return NULL;
-	}
-	data->len = strlen (value) + 1;
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-	data->data = value;
-	data->is_static = 1;
-
-	return data;
+    return strn_to_data(value, strlen(value));
 }
 
 data_t *
-data_from_dynstr (char *value)
+strn_to_data(char *value, const int vallen)
 {
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
+    data_t *data = get_new_data();
 
-	data_t *data = get_new_data ();
+    if (!data) {
+        return NULL;
+    }
+    data->len = vallen + 1;
+    data->data_type = GF_DATA_TYPE_STR;
 
-	data->len = strlen (value) + 1;
-	data->data = value;
+    data->data = value;
+    data->is_static = _gf_true;
 
-	return data;
+    return data;
 }
 
-data_t *
-data_from_dynmstr (char *value)
+static data_t *
+data_from_dynstr(char *value)
 {
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-	data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-	data->len = strlen (value) + 1;
-	data->data = value;
-        data->is_stdalloc = 1;
+    if (!data)
+        return NULL;
+    data->len = strlen(value) + 1;
+    data->data = value;
+    data->data_type = GF_DATA_TYPE_STR;
 
-	return data;
+    return data;
 }
 
 data_t *
-data_from_dynptr (void *value, int32_t len)
+data_from_dynptr(void *value, int32_t len)
 {
-	data_t *data = get_new_data ();
+    data_t *data = get_new_data();
+
+    if (!data)
+        return NULL;
 
-	data->len = len;
-	data->data = value;
+    data->len = len;
+    data->data = value;
+    data->data_type = GF_DATA_TYPE_PTR;
 
-	return data;
+    return data;
 }
 
 data_t *
-bin_to_data (void *value, int32_t len)
+bin_to_data(void *value, int32_t len)
 {
-	if (!value) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@value=%p", value);
-		return NULL;
-	}
+    if (!value) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "value is NULL");
+        return NULL;
+    }
 
-	data_t *data = get_new_data ();
+    data_t *data = get_new_data();
 
-	data->is_static = 1;
-	data->len = len;
-	data->data = value;
+    if (!data)
+        return NULL;
 
-	return data;
+    data->is_static = _gf_true;
+    data->len = len;
+    data->data = value;
+
+    return data;
 }
 
+static char *data_type_name[GF_DATA_TYPE_MAX] = {
+    [GF_DATA_TYPE_UNKNOWN] = "unknown",
+    [GF_DATA_TYPE_STR_OLD] = "string-old-version",
+    [GF_DATA_TYPE_INT] = "integer",
+    [GF_DATA_TYPE_UINT] = "unsigned integer",
+    [GF_DATA_TYPE_DOUBLE] = "float",
+    [GF_DATA_TYPE_STR] = "string",
+    [GF_DATA_TYPE_PTR] = "pointer",
+    [GF_DATA_TYPE_GFUUID] = "gf-uuid",
+    [GF_DATA_TYPE_IATT] = "iatt",
+    [GF_DATA_TYPE_MDATA] = "mdata",
+};
+
 int64_t
-data_to_int64 (data_t *data)
-{
-	if (!data)
-		return -1;
+data_to_int64(data_t *data)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+
+    char *endptr = NULL;
+    int64_t value = 0;
+
+    errno = 0;
+    value = strtoll(data->data, &endptr, 0);
+
+    if (endptr && *endptr != '\0')
+        /* Unrecognized characters at the end of string. */
+        errno = EINVAL;
+    if (errno) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, errno,
+                         LG_MSG_DATA_CONVERSION_ERROR,
+                         "Error in data conversion: '%s' can't "
+                         "be represented as int64_t",
+                         data->data);
+        return -1;
+    }
+    return value;
+}
+
+/* Like above but implies signed range check. */
+
+#define DATA_TO_RANGED_SIGNED(endptr, value, data, type, min, max)             \
+    do {                                                                       \
+        errno = 0;                                                             \
+        value = strtoll(data->data, &endptr, 0);                               \
+        if (endptr && *endptr != '\0')                                         \
+            errno = EINVAL;                                                    \
+        if (errno || value > max || value < min) {                             \
+            gf_msg_callingfn("dict", GF_LOG_WARNING, errno,                    \
+                             LG_MSG_DATA_CONVERSION_ERROR,                     \
+                             "Error in data conversion: '%s' can't "           \
+                             "be represented as " #type,                       \
+                             data->data);                                      \
+            return -1;                                                         \
+        }                                                                      \
+        return (type)value;                                                    \
+    } while (0)
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+int32_t
+data_to_int32(data_t *data)
+{
+    char *endptr = NULL;
+    int64_t value = 0;
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
-	return (int64_t) strtoull (str, NULL, 0);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int32_t, INT_MIN, INT_MAX);
 }
 
-int32_t
-data_to_int32 (data_t *data)
+int16_t
+data_to_int16(data_t *data)
 {
-	if (!data)
-		return -1;
+    char *endptr = NULL;
+    int64_t value = 0;
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int16_t, SHRT_MIN, SHRT_MAX);
+}
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+int8_t
+data_to_int8(data_t *data)
+{
+    char *endptr = NULL;
+    int64_t value = 0;
 
-	return strtoul (str, NULL, 0);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, "null", -1);
+    DATA_TO_RANGED_SIGNED(endptr, value, data, int8_t, CHAR_MIN, CHAR_MAX);
 }
 
-int16_t
-data_to_int16 (data_t *data)
+uint64_t
+data_to_uint64(data_t *data)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+
+    char *endptr = NULL;
+    uint64_t value = 0;
+
+    errno = 0;
+    value = strtoull(data->data, &endptr, 0);
+
+    if (endptr && *endptr != '\0')
+        errno = EINVAL;
+    if (errno) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, errno,
+                         LG_MSG_DATA_CONVERSION_ERROR,
+                         "Error in data conversion: '%s' can't "
+                         "be represented as uint64_t",
+                         data->data);
+        return -1;
+    }
+    return value;
+}
+
+/* Like above but implies unsigned range check. */
+
+#define DATA_TO_RANGED_UNSIGNED(endptr, value, data, type, max)                \
+    do {                                                                       \
+        errno = 0;                                                             \
+        value = strtoull(data->data, &endptr, 0);                              \
+        if (endptr && *endptr != '\0')                                         \
+            errno = EINVAL;                                                    \
+        if (errno || value > max) {                                            \
+            gf_msg_callingfn("dict", GF_LOG_WARNING, errno,                    \
+                             LG_MSG_DATA_CONVERSION_ERROR,                     \
+                             "Error in data conversion: '%s' can't "           \
+                             "be represented as " #type,                       \
+                             data->data);                                      \
+            return -1;                                                         \
+        }                                                                      \
+        return (type)value;                                                    \
+    } while (0)
+
+uint32_t
+data_to_uint32(data_t *data)
 {
-	if (!data)
-		return -1;
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint32_t, UINT_MAX);
+}
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+uint16_t
+data_to_uint16(data_t *data)
+{
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-	return strtol (str, NULL, 0);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint16_t, USHRT_MAX);
 }
 
-
-int8_t
-data_to_int8 (data_t *data)
+uint8_t
+data_to_uint8(data_t *data)
 {
-	if (!data)
-		return -1;
+    char *endptr = NULL;
+    uint64_t value = 0;
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, "null", -1);
+    DATA_TO_RANGED_UNSIGNED(endptr, value, data, uint8_t, UCHAR_MAX);
+}
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+char *
+data_to_str(data_t *data)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, "null", NULL);
+    return data->data;
+}
 
-	return (int8_t)strtol (str, NULL, 0);
+void *
+data_to_ptr(data_t *data)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, "null", NULL);
+    return data->data;
 }
 
+void *
+data_to_bin(data_t *data)
+{
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, "null", NULL);
+    return data->data;
+}
 
-uint64_t
-data_to_uint64 (data_t *data)
+struct iatt *
+data_to_iatt(data_t *data, char *key)
 {
-	if (!data)
-		return -1;
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_IATT, key, NULL);
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    /* We only check for smaller size. If it's bigger we simply ignore
+     * the extra data. This way it's easy to do changes in the future that
+     * pass more data but are backward compatible (if the initial contents
+     * of the struct are maintained, of course). */
+    if (data->len < sizeof(struct iatt)) {
+        gf_smsg("glusterfs", GF_LOG_ERROR, ENOBUFS, LG_MSG_UNDERSIZED_BUF,
+                "key=%s", key, NULL);
+        return NULL;
+    }
 
-	return strtoll (str, NULL, 0);
+    return (struct iatt *)data->data;
 }
 
-uint32_t
-data_to_uint32 (data_t *data)
+int
+dict_null_foreach_fn(dict_t *d, char *k, data_t *v, void *tmp)
 {
-	if (!data)
-		return -1;
+    return 0;
+}
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+int
+dict_remove_foreach_fn(dict_t *d, char *k, data_t *v, void *_tmp)
+{
+    if (!d || !k) {
+        gf_smsg("glusterfs", GF_LOG_WARNING, EINVAL, LG_MSG_KEY_OR_VALUE_NULL,
+                "d=%s", d ? "key" : "dictionary", NULL);
+        return -1;
+    }
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    dict_del(d, k);
+    return 0;
+}
 
-	return strtol (str, NULL, 0);
+gf_boolean_t
+dict_match_everything(dict_t *d, char *k, data_t *v, void *data)
+{
+    return _gf_true;
 }
 
-uint16_t
-data_to_uint16 (data_t *data)
+int
+dict_foreach(dict_t *dict,
+             int (*fn)(dict_t *this, char *key, data_t *value, void *data),
+             void *data)
 {
-	if (!data)
-		return -1;
+    int ret = dict_foreach_match(dict, dict_match_everything, NULL, fn, data);
 
-	char *str = alloca (data->len + 1);
-        if (!str)
-                return -1;
+    if (ret > 0)
+        ret = 0;
+
+    return ret;
+}
 
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+/* return values:
+   -1 = failure,
+    0 = no matches found,
+   +n = n number of matches
+*/
+int
+dict_foreach_match(dict_t *dict,
+                   gf_boolean_t (*match)(dict_t *this, char *key, data_t *value,
+                                         void *mdata),
+                   void *match_data,
+                   int (*action)(dict_t *this, char *key, data_t *value,
+                                 void *adata),
+                   void *action_data)
+{
+    if (!dict || !match || !action) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict|match|action is "
+                         "NULL");
+        return -1;
+    }
+
+    int ret = -1;
+    int count = 0;
+    data_pair_t *pairs = dict->members_list;
+    data_pair_t *next = NULL;
+
+    while (pairs) {
+        next = pairs->next;
+        if (match(dict, pairs->key, pairs->value, match_data)) {
+            ret = action(dict, pairs->key, pairs->value, action_data);
+            if (ret < 0)
+                return ret;
+            count++;
+        }
+        pairs = next;
+    }
 
-	return strtol (str, NULL, 0);
+    return count;
 }
 
-char *
-data_to_str (data_t *data)
+static gf_boolean_t
+dict_fnmatch(dict_t *d, char *k, data_t *val, void *match_data)
 {
-	if (!data) {
-		gf_log ("dict", GF_LOG_WARNING,
-			"@data=%p", data);
-		return NULL;
-	}
-	return data->data;
+    return (fnmatch(match_data, k, 0) == 0);
 }
-
-void *
-data_to_ptr (data_t *data)
+/* return values:
+   -1 = failure,
+    0 = no matches found,
+   +n = n number of matches
+*/
+int
+dict_foreach_fnmatch(dict_t *dict, char *pattern,
+                     int (*fn)(dict_t *this, char *key, data_t *value,
+                               void *data),
+                     void *data)
 {
-	if (!data) {
-		return NULL;
-	}
-	return data->data;
+    return dict_foreach_match(dict, dict_fnmatch, pattern, fn, data);
 }
 
-void *
-data_to_bin (data_t *data)
+/**
+ * dict_keys_join - pack the keys of the dictionary in a buffer.
+ *
+ * @value     : buffer in which the keys will be packed (can be NULL)
+ * @size      : size of the buffer which is sent (can be 0, in which case buffer
+ *              is not packed but only length is returned)
+ * @dict      : dictionary of which all the keys will be packed
+ * @filter_fn : keys matched in filter_fn() is counted.
+ *
+ * @return : @length of string after joining keys.
+ *
+ */
+
+int
+dict_keys_join(void *value, int size, dict_t *dict, int (*filter_fn)(char *k))
 {
-	if (!data) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data=%p", data);
-		return NULL;
-	}
-	return data->data;
-}
+    int len = 0;
+    data_pair_t *pairs = dict->members_list;
+    data_pair_t *next = NULL;
 
-void
-dict_foreach (dict_t *dict,
-	      void (*fn)(dict_t *this,
-			 char *key,
-			 data_t *value,
-			 void *data),
-	      void *data)
-{
-        if (!data || !dict) {
-                gf_log_callingfn ("dict", GF_LOG_CRITICAL,
-                        "@data=%p, @dict=%p", data, dict);
-                return;
+    while (pairs) {
+        next = pairs->next;
+
+        if (filter_fn && filter_fn(pairs->key)) {
+            pairs = next;
+            continue;
         }
 
-        data_pair_t *pairs = dict->members_list;
-        data_pair_t *next = NULL;
+        if (value && (size > len))
+            strncpy(value + len, pairs->key, size - len);
 
-        while (pairs) {
-                next = pairs->next;
-                fn (dict, pairs->key, pairs->value, data);
-                pairs = next;
-        }
-}
+        len += (strlen(pairs->key) + 1);
 
+        pairs = next;
+    }
 
-static void
-_copy (dict_t *unused,
-       char *key,
-       data_t *value,
-       void *newdict)
-{
-	dict_set ((dict_t *)newdict, key, (value));
+    return len;
 }
 
+static int
+dict_copy_one(dict_t *unused, char *key, data_t *value, void *newdict)
+{
+    return dict_set((dict_t *)newdict, key, (value));
+}
 
 dict_t *
-dict_copy (dict_t *dict,
-	   dict_t *new)
+dict_copy(dict_t *dict, dict_t *new)
 {
-	if (!dict) {
-		gf_log ("dict", GF_LOG_CRITICAL,
-			"@data=%p", dict);
-		return NULL;
-	}
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return NULL;
+    }
+
+    if (!new)
+        new = get_new_dict_full(dict->hash_size);
 
-	if (!new)
-		new = get_new_dict_full (dict->hash_size);
+    dict_foreach(dict, dict_copy_one, new);
 
-	dict_foreach (dict, _copy, new);
+    return new;
+}
 
-	return new;
+int
+dict_reset(dict_t *dict)
+{
+    int32_t ret = -1;
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+    dict_foreach(dict, dict_remove_foreach_fn, NULL);
+    ret = 0;
+out:
+    return ret;
 }
 
 dict_t *
-dict_copy_with_ref (dict_t *dict,
-		    dict_t *new)
+dict_copy_with_ref(dict_t *dict, dict_t *new)
 {
-	dict_t *local_new = NULL;
+    dict_t *local_new = NULL;
 
-	GF_VALIDATE_OR_GOTO("dict", dict, fail);
+    GF_VALIDATE_OR_GOTO("dict", dict, fail);
 
-	if (new == NULL) {
-		local_new = dict_new ();
-		GF_VALIDATE_OR_GOTO("dict", local_new, fail);
-		new = local_new;
-	}
+    if (new == NULL) {
+        local_new = dict_new();
+        GF_VALIDATE_OR_GOTO("dict", local_new, fail);
+        new = local_new;
+    }
 
-	dict_foreach (dict, _copy, new);
+    dict_foreach(dict, dict_copy_one, new);
 fail:
-	return new;
+    return new;
 }
 
 /*
@@ -1306,873 +1490,1297 @@ fail:
 
 /**
  * Common cleaned up interface:
- * 
+ *
  * Return value:  0   success
  *               -val error, val = errno
  */
 
-
 static int
-dict_get_with_ref (dict_t *this, char *key, data_t **data)
+dict_get_with_refn(dict_t *this, char *key, const int keylen, data_t **data)
 {
-	data_pair_t * pair = NULL;
-	int           ret  = -ENOENT;
+    data_pair_t *pair = NULL;
+    int ret = -ENOENT;
+    uint32_t hash;
+
+    hash = (uint32_t)XXH64(key, keylen, 0);
 
-	if (!this || !key || !data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
 
-	LOCK (&this->lock);
-	{
-		pair = _dict_lookup (this, key);
-	}
-	UNLOCK (&this->lock);
+        if (pair) {
+            ret = 0;
+            *data = data_ref(pair->value);
+        }
+    }
+    UNLOCK(&this->lock);
+
+    return ret;
+}
 
-	if (pair) {
-		ret = 0;
-		*data = data_ref (pair->value);
-	}
+int
+dict_get_with_ref(dict_t *this, char *key, data_t **data)
+{
+    if (!this || !key || !data) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict OR key (%s) is NULL", key);
+        return -EINVAL;
+    }
 
-err:  
-	return ret;
+    return dict_get_with_refn(this, key, strlen(key), data);
 }
 
 static int
-_data_to_ptr (data_t *data, void **val)
+data_to_ptr_common(data_t *data, void **val)
 {
-	int ret = 0;
+    int ret = 0;
 
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	*val = data->data;
+    *val = data->data;
 err:
-	return ret;
+    return ret;
 }
 
-
 static int
-_data_to_int8 (data_t *data, int8_t *val)
+data_to_int8_ptr(data_t *data, int8_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
+    int ret = 0;
 
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
-
-	errno = 0;
-	*val = strtol (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_int16 (data_t *data, int16_t *val)
+data_to_int16_ptr(data_t *data, int16_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
-
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    int ret = 0;
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	errno = 0;
-	*val = strtol (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_int32 (data_t *data, int32_t *val)
+data_to_int32_ptr(data_t *data, int32_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
+    int ret = 0;
 
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
-
-	errno = 0;
-	*val = strtol (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtol(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_int64 (data_t *data, int64_t *val)
+data_to_int64_ptr(data_t *data, int64_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
-
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    int ret = 0;
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	errno = 0;
-	*val = strtoll (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtoll(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_uint16 (data_t *data, uint16_t *val)
+data_to_uint16_ptr(data_t *data, uint16_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
+    int ret = 0;
 
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
-
-	errno = 0;
-	*val = strtoul (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtoul(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_uint32 (data_t *data, uint32_t *val)
+data_to_uint32_ptr(data_t *data, uint32_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
-
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    int ret = 0;
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	errno = 0;
-	*val = strtoul (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtoul(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_uint64 (data_t *data, uint64_t *val)
+data_to_uint64_ptr(data_t *data, uint64_t *val)
 {
-	int    ret = 0;
-	char * str = NULL;
+    int ret = 0;
 
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
-
-	errno = 0;
-	*val = strtoull (str, NULL, 0);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtoull(data->data, NULL, 0);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 static int
-_data_to_double (data_t *data, double *val)
+data_to_double_ptr(data_t *data, double *val)
 {
-	int    ret = 0;
-	char * str = NULL;
-
-	if (!data || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    int ret = 0;
 
-	str = alloca (data->len + 1);
-	if (!str) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	memcpy (str, data->data, data->len);
-	str[data->len] = '\0';
+    if (!data || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	errno = 0;
-	*val = strtod (str, NULL);
-	if (errno != 0)
-		ret = -errno;
+    errno = 0;
+    *val = strtod(data->data, NULL);
+    if (errno != 0)
+        ret = -errno;
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_int8 (dict_t *this, char *key, int8_t *val)
+dict_get_int8(dict_t *this, char *key, int8_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = data_to_int8_ptr(data, val);
 
-	ret = _data_to_int8 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_int8 (dict_t *this, char *key, int8_t val)
+dict_set_int8(dict_t *this, char *key, int8_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	data = data_from_int8 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    data = data_from_int8(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_int16 (dict_t *this, char *key, int16_t *val)
+dict_get_int16(dict_t *this, char *key, int16_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
+
+    ret = data_to_int16_ptr(data, val);
 
-	ret = _data_to_int16 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_int16 (dict_t *this, char *key, int16_t val)
+dict_set_int16(dict_t *this, char *key, int16_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	data = data_from_int16 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    data = data_from_int16(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_int32 (dict_t *this, char *key, int32_t *val)
+dict_get_int32n(dict_t *this, char *key, const int keylen, int32_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!this || !key || !val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = dict_get_with_refn(this, key, keylen, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
+
+    ret = data_to_int32_ptr(data, val);
 
-	ret = _data_to_int32 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
-
 int
-dict_set_int32 (dict_t *this, char *key, int32_t val)
+dict_get_int32(dict_t *this, char *key, int32_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	data = data_from_int32 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
+
+    ret = data_to_int32_ptr(data, val);
 
 err:
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
 int
-dict_get_int64 (dict_t *this, char *key, int64_t *val)
+dict_set_int32n(dict_t *this, char *key, const int keylen, int32_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    data = data_from_int32(val);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
-	ret = _data_to_int64 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    return ret;
 }
 
-
 int
-dict_set_int64 (dict_t *this, char *key, int64_t val)
+dict_set_int32(dict_t *this, char *key, int32_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_int32(val);
+    int ret = 0;
 
-	data = data_from_int64 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_uint16 (dict_t *this, char *key, uint16_t *val)
+dict_get_int64(dict_t *this, char *key, int64_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = data_to_int64_ptr(data, val);
 
-	ret = _data_to_uint16 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
+int
+dict_set_int64(dict_t *this, char *key, int64_t val)
+{
+    data_t *data = data_from_int64(val);
+    int ret = 0;
+
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+
+err:
+    return ret;
+}
 
 int
-dict_set_uint16 (dict_t *this, char *key, uint16_t val)
+dict_get_uint16(dict_t *this, char *key, uint16_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	data = data_from_uint16 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
+
+    ret = data_to_uint16_ptr(data, val);
 
 err:
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
 int
-dict_get_uint32 (dict_t *this, char *key, uint32_t *val)
+dict_set_uint16(dict_t *this, char *key, uint16_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_uint16(val);
+    int ret = 0;
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
-	ret = _data_to_uint32 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    return ret;
 }
 
+int
+dict_get_uint32(dict_t *this, char *key, uint32_t *val)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
+
+    ret = data_to_uint32_ptr(data, val);
+
+err:
+    if (data)
+        data_unref(data);
+    return ret;
+}
 
 int
-dict_set_uint32 (dict_t *this, char *key, uint32_t val)
+dict_set_uint32(dict_t *this, char *key, uint32_t val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_uint32(val);
+    int ret = 0;
 
-	data = data_from_uint32 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_uint64 (dict_t *this, char *key, uint64_t *val)
+dict_get_uint64(dict_t *this, char *key, uint64_t *val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_UINT, key, -EINVAL);
+
+    ret = data_to_uint64_ptr(data, val);
 
-	ret = _data_to_uint64 (data, val);
-    
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
+int
+dict_set_uint64(dict_t *this, char *key, uint64_t val)
+{
+    data_t *data = data_from_uint64(val);
+    int ret = 0;
+
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
+err:
+    return ret;
+}
+
+/*
+ * dict_check_flag can be used to check a one bit flag in an array of flags
+ * The flag argument indicates the bit position (within the array of bits).
+ * Currently limited to max of 256 flags for a key.
+ * return value,
+ * 1 : flag is set
+ * 0 : flag is not set
+ * <0: Error
+ */
 int
-dict_set_uint64 (dict_t *this, char *key, uint64_t val)
+dict_check_flag(dict_t *this, char *key, int flag)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = -ENOENT;
 
-	data = data_from_uint64 (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (BIT_VALUE((unsigned char *)(data->data), flag))
+        ret = 1;
+    else
+        ret = 0;
+
+    data_unref(data);
+    return ret;
+}
+
+/*
+ * _dict_modify_flag can be used to set/clear a bit flag in an array of flags
+ * flag: indicates the bit position. limited to max of DICT_MAX_FLAGS.
+ * op: Indicates operation DICT_FLAG_SET / DICT_FLAG_CLEAR
+ */
+static int
+_dict_modify_flag(dict_t *this, char *key, int flag, int op)
+{
+    data_t *data = NULL;
+    int ret = 0;
+    data_pair_t *pair = NULL;
+    char *ptr = NULL;
+    int hashval = 0;
+    uint32_t hash;
+
+    if (!this || !key) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict OR key (%s) is NULL", key);
+        ret = -EINVAL;
+        goto err;
+    }
+
+    /*
+     * Using a size of 32 bytes to support max of 256
+     * flags in a single key. This should be suffcient.
+     */
+    GF_ASSERT(flag >= 0 && flag < DICT_MAX_FLAGS);
+
+    hash = (uint32_t)XXH64(key, strlen(key), 0);
+    LOCK(&this->lock);
+    {
+        pair = dict_lookup_common(this, key, hash);
+
+        if (pair) {
+            data = pair->value;
+            if (op == DICT_FLAG_SET)
+                BIT_SET((unsigned char *)(data->data), flag);
+            else
+                BIT_CLEAR((unsigned char *)(data->data), flag);
+        } else {
+            ptr = GF_CALLOC(1, DICT_MAX_FLAGS / 8, gf_common_mt_char);
+            if (!ptr) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                        "flag bit array", NULL);
+                ret = -ENOMEM;
+                goto err;
+            }
 
-	ret = dict_set (this, key, data);
+            data = data_from_dynptr(ptr, DICT_MAX_FLAGS / 8);
+
+            if (!data) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY, "data",
+                        NULL);
+                GF_FREE(ptr);
+                ret = -ENOMEM;
+                goto err;
+            }
+
+            if (op == DICT_FLAG_SET)
+                BIT_SET((unsigned char *)(data->data), flag);
+            else
+                BIT_CLEAR((unsigned char *)(data->data), flag);
+
+            if (this->free_pair.key) { /* the free pair is in use */
+                pair = mem_get0(THIS->ctx->dict_pair_pool);
+                if (!pair) {
+                    gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                            "dict pair", NULL);
+                    ret = -ENOMEM;
+                    goto err;
+                }
+            } else { /* use the free pair */
+                pair = &this->free_pair;
+            }
+
+            pair->key = (char *)GF_MALLOC(strlen(key) + 1, gf_common_mt_char);
+            if (!pair->key) {
+                gf_smsg("dict", GF_LOG_ERROR, ENOMEM, LG_MSG_NO_MEMORY,
+                        "dict pair", NULL);
+                ret = -ENOMEM;
+                goto err;
+            }
+            strcpy(pair->key, key);
+            pair->key_hash = hash;
+            pair->value = data_ref(data);
+            this->totkvlen += (strlen(key) + 1 + data->len);
+            hashval = hash % this->hash_size;
+            pair->hash_next = this->members[hashval];
+            this->members[hashval] = pair;
+
+            pair->next = this->members_list;
+            pair->prev = NULL;
+            if (this->members_list)
+                this->members_list->prev = pair;
+            this->members_list = pair;
+            this->count++;
+
+            if (this->max_count < this->count)
+                this->max_count = this->count;
+        }
+    }
+
+    UNLOCK(&this->lock);
+    return 0;
 
 err:
-	return ret;
+    if (key && this)
+        UNLOCK(&this->lock);
+
+    if (pair) {
+        if (pair->key) {
+            GF_FREE(pair->key);
+            pair->key = NULL;
+        }
+        if (pair != &this->free_pair) {
+            mem_put(pair);
+        }
+    }
+
+    if (data)
+        data_destroy(data);
+
+    gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_DICT_SET_FAILED, "key=%s", key,
+            NULL);
+
+    return ret;
 }
 
+/*
+ * Todo:
+ * Add below primitives as needed:
+ * dict_check_flags(this, key, flag...): variadic function to check
+ *                                       multiple flags at a time.
+ * dict_set_flags(this, key, flag...): set multiple flags
+ * dict_clear_flags(this, key, flag...): reset multiple flags
+ */
+
 int
-dict_get_double (dict_t *this, char *key, double *val)
+dict_set_flag(dict_t *this, char *key, int flag)
 {
-	data_t *data = NULL;
-	int     ret  = 0;
+    return _dict_modify_flag(this, key, flag, DICT_FLAG_SET);
+}
 
-	if (!this || !key || !val) {
-		ret = -EINVAL;
-		goto err;
-	}
+int
+dict_clear_flag(dict_t *this, char *key, int flag)
+{
+    return _dict_modify_flag(this, key, flag, DICT_FLAG_CLEAR);
+}
+
+int
+dict_get_double(dict_t *this, char *key, double *val)
+{
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!val) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_DOUBLE, key, -EINVAL);
 
-	ret = _data_to_double (data, val);
+    ret = data_to_double_ptr(data, val);
 
 err:
-	if (data)
-		data_unref (data);
-	return ret;
+    if (data)
+        data_unref(data);
+    return ret;
 }
 
 int
-dict_set_double (dict_t *this, char *key, double val)
+dict_set_double(dict_t *this, char *key, double val)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_double(val);
+    int ret = 0;
 
-	data = data_from_double (val);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_set_static_ptr (dict_t *this, char *key, void *ptr)
+dict_set_static_ptr(dict_t *this, char *key, void *ptr)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_ptr_common(ptr, _gf_true);
+    int ret = 0;
 
-	data = data_from_static_ptr (ptr);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_set_dynptr (dict_t *this, char *key, void *ptr, size_t len)
+dict_set_dynptr(dict_t *this, char *key, void *ptr, size_t len)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = data_from_dynptr(ptr, len);
+    int ret = 0;
 
-	data = data_from_dynptr (ptr, len);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
 int
-dict_get_ptr (dict_t *this, char *key, void **ptr)
+dict_get_ptr(dict_t *this, char *key, void **ptr)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	if (!this || !key || !ptr) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!ptr) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret != 0) {
-		goto err;
-	}
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	ret = _data_to_ptr (data, ptr);
-	if (ret != 0) {
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, -EINVAL);
 
-err: 
-	if (data)
-		data_unref (data);
+    ret = data_to_ptr_common(data, ptr);
+    if (ret != 0) {
+        goto err;
+    }
 
-	return ret;
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
 }
 
 int
-dict_set_ptr (dict_t *this, char *key, void *ptr)
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
+
+    if (!ptr) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret != 0) {
+        goto err;
+    }
 
-	data = data_from_ptr (ptr);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, -EINVAL);
 
-	ret = dict_set (this, key, data);
+    *len = data->len;
+
+    ret = data_to_ptr_common(data, ptr);
+    if (ret != 0) {
+        goto err;
+    }
 
 err:
-	return ret;
+    if (data)
+        data_unref(data);
+
+    return ret;
 }
 
+/* Get string - with known key length */
+int
+dict_get_strn(dict_t *this, char *key, const int keylen, char **str)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!this || !key || !str) {
+        goto err;
+    }
+    ret = dict_get_with_refn(this, key, keylen, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, key, -EINVAL);
+
+    *str = data->data;
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
 
 int
-dict_get_str (dict_t *this, char *key, char **str)
+dict_get_str(dict_t *this, char *key, char **str)
 {
-	data_t * data = NULL;
-	int      ret  = -EINVAL;
+    data_t *data = NULL;
+    int ret = -EINVAL;
 
-	if (!this || !key || !str) {
-		goto err;
-	}
+    if (!str) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret < 0) {
-		goto err;
-	}
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_STR, key, -EINVAL);
 
-	if (!data || !data->data) {
-		goto err;
-	}
-	*str = data->data;
+    *str = data->data;
 
-err: 
-	if (data)
-		data_unref (data);
+err:
+    if (data)
+        data_unref(data);
 
-	return ret;
+    return ret;
 }
 
 int
-dict_set_str (dict_t *this, char *key, char *str)
+dict_set_str(dict_t *this, char *key, char *str)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = str_to_data(str);
+    int ret = 0;
 
-	data = str_to_data (str);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
+/* Set string - with known key length */
 int
-dict_set_dynstr (dict_t *this, char *key, char *str)
+dict_set_strn(dict_t *this, char *key, const int keylen, char *str)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	data = data_from_dynstr (str);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    data = str_to_data(str);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
-/*
-        for malloced strings we should do a free instead of GF_FREE
-*/
+/* Set string - with known key length and known value length */
 int
-dict_set_dynmstr (dict_t *this, char *key, char *str)
+dict_set_nstrn(dict_t *this, char *key, const int keylen, char *str,
+               const int vallen)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = strn_to_data(str, vallen);
+    int ret = 0;
 
-	data = data_from_dynmstr (str);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
 err:
-	return ret;
+    return ret;
 }
 
+int
+dict_set_dynstr_with_alloc(dict_t *this, char *key, const char *str)
+{
+    char *alloc_str = gf_strdup(str);
+    int ret = -1;
+
+    if (!alloc_str)
+        return ret;
+
+    ret = dict_set_dynstr(this, key, alloc_str);
+    if (ret == -EINVAL)
+        GF_FREE(alloc_str);
+
+    return ret;
+}
+
+int
+dict_set_dynstr(dict_t *this, char *key, char *str)
+{
+    const int keylen = strlen(key);
+    return dict_set_dynstrn(this, key, keylen, str);
+}
 
 int
-dict_get_bin (dict_t *this, char *key, void **bin)
+dict_set_dynstrn(dict_t *this, char *key, const int keylen, char *str)
 {
-	data_t * data = NULL;
-	int      ret  = -EINVAL;
+    data_t *data = data_from_dynstr(str);
+    int ret = 0;
+
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	if (!this || !key || !bin) {
-		goto err;
-	}
+    ret = dict_setn(this, key, keylen, data);
+    if (ret < 0)
+        data_destroy(data);
 
-	ret = dict_get_with_ref (this, key, &data);
-	if (ret < 0) {
-		goto err;
-	}
+err:
+    return ret;
+}
 
-	if (!data || !data->data) {
-		goto err;
-	}
-	*bin = data->data;
+/* This function is called only by the volgen for now.
+   Check how else you can handle it */
+int
+dict_set_option(dict_t *this, char *key, char *str)
+{
+    data_t *data = data_from_dynstr(str);
+    int ret = 0;
 
-err: 
-	if (data)
-		data_unref (data);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	return ret;
+    data->data_type = GF_DATA_TYPE_STR_OLD;
+    ret = dict_set(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+err:
+    return ret;
 }
 
+int
+dict_add_dynstr_with_alloc(dict_t *this, char *key, char *str)
+{
+    data_t *data = NULL;
+    int ret = 0;
+    char *alloc_str = gf_strdup(str);
+
+    if (!alloc_str)
+        goto out;
+
+    data = data_from_dynstr(alloc_str);
+    if (!data) {
+        GF_FREE(alloc_str);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = dict_add(this, key, data);
+    if (ret < 0)
+        data_destroy(data);
+
+out:
+    return ret;
+}
 
 int
-dict_set_bin (dict_t *this, char *key, void *ptr, size_t size)
+dict_get_bin(dict_t *this, char *key, void **bin)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!bin) {
+        goto err;
+    }
+
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_PTR, key, ret);
+
+    *bin = data->data;
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
+
+/********************************************************************
+ *
+ * dict_set_bin_common:
+ *      This is the common function to set key and its value in
+ *      dictionary. Flag(is_static) should be set appropriately based
+ *      on the type of memory type used for value(*ptr). If flag is set
+ *      to false value(*ptr) will be freed using GF_FREE() on destroy.
+ *
+ *******************************************************************/
+static int
+dict_set_bin_common(dict_t *this, char *key, void *ptr, size_t size,
+                    gf_boolean_t is_static, gf_dict_data_type_t type)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    data_t *data = NULL;
+    int ret = 0;
 
-	if (!ptr || (size < 0)) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!ptr || (size > DICT_KEY_VALUE_MAX_SIZE)) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	data = bin_to_data (ptr, size);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    data = bin_to_data(ptr, size);
+    if (!data) {
+        ret = -EINVAL;
+        goto err;
+    }
 
-	data->data = ptr;
-	data->len  = size;
-	data->is_static = 0;
+    data->is_static = is_static;
+    data->data_type = type;
 
-	ret = dict_set (this, key, data);
+    ret = dict_set(this, key, data);
+    if (ret < 0) {
+        /* don't free data->data, let callers handle it */
+        data->data = NULL;
+        data_destroy(data);
+    }
 
 err:
-	return ret;
+    return ret;
 }
 
+/********************************************************************
+ *
+ * dict_set_bin:
+ *      Set key and its value in the dictionary. This function should
+ *      be called if the value is stored in dynamic memory.
+ *
+ *******************************************************************/
+int
+dict_set_bin(dict_t *this, char *key, void *ptr, size_t size)
+{
+    return dict_set_bin_common(this, key, ptr, size, _gf_false,
+                               GF_DATA_TYPE_PTR);
+}
+
+/********************************************************************
+ *
+ * dict_set_static_bin:
+ *      Set key and its value in the dictionary. This function should
+ *      be called if the value is stored in static memory.
+ *
+ *******************************************************************/
+int
+dict_set_static_bin(dict_t *this, char *key, void *ptr, size_t size)
+{
+    return dict_set_bin_common(this, key, ptr, size, _gf_true,
+                               GF_DATA_TYPE_PTR);
+}
+
+/*  */
+int
+dict_set_gfuuid(dict_t *this, char *key, uuid_t gfid, bool is_static)
+{
+    return dict_set_bin_common(this, key, gfid, sizeof(uuid_t), is_static,
+                               GF_DATA_TYPE_GFUUID);
+}
+
+int
+dict_get_gfuuid(dict_t *this, char *key, uuid_t *gfid)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!gfid) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_GFUUID, key, -EINVAL);
+
+    memcpy(*gfid, data->data, min(data->len, sizeof(uuid_t)));
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
 
 int
-dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size)
+dict_set_mdata(dict_t *this, char *key, struct mdata_iatt *mdata,
+               bool is_static)
 {
-	data_t * data = NULL;
-	int      ret  = 0;
+    return dict_set_bin_common(this, key, mdata, sizeof(struct mdata_iatt),
+                               is_static, GF_DATA_TYPE_MDATA);
+}
 
-	if (!ptr || (size < 0)) {
-		ret = -EINVAL;
-		goto err;
-	}
+int
+dict_get_mdata(dict_t *this, char *key, struct mdata_iatt *mdata)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
 
-	data = bin_to_data (ptr, size);
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
+    if (!mdata) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
 
-	data->data = ptr;
-	data->len  = size;
-	data->is_static = 1;
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_MDATA, key, -EINVAL);
+    if (data->len < sizeof(struct mdata_iatt)) {
+        gf_smsg("glusterfs", GF_LOG_ERROR, ENOBUFS, LG_MSG_UNDERSIZED_BUF,
+                "key=%s", key, NULL);
+        ret = -ENOBUFS;
+        goto err;
+    }
 
-	ret = dict_set (this, key, data);
+    memcpy(mdata, data->data, min(data->len, sizeof(struct mdata_iatt)));
 
 err:
-	return ret;
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
+
+int
+dict_set_iatt(dict_t *this, char *key, struct iatt *iatt, bool is_static)
+{
+    return dict_set_bin_common(this, key, iatt, sizeof(struct iatt), is_static,
+                               GF_DATA_TYPE_IATT);
 }
 
+int
+dict_get_iatt(dict_t *this, char *key, struct iatt *iatt)
+{
+    data_t *data = NULL;
+    int ret = -EINVAL;
+
+    if (!iatt) {
+        goto err;
+    }
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        goto err;
+    }
+
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_IATT, key, -EINVAL);
+
+    memcpy(iatt, data->data, min(data->len, sizeof(struct iatt)));
+
+err:
+    if (data)
+        data_unref(data);
+
+    return ret;
+}
 
 /**
  * dict_get_str_boolean - get a boolean value based on string representation.
@@ -2198,41 +2806,77 @@ err:
  */
 
 int
-dict_get_str_boolean (dict_t *this, char *key, int default_val)
+dict_get_str_boolean(dict_t *this, char *key, int default_val)
 {
-        data_t       *data = NULL;
-        gf_boolean_t  boo = _gf_false;
-        int           ret  = 0;
-
-        ret = dict_get_with_ref (this, key, &data);
-        if (ret < 0) {
-                if (ret == -ENOENT)
-                        ret = default_val;
-                else
-                        ret = -1;
-                goto err;
-        }
+    data_t *data = NULL;
+    gf_boolean_t boo = _gf_false;
+    int ret = 0;
 
-        GF_ASSERT (data);
+    ret = dict_get_with_ref(this, key, &data);
+    if (ret < 0) {
+        if (ret == -ENOENT)
+            ret = default_val;
+        else
+            ret = -1;
+        goto err;
+    }
 
-        if (!data->data) {
-                ret = -1;
-                goto err;
-        }
+    VALIDATE_DATA_AND_LOG(data, GF_DATA_TYPE_INT, key, -EINVAL);
 
-        ret = gf_string2boolean (data->data, &boo);
-        if (ret == -1)
-                goto err;
+    ret = gf_strn2boolean(data->data, data->len - 1, &boo);
+    if (ret == -1)
+        goto err;
 
-        ret = boo;
+    ret = boo;
 
 err:
-        if (data)
-                data_unref (data);
+    if (data)
+        data_unref(data);
 
-        return ret;
+    return ret;
 }
 
+int
+dict_rename_key(dict_t *this, char *key, char *replace_key)
+{
+    data_pair_t *pair = NULL;
+    int ret = -EINVAL;
+    uint32_t hash;
+    uint32_t replacekey_hash;
+    int replacekey_len;
+
+    /* replacing a key by itself is a NO-OP */
+    if (strcmp(key, replace_key) == 0)
+        return 0;
+
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        return ret;
+    }
+
+    hash = (uint32_t)XXH64(key, strlen(key), 0);
+    replacekey_len = strlen(replace_key);
+    replacekey_hash = (uint32_t)XXH64(replace_key, replacekey_len, 0);
+
+    LOCK(&this->lock);
+    {
+        /* no need to data_ref(pair->value), dict_set_lk() does it */
+        pair = dict_lookup_common(this, key, hash);
+        if (!pair)
+            ret = -ENODATA;
+        else
+            ret = dict_set_lk(this, replace_key, replacekey_len, pair->value,
+                              replacekey_hash, 1);
+    }
+    UNLOCK(&this->lock);
+
+    if (!ret)
+        /* only delete the key on success */
+        dict_del(this, key);
+
+    return ret;
+}
 
 /**
  * Serialization format:
@@ -2242,13 +2886,9 @@ err:
  *     4        4         4       <key len>   <value len>
  */
 
-#define DICT_HDR_LEN               4
-#define DICT_DATA_HDR_KEY_LEN      4
-#define DICT_DATA_HDR_VAL_LEN      4
-
 /**
- * _dict_serialized_length - return the length of serialized dict. This
- *                           procedure has to be called with this->lock held.
+ * dict_serialized_length_lk - return the length of serialized dict. This
+ *                             procedure has to be called with this->lock held.
  *
  * @this  : dict to be serialized
  * @return: success: len
@@ -2256,167 +2896,103 @@ err:
  */
 
 int
-_dict_serialized_length (dict_t *this)
-{
-	int ret            = -EINVAL;
-	int count          = 0;
-	int len            = 0;
-	int i              = 0;
-	data_pair_t * pair = NULL;
-
-	len = DICT_HDR_LEN;
-	count = this->count;
-
-	if (count < 0) {
-		gf_log ("dict", GF_LOG_ERROR, "count (%d) < 0!", count);
-		goto out;
-	}
-
-	pair = this->members_list;
-
-	while (count) {
-		if (!pair) {
-			gf_log ("dict", GF_LOG_ERROR, 
-				"less than count data pairs found!");
-			goto out;
-		}
-
-		len += DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN;
-
-		if (!pair->key) {
-			gf_log ("dict", GF_LOG_ERROR, "pair->key is null!");
-			goto out;
-		}
-
-		len += strlen (pair->key) + 1  /* for '\0' */;
-
-		if (!pair->value) {
-			gf_log ("dict", GF_LOG_ERROR,
-				"pair->value is null!");
-			goto out;
-		}
-
-		if (pair->value->vec) {
-			for (i = 0; i < pair->value->len; i++) {
-				if (pair->value->vec[i].iov_len < 0) {
-					gf_log ("dict", GF_LOG_ERROR,
-						"iov_len (%"GF_PRI_SIZET") < 0!",
-						pair->value->vec[i].iov_len);
-					goto out;
-				}
-
-				len += pair->value->vec[i].iov_len;
-			}
-		} else {
-			if (pair->value->len < 0) {
-				gf_log ("dict", GF_LOG_ERROR,
-					"value->len (%d) < 0",
-					pair->value->len);
-				goto out;
-			}
-
-			len += pair->value->len;
-		}
-
-		pair = pair->next;
-		count--;
-	}
-	
-	ret = len;
+dict_serialized_length_lk(dict_t *this)
+{
+    int ret = -EINVAL;
+    int count = this->count;
+    const int keyhdrlen = DICT_DATA_HDR_KEY_LEN + DICT_DATA_HDR_VAL_LEN;
+
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
+
+    ret = DICT_HDR_LEN + this->totkvlen + (count * keyhdrlen);
 out:
-	return ret;
+    return ret;
 }
 
 /**
- * _dict_serialize - serialize a dictionary into a buffer. This procedure has 
- *                   to be called with this->lock held.
+ * dict_serialize_lk - serialize a dictionary into a buffer. This procedure has
+ *                     to be called with this->lock held.
  *
  * @this: dict to serialize
- * @buf:  buffer to serialize into. This must be 
- *        atleast dict_serialized_length (this) large
+ * @buf:  buffer to serialize into. This must be
+ *        at least dict_serialized_length (this) large
  *
  * @return: success: 0
  *          failure: -errno
  */
 
-int
-_dict_serialize (dict_t *this, char *buf)
-{
-	int           ret     = -1;
-	data_pair_t * pair    = NULL;
-	int32_t       count   = 0;
-	int32_t       keylen  = 0;
-	int32_t       vallen  = 0;
-	int32_t       netword = 0;
-	
-	if (!buf) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"buf is null!");
-		goto out;
-	}
-	
-	count = this->count;
-	if (count < 0) {
-		gf_log ("dict", GF_LOG_ERROR, "count (%d) < 0!", count);
-		goto out;
-	}
-
-	netword = hton32 (count);
-	memcpy (buf, &netword, sizeof(netword));
-	buf += DICT_HDR_LEN;
-	pair = this->members_list;
-
-	while (count) {
-		if (!pair) {
-			gf_log ("dict", GF_LOG_ERROR,
-				"less than count data pairs found!");
-			goto out;
-		}
-
-		if (!pair->key) {
-			gf_log ("dict", GF_LOG_ERROR,
-				"pair->key is null!");
-			goto out;
-		}
-
-		keylen  = strlen (pair->key);
-		netword = hton32 (keylen);
-		memcpy (buf, &netword, sizeof(netword));
-		buf += DICT_DATA_HDR_KEY_LEN;
-
-		if (!pair->value) {
-			gf_log ("dict", GF_LOG_ERROR,
-				"pair->value is null!");
-			goto out;
-		}
-
-		vallen  = pair->value->len;
-		netword = hton32 (vallen);
-		memcpy (buf, &netword, sizeof(netword));
-		buf += DICT_DATA_HDR_VAL_LEN;
-
-		memcpy (buf, pair->key, keylen);
-		buf += keylen;
-		*buf++ = '\0';
-
-		if (!pair->value->data) {
-			gf_log ("dict", GF_LOG_ERROR,
-				"pair->value->data is null!");
-			goto out;
-		}
-		memcpy (buf, pair->value->data, vallen);
-		buf += vallen;
-
-		pair = pair->next;
-		count--;
-	}
-
-	ret = 0;
+static int
+dict_serialize_lk(dict_t *this, char *buf)
+{
+    int ret = -1;
+    data_pair_t *pair = this->members_list;
+    int32_t count = this->count;
+    int32_t keylen = 0;
+    int32_t netword = 0;
+
+    if (!buf) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, NULL);
+        goto out;
+    }
+
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
+
+    netword = hton32(count);
+    memcpy(buf, &netword, sizeof(netword));
+    buf += DICT_HDR_LEN;
+
+    while (count) {
+        if (!pair) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_PAIRS_LESS_THAN_COUNT,
+                    NULL);
+            goto out;
+        }
+
+        if (!pair->key) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR, NULL);
+            goto out;
+        }
+
+        keylen = strlen(pair->key);
+        netword = hton32(keylen);
+        memcpy(buf, &netword, sizeof(netword));
+        buf += DICT_DATA_HDR_KEY_LEN;
+
+        if (!pair->value) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR, NULL);
+            goto out;
+        }
+
+        netword = hton32(pair->value->len);
+        memcpy(buf, &netword, sizeof(netword));
+        buf += DICT_DATA_HDR_VAL_LEN;
+
+        memcpy(buf, pair->key, keylen);
+        buf += keylen;
+        *buf++ = '\0';
+
+        if (pair->value->data) {
+            memcpy(buf, pair->value->data, pair->value->len);
+            buf += pair->value->len;
+        }
+
+        pair = pair->next;
+        count--;
+    }
+
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
-
 /**
  * dict_serialized_length - return the length of serialized dict
  *
@@ -2426,245 +3002,488 @@ out:
  */
 
 int
-dict_serialized_length (dict_t *this)
-{
-	int ret            = -EINVAL;
-
-	if (!this) {
-		gf_log ("dict", GF_LOG_ERROR, "this is null!");
-		goto out;
-	}
-	
-        LOCK (&this->lock);
-        {
-                ret = _dict_serialized_length (this);
-        }
-        UNLOCK (&this->lock);
+dict_serialized_length(dict_t *this)
+{
+    int ret = -EINVAL;
+
+    if (!this) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
+
+    LOCK(&this->lock);
+    {
+        ret = dict_serialized_length_lk(this);
+    }
+    UNLOCK(&this->lock);
 
 out:
-	return ret;
+    return ret;
 }
 
 /**
  * dict_serialize - serialize a dictionary into a buffer
  *
  * @this: dict to serialize
- * @buf:  buffer to serialize into. This must be 
- *        atleast dict_serialized_length (this) large
+ * @buf:  buffer to serialize into. This must be
+ *        at least dict_serialized_length (this) large
  *
  * @return: success: 0
  *          failure: -errno
  */
 
 int
-dict_serialize (dict_t *this, char *buf)
-{
-	int           ret    = -1;
-	
-	if (!this) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"this is null!");
-		goto out;
-	}
-	if (!buf) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"buf is null!");
-		goto out;
-	}
-
-        LOCK (&this->lock);
-        {
-                ret = _dict_serialize (this, buf);
-        }
-        UNLOCK (&this->lock);
+dict_serialize(dict_t *this, char *buf)
+{
+    int ret = -1;
+
+    if (!this || !buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
+
+    LOCK(&this->lock);
+    {
+        ret = dict_serialize_lk(this, buf);
+    }
+    UNLOCK(&this->lock);
 out:
-	return ret;
+    return ret;
 }
 
-
 /**
  * dict_unserialize - unserialize a buffer into a dict
  *
  * @buf:  buf containing serialized dict
  * @size: size of the @buf
  * @fill: dict to fill in
- * 
+ *
  * @return: success: 0
  *          failure: -errno
  */
 
 int32_t
-dict_unserialize (char *orig_buf, int32_t size, dict_t **fill)
-{
-	char   *buf = NULL;
-	int     ret   = -1;
-	int32_t count = 0;
-	int     i     = 0;
-
-	data_t * value   = NULL;
-	char   * key     = NULL;
-	int32_t  keylen  = 0;
-	int32_t  vallen  = 0;
-	int32_t  hostord = 0;
-
-	buf = orig_buf;
-
-	if (!buf) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"buf is null!");
-		goto out;
-	}
-
-	if (size == 0) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"size is 0!");
-		goto out;
-	}
-
-	if (!fill) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"fill is null!");
-		goto out;
-	}
-
-	if (!*fill) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"*fill is null!");
-		goto out;
-	}
-
-	if ((buf + DICT_HDR_LEN) > (orig_buf + size)) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"undersized buffer passed");
-		goto out;
-	}
-
-	memcpy (&hostord, buf, sizeof(hostord));
-	count = ntoh32 (hostord);
-	buf += DICT_HDR_LEN;
-
-	if (count < 0) {
-		gf_log ("dict", GF_LOG_ERROR,
-			"count (%d) <= 0", count);
-		goto out;
-	}
-
-	/* count will be set by the dict_set's below */
-	(*fill)->count = 0;
-
-	for (i = 0; i < count; i++) {
-		if ((buf + DICT_DATA_HDR_KEY_LEN) > (orig_buf + size)) {
-			gf_log ("dict", GF_LOG_DEBUG,
-				"No room for keylen (size %d).",
-				DICT_DATA_HDR_KEY_LEN);
-			gf_log ("dict", GF_LOG_ERROR,
-				"undersized buffer passed");
-			goto out;
-		}
-		memcpy (&hostord, buf, sizeof(hostord));
-		keylen = ntoh32 (hostord);
-		buf += DICT_DATA_HDR_KEY_LEN;
-
-		if ((buf + DICT_DATA_HDR_VAL_LEN) > (orig_buf + size)) {
-			gf_log ("dict", GF_LOG_DEBUG,
-				"No room for vallen (size %d).",
-				DICT_DATA_HDR_VAL_LEN);
-			gf_log ("dict", GF_LOG_ERROR,
-				"undersized buffer passed");
-			goto out;
-		}
-		memcpy (&hostord, buf, sizeof(hostord));
-		vallen = ntoh32 (hostord);
-		buf += DICT_DATA_HDR_VAL_LEN;
-
-		if ((buf + keylen) > (orig_buf + size)) {
-			gf_log ("dict", GF_LOG_DEBUG,
-				"No room for key (size %d).", keylen);
-			gf_log ("dict", GF_LOG_ERROR,
-				"undersized buffer passed");
-			goto out;
-		}
-		key = buf;
-		buf += keylen + 1;  /* for '\0' */
-
-		if ((buf + vallen) > (orig_buf + size)) {
-			gf_log ("dict", GF_LOG_DEBUG,
-				"No room for value (size %d).", vallen);
-			gf_log ("dict", GF_LOG_ERROR,
-				"undersized buffer passed");
-			goto out;
-		}
-		value = get_new_data ();
-		value->len  = vallen;
-		value->data = buf;
-		value->is_static = 1;
-		buf += vallen;
-
-		dict_set (*fill, key, value);
-	}
-
-	ret = 0;
+dict_unserialize(char *orig_buf, int32_t size, dict_t **fill)
+{
+    char *buf = orig_buf;
+    int ret = -1;
+    int32_t count = 0;
+    int i = 0;
+
+    data_t *value = NULL;
+    char *key = NULL;
+    int32_t keylen = 0;
+    int32_t vallen = 0;
+    int32_t hostord = 0;
+
+    if (!buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "buf is null!");
+        goto out;
+    }
+
+    if (size == 0) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "size is 0!");
+        goto out;
+    }
+
+    if (!fill) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "fill is null!");
+        goto out;
+    }
+
+    if (!*fill) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "*fill is null!");
+        goto out;
+    }
+
+    if ((buf + DICT_HDR_LEN) > (orig_buf + size)) {
+        gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                         "undersized buffer "
+                         "passed. available (%lu) < required (%lu)",
+                         (long)(orig_buf + size), (long)(buf + DICT_HDR_LEN));
+        goto out;
+    }
+
+    memcpy(&hostord, buf, sizeof(hostord));
+    count = ntoh32(hostord);
+    buf += DICT_HDR_LEN;
+
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_COUNT_LESS_THAN_ZERO,
+                "count=%d", count, NULL);
+        goto out;
+    }
+
+    /* count will be set by the dict_set's below */
+    (*fill)->count = 0;
+
+    for (i = 0; i < count; i++) {
+        if ((buf + DICT_DATA_HDR_KEY_LEN) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized "
+                             "buffer passed. available (%lu) < "
+                             "required (%lu)",
+                             (long)(orig_buf + size),
+                             (long)(buf + DICT_DATA_HDR_KEY_LEN));
+            goto out;
+        }
+        memcpy(&hostord, buf, sizeof(hostord));
+        keylen = ntoh32(hostord);
+        buf += DICT_DATA_HDR_KEY_LEN;
+
+        if ((buf + DICT_DATA_HDR_VAL_LEN) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized "
+                             "buffer passed. available (%lu) < "
+                             "required (%lu)",
+                             (long)(orig_buf + size),
+                             (long)(buf + DICT_DATA_HDR_VAL_LEN));
+            goto out;
+        }
+        memcpy(&hostord, buf, sizeof(hostord));
+        vallen = ntoh32(hostord);
+        buf += DICT_DATA_HDR_VAL_LEN;
+
+        if ((keylen < 0) || (vallen < 0)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized length passed "
+                             "key:%d val:%d",
+                             keylen, vallen);
+            goto out;
+        }
+        if ((buf + keylen) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized buffer passed. "
+                             "available (%lu) < required (%lu)",
+                             (long)(orig_buf + size), (long)(buf + keylen));
+            goto out;
+        }
+        key = buf;
+        buf += keylen + 1; /* for '\0' */
+
+        if ((buf + vallen) > (orig_buf + size)) {
+            gf_msg_callingfn("dict", GF_LOG_ERROR, 0, LG_MSG_UNDERSIZED_BUF,
+                             "undersized buffer passed. "
+                             "available (%lu) < required (%lu)",
+                             (long)(orig_buf + size), (long)(buf + vallen));
+            goto out;
+        }
+        value = get_new_data();
+
+        if (!value) {
+            ret = -1;
+            goto out;
+        }
+        value->len = vallen;
+        value->data = gf_memdup(buf, vallen);
+        value->data_type = GF_DATA_TYPE_STR_OLD;
+        value->is_static = _gf_false;
+        buf += vallen;
+
+        ret = dict_addn(*fill, key, keylen, value);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
-
 /**
  * dict_allocate_and_serialize - serialize a dictionary into an allocated buffer
  *
  * @this: dict to serialize
  * @buf:  pointer to pointer to character. The allocated buffer is stored in
- *        this pointer. The buffer has to be freed by the caller. 
+ *        this pointer. The buffer has to be freed by the caller.
  *
  * @return: success: 0
  *          failure: -errno
  */
 
 int32_t
-dict_allocate_and_serialize (dict_t *this, char **buf, size_t *length)
-{
-	int           ret    = -EINVAL;
-        ssize_t       len = 0; 
-
-	if (!this) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"NULL passed as this pointer");
-		goto out;
-	}
-	if (!buf) {
-		gf_log ("dict", GF_LOG_DEBUG,
-			"NULL passed as buf");
-		goto out;
-	}
-	
-        LOCK (&this->lock);
-        { 
-                len = _dict_serialized_length (this);
-                if (len < 0) {
-                        ret = len;
-                        goto unlock;
-                }
+dict_allocate_and_serialize(dict_t *this, char **buf, u_int *length)
+{
+    int ret = -EINVAL;
+    ssize_t len = 0;
+
+    if (!this || !buf) {
+        gf_msg_debug("dict", 0, "dict OR buf is NULL");
+        goto out;
+    }
+
+    LOCK(&this->lock);
+    {
+        len = dict_serialized_length_lk(this);
+        if (len < 0) {
+            ret = len;
+            goto unlock;
+        }
 
-                *buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (*buf == NULL) {
-                        ret = -ENOMEM;
-                        gf_log ("dict", GF_LOG_ERROR, "out of memory");
-                        goto unlock;
-                }
+        *buf = GF_MALLOC(len, gf_common_mt_char);
+        if (*buf == NULL) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
 
-                ret = _dict_serialize (this, *buf);
-                if (ret < 0) {
-                        GF_FREE (*buf);
-                        *buf = NULL;
-                        goto unlock;
-                }
+        ret = dict_serialize_lk(this, *buf);
+        if (ret < 0) {
+            GF_FREE(*buf);
+            *buf = NULL;
+            goto unlock;
+        }
 
-                if (length != NULL) {
-                        *length = len;
-                }
+        if (length != NULL) {
+            *length = len;
         }
+    }
 unlock:
-        UNLOCK (&this->lock);
+    UNLOCK(&this->lock);
+out:
+    return ret;
+}
+
+/**
+ * dict_serialize_value_with_delim_lk: serialize the values in the dictionary
+ * into a buffer separated by delimiter (except the last)
+ *
+ * @this      : dictionary to serialize
+ * @buf       : the buffer to store the serialized data
+ * @serz_len  : the length of the serialized data (excluding the last delimiter)
+ * @delimiter : the delimiter to separate the values
+ *
+ * @return    : 0 -> success
+ *            : -errno -> failure
+ */
+int
+dict_serialize_value_with_delim_lk(dict_t *this, char *buf, int32_t *serz_len,
+                                   char delimiter)
+{
+    int ret = -1;
+    int32_t count = this->count;
+    int32_t vallen = 0;
+    int32_t total_len = 0;
+    data_pair_t *pair = this->members_list;
+
+    if (!buf) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, NULL);
+        goto out;
+    }
+
+    if (count < 0) {
+        gf_smsg("dict", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG, "count=%d",
+                count, NULL);
+        goto out;
+    }
+
+    while (count) {
+        if (!pair) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_PAIRS_LESS_THAN_COUNT,
+                    NULL);
+            goto out;
+        }
+
+        if (!pair->key || !pair->value) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_KEY_OR_VALUE_NULL, NULL);
+            goto out;
+        }
+
+        if (!pair->value->data) {
+            gf_smsg("dict", GF_LOG_ERROR, 0, LG_MSG_NULL_VALUE_IN_DICT, NULL);
+            goto out;
+        }
+
+        vallen = pair->value->len - 1;  // length includes \0
+        memcpy(buf, pair->value->data, vallen);
+        buf += vallen;
+        *buf++ = delimiter;
+
+        total_len += (vallen + 1);
+
+        pair = pair->next;
+        count--;
+    }
+
+    *--buf = '\0';  // remove the last delimiter
+    total_len--;    // adjust the length
+    ret = 0;
+
+    if (serz_len)
+        *serz_len = total_len;
+
+out:
+    return ret;
+}
+
+int
+dict_serialize_value_with_delim(dict_t *this, char *buf, int32_t *serz_len,
+                                char delimiter)
+{
+    int ret = -1;
+
+    if (!this || !buf) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is null!");
+        goto out;
+    }
+
+    LOCK(&this->lock);
+    {
+        ret = dict_serialize_value_with_delim_lk(this, buf, serz_len,
+                                                 delimiter);
+    }
+    UNLOCK(&this->lock);
+out:
+    return ret;
+}
+
+int
+dict_dump_to_str(dict_t *dict, char *dump, int dumpsize, char *format)
+{
+    int ret = 0;
+    int dumplen = 0;
+    data_pair_t *trav = NULL;
+
+    if (!dict)
+        return 0;
+
+    for (trav = dict->members_list; trav; trav = trav->next) {
+        ret = snprintf(&dump[dumplen], dumpsize - dumplen, format, trav->key,
+                       trav->value->data);
+        if ((ret == -1) || !ret)
+            return ret;
+
+        dumplen += ret;
+    }
+    return 0;
+}
+
+void
+dict_dump_to_log(dict_t *dict)
+{
+    int ret = -1;
+    char *dump = NULL;
+    const int dump_size = 64 * 1024;
+    char *format = "(%s:%s)";
+
+    if (!dict) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+
+    dump = GF_MALLOC(dump_size, gf_common_mt_char);
+    if (!dump) {
+        gf_msg_callingfn("dict", GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                         "dump buffer is NULL");
+        goto out;
+    }
+
+    ret = dict_dump_to_str(dict, dump, dump_size, format);
+    if (ret) {
+        gf_smsg("dict", GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT, NULL);
+        goto out;
+    }
+    gf_smsg("dict", GF_LOG_INFO, 0, LG_MSG_DICT_ERROR, "dict=%p", dict,
+            "dump=%s", dump, NULL);
+out:
+    GF_FREE(dump);
+
+    return;
+}
+
+void
+dict_dump_to_statedump(dict_t *dict, char *dict_name, char *domain)
+{
+    int ret = -1;
+    char *dump = NULL;
+    const int dump_size = 64 * 1024;
+    char key[4096] = {
+        0,
+    };
+    char *format = "\n\t%s:%s";
+
+    if (!dict) {
+        gf_msg_callingfn(domain, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "dict is NULL");
+        goto out;
+    }
+
+    dump = GF_MALLOC(dump_size, gf_common_mt_char);
+    if (!dump) {
+        gf_msg_callingfn(domain, GF_LOG_WARNING, ENOMEM, LG_MSG_NO_MEMORY,
+                         "dump buffer is NULL");
+        goto out;
+    }
+
+    ret = dict_dump_to_str(dict, dump, dump_size, format);
+    if (ret) {
+        gf_smsg(domain, GF_LOG_WARNING, 0, LG_MSG_FAILED_TO_LOG_DICT, "name=%s",
+                dict_name, NULL);
+        goto out;
+    }
+    gf_proc_dump_build_key(key, domain, "%s", dict_name);
+    gf_proc_dump_write(key, "%s", dump);
+
 out:
-	return ret;
+    GF_FREE(dump);
+
+    return;
+}
+
+dict_t *
+dict_for_key_value(const char *name, const char *value, size_t size,
+                   gf_boolean_t is_static)
+{
+    dict_t *xattr = dict_new();
+    int ret = 0;
+
+    if (!xattr)
+        return NULL;
+
+    if (is_static)
+        ret = dict_set_static_bin(xattr, (char *)name, (void *)value, size);
+    else
+        ret = dict_set_bin(xattr, (char *)name, (void *)value, size);
+
+    if (ret) {
+        dict_destroy(xattr);
+        xattr = NULL;
+    }
+
+    return xattr;
+}
+
+/*
+ * "strings" should be NULL terminated strings array.
+ */
+int
+dict_has_key_from_array(dict_t *dict, char **strings, gf_boolean_t *result)
+{
+    int i = 0;
+    uint32_t hash = 0;
+
+    if (!dict || !strings || !result)
+        return -EINVAL;
+
+    LOCK(&dict->lock);
+    {
+        for (i = 0; strings[i]; i++) {
+            hash = (uint32_t)XXH64(strings[i], strlen(strings[i]), 0);
+            if (dict_lookup_common(dict, strings[i], hash)) {
+                *result = _gf_true;
+                goto unlock;
+            }
+        }
+        *result = _gf_false;
+    }
+unlock:
+    UNLOCK(&dict->lock);
+    return 0;
 }
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
deleted file mode 100644
index 417967051bb..00000000000
--- a/libglusterfs/src/dict.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _DICT_H
-#define _DICT_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-#include <sys/uio.h>
-#include <pthread.h>
-
-#include "common-utils.h"
-
-typedef struct _data data_t;
-typedef struct _dict dict_t;
-typedef struct _data_pair data_pair_t;
-
-struct _data {
-  unsigned char is_static:1;
-  unsigned char is_const:1;
-  unsigned char is_stdalloc:1;
-  int32_t len;
-  struct iovec *vec;
-  char *data;
-  int32_t refcount;
-  gf_lock_t lock;
-};
-
-struct _data_pair {
-  struct _data_pair *hash_next;
-  struct _data_pair *prev;
-  struct _data_pair *next;
-  data_t *value;
-  char *key;
-};
-
-struct _dict {
-  unsigned char is_static:1;
-  int32_t hash_size;
-  int32_t count;
-  int32_t refcount;
-  data_pair_t **members;
-  data_pair_t *members_list;
-  char *extra_free;
-  char *extra_stdfree;
-  gf_lock_t lock;
-};
-
-
-int32_t is_data_equal (data_t *one, data_t *two);
-void data_destroy (data_t *data);
-
-int32_t dict_set (dict_t *this, char *key, data_t *value);
-data_t *dict_get (dict_t *this, char *key);
-void dict_del (dict_t *this, char *key);
-
-int32_t dict_serialized_length (dict_t *dict);
-int32_t dict_serialize (dict_t *dict, char *buf);
-int32_t dict_unserialize (char *buf, int32_t size, dict_t **fill);
-
-int32_t
-dict_allocate_and_serialize (dict_t *this, char **buf, size_t *length);
-
-int32_t dict_iovec_len (dict_t *dict);
-int32_t dict_to_iovec (dict_t *dict, struct iovec *vec, int32_t count);
-			  
-void dict_destroy (dict_t *dict);
-void dict_unref (dict_t *dict);
-dict_t *dict_ref (dict_t *dict);
-data_t *data_ref (data_t *data);
-void data_unref (data_t *data);
-
-/* 
-   TODO: provide converts for differnt byte sizes, signedness, and void *
- */
-data_t *int_to_data (int64_t value);
-data_t *str_to_data (char *value);
-data_t *data_from_dynstr (char *value);
-data_t *data_from_dynptr (void *value, int32_t len);
-data_t *bin_to_data (void *value, int32_t len);
-data_t *static_str_to_data (char *value);
-data_t *static_bin_to_data (void *value);
-
-int64_t data_to_int64 (data_t *data);
-int32_t data_to_int32 (data_t *data);
-int16_t data_to_int16 (data_t *data);
-int8_t data_to_int8 (data_t *data);
-
-uint64_t data_to_uint64 (data_t *data);
-uint32_t data_to_uint32 (data_t *data);
-uint16_t data_to_uint16 (data_t *data);
-
-data_t *data_from_ptr (void *value);
-data_t *data_from_static_ptr (void *value);
-
-data_t *data_from_int64 (int64_t value);
-data_t *data_from_int32 (int32_t value);
-data_t *data_from_int16 (int16_t value);
-data_t *data_from_int8 (int8_t value);
-
-data_t *data_from_uint64 (uint64_t value);
-data_t *data_from_uint32 (uint32_t value);
-data_t *data_from_uint16 (uint16_t value);
-
-char *data_to_str (data_t *data);
-void *data_to_bin (data_t *data);
-void *data_to_ptr (data_t *data);
-
-data_t *get_new_data ();
-dict_t *get_new_dict_full (int size_hint);
-dict_t *get_new_dict ();
-
-data_pair_t *get_new_data_pair ();
-
-void dict_foreach (dict_t *this,
-		   void (*fn)(dict_t *this,
-			      char *key,
-			      data_t *value,
-			      void *data),
-		   void *data);
-
-dict_t *dict_copy (dict_t *this,
-		   dict_t *new);
-
-/* CLEANED UP FUNCTIONS DECLARATIONS */
-GF_MUST_CHECK dict_t *dict_new (void);
-dict_t *dict_copy_with_ref (dict_t *this,
-			    dict_t *new);
-
-GF_MUST_CHECK int dict_get_int8 (dict_t *this, char *key, int8_t *val);
-GF_MUST_CHECK int dict_set_int8 (dict_t *this, char *key, int8_t val);
-
-GF_MUST_CHECK int dict_get_int16 (dict_t *this, char *key, int16_t *val);
-GF_MUST_CHECK int dict_set_int16 (dict_t *this, char *key, int16_t val);
-
-GF_MUST_CHECK int dict_get_int32 (dict_t *this, char *key, int32_t *val);
-GF_MUST_CHECK int dict_set_int32 (dict_t *this, char *key, int32_t val);
-
-GF_MUST_CHECK int dict_get_int64 (dict_t *this, char *key, int64_t *val);
-GF_MUST_CHECK int dict_set_int64 (dict_t *this, char *key, int64_t val);
-
-GF_MUST_CHECK int dict_get_uint16 (dict_t *this, char *key, uint16_t *val);
-GF_MUST_CHECK int dict_set_uint16 (dict_t *this, char *key, uint16_t val);
-
-GF_MUST_CHECK int dict_get_uint32 (dict_t *this, char *key, uint32_t *val);
-GF_MUST_CHECK int dict_set_uint32 (dict_t *this, char *key, uint32_t val);
-
-GF_MUST_CHECK int dict_get_uint64 (dict_t *this, char *key, uint64_t *val);
-GF_MUST_CHECK int dict_set_uint64 (dict_t *this, char *key, uint64_t val);
-
-GF_MUST_CHECK int dict_get_double (dict_t *this, char *key, double *val);
-GF_MUST_CHECK int dict_set_double (dict_t *this, char *key, double val);
-
-GF_MUST_CHECK int dict_set_static_ptr (dict_t *this, char *key, void *ptr);
-GF_MUST_CHECK int dict_get_ptr (dict_t *this, char *key, void **ptr);
-GF_MUST_CHECK int dict_set_ptr (dict_t *this, char *key, void *ptr);
-GF_MUST_CHECK int dict_set_dynptr (dict_t *this, char *key, void *ptr, size_t size);
-
-GF_MUST_CHECK int dict_get_bin (dict_t *this, char *key, void **ptr);
-GF_MUST_CHECK int dict_set_bin (dict_t *this, char *key, void *ptr, size_t size);
-GF_MUST_CHECK int dict_set_static_bin (dict_t *this, char *key, void *ptr, size_t size);
-
-GF_MUST_CHECK int dict_set_str (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_set_dynmstr (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_set_dynstr (dict_t *this, char *key, char *str);
-GF_MUST_CHECK int dict_get_str (dict_t *this, char *key, char **str);
-
-GF_MUST_CHECK int dict_get_str_boolean (dict_t *this, char *key, int default_val);
-#endif
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
new file mode 100644
index 00000000000..fb4fb845b40
--- /dev/null
+++ b/libglusterfs/src/event-epoll.c
@@ -0,0 +1,1032 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <pthread.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include "glusterfs/gf-event.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+#ifdef HAVE_SYS_EPOLL_H
+#include <sys/epoll.h>
+
+struct event_slot_epoll {
+    int fd;
+    int events;
+    int gen;
+    int idx;
+    gf_atomic_t ref;
+    int do_close;
+    int in_handler;
+    int handled_error;
+    void *data;
+    event_handler_t handler;
+    gf_lock_t lock;
+    struct list_head poller_death;
+};
+
+struct event_thread_data {
+    struct event_pool *event_pool;
+    int event_index;
+};
+
+static struct event_slot_epoll *
+__event_newtable(struct event_pool *event_pool, int table_idx)
+{
+    struct event_slot_epoll *table = NULL;
+    int i = -1;
+
+    table = GF_CALLOC(sizeof(*table), EVENT_EPOLL_SLOTS, gf_common_mt_ereg);
+    if (!table)
+        return NULL;
+
+    for (i = 0; i < EVENT_EPOLL_SLOTS; i++) {
+        table[i].fd = -1;
+        LOCK_INIT(&table[i].lock);
+        INIT_LIST_HEAD(&table[i].poller_death);
+    }
+
+    event_pool->ereg[table_idx] = table;
+    event_pool->slots_used[table_idx] = 0;
+
+    return table;
+}
+
+static int
+event_slot_ref(struct event_slot_epoll *slot)
+{
+    if (!slot)
+        return -1;
+
+    return GF_ATOMIC_INC(slot->ref);
+}
+
+static int
+__event_slot_alloc(struct event_pool *event_pool, int fd,
+                   char notify_poller_death, struct event_slot_epoll **slot)
+{
+    int i = 0;
+    int j = 0;
+    int table_idx = -1;
+    int gen = -1;
+    struct event_slot_epoll *table = NULL;
+
+retry:
+
+    while (i < EVENT_EPOLL_TABLES) {
+        switch (event_pool->slots_used[i]) {
+            case EVENT_EPOLL_SLOTS:
+                break;
+            case 0:
+                if (!event_pool->ereg[i]) {
+                    table = __event_newtable(event_pool, i);
+                    if (!table)
+                        return -1;
+                } else {
+                    table = event_pool->ereg[i];
+                }
+                break;
+            default:
+                table = event_pool->ereg[i];
+                break;
+        }
+
+        if (table)
+            /* break out of the loop */
+            break;
+        i++;
+    }
+
+    if (!table)
+        return -1;
+
+    table_idx = i;
+
+    for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
+        if (table[j].fd == -1) {
+            /* wipe everything except bump the generation */
+            gen = table[j].gen;
+            memset(&table[j], 0, sizeof(table[j]));
+            table[j].gen = gen + 1;
+
+            LOCK_INIT(&table[j].lock);
+            INIT_LIST_HEAD(&table[j].poller_death);
+
+            table[j].fd = fd;
+            if (notify_poller_death) {
+                table[j].idx = table_idx * EVENT_EPOLL_SLOTS + j;
+                list_add_tail(&table[j].poller_death,
+                              &event_pool->poller_death);
+            }
+
+            event_pool->slots_used[table_idx]++;
+
+            break;
+        }
+    }
+
+    if (j == EVENT_EPOLL_SLOTS) {
+        table = NULL;
+        i++;
+        goto retry;
+    } else {
+        (*slot) = &table[j];
+        event_slot_ref(*slot);
+        return table_idx * EVENT_EPOLL_SLOTS + j;
+    }
+}
+
+static int
+event_slot_alloc(struct event_pool *event_pool, int fd,
+                 char notify_poller_death, struct event_slot_epoll **slot)
+{
+    int idx = -1;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_slot_alloc(event_pool, fd, notify_poller_death, slot);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return idx;
+}
+
+static void
+__event_slot_dealloc(struct event_pool *event_pool, int idx)
+{
+    int table_idx = 0;
+    int offset = 0;
+    struct event_slot_epoll *table = NULL;
+    struct event_slot_epoll *slot = NULL;
+    int fd = -1;
+
+    table_idx = idx / EVENT_EPOLL_SLOTS;
+    offset = idx % EVENT_EPOLL_SLOTS;
+
+    table = event_pool->ereg[table_idx];
+    if (!table)
+        return;
+
+    slot = &table[offset];
+    slot->gen++;
+
+    fd = slot->fd;
+    slot->fd = -1;
+    slot->handled_error = 0;
+    slot->in_handler = 0;
+    list_del_init(&slot->poller_death);
+    if (fd != -1)
+        event_pool->slots_used[table_idx]--;
+
+    return;
+}
+
+static void
+event_slot_dealloc(struct event_pool *event_pool, int idx)
+{
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        __event_slot_dealloc(event_pool, idx);
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return;
+}
+
+static struct event_slot_epoll *
+event_slot_get(struct event_pool *event_pool, int idx)
+{
+    struct event_slot_epoll *slot = NULL;
+    struct event_slot_epoll *table = NULL;
+    int table_idx = 0;
+    int offset = 0;
+
+    table_idx = idx / EVENT_EPOLL_SLOTS;
+    offset = idx % EVENT_EPOLL_SLOTS;
+
+    table = event_pool->ereg[table_idx];
+    if (!table)
+        return NULL;
+
+    slot = &table[offset];
+
+    event_slot_ref(slot);
+    return slot;
+}
+
+static void
+__event_slot_unref(struct event_pool *event_pool, struct event_slot_epoll *slot,
+                   int idx)
+{
+    int ref = -1;
+    int fd = -1;
+    int do_close = 0;
+
+    ref = GF_ATOMIC_DEC(slot->ref);
+    if (ref)
+        /* slot still alive */
+        goto done;
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        do_close = slot->do_close;
+        slot->do_close = 0;
+    }
+    UNLOCK(&slot->lock);
+
+    __event_slot_dealloc(event_pool, idx);
+
+    if (do_close)
+        sys_close(fd);
+done:
+    return;
+}
+
+static void
+event_slot_unref(struct event_pool *event_pool, struct event_slot_epoll *slot,
+                 int idx)
+{
+    int ref = -1;
+    int fd = -1;
+    int do_close = 0;
+
+    ref = GF_ATOMIC_DEC(slot->ref);
+    if (ref)
+        /* slot still alive */
+        goto done;
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        do_close = slot->do_close;
+        slot->do_close = 0;
+    }
+    UNLOCK(&slot->lock);
+
+    event_slot_dealloc(event_pool, idx);
+
+    if (do_close)
+        sys_close(fd);
+done:
+    return;
+}
+
+static struct event_pool *
+event_pool_new_epoll(int count, int eventthreadcount)
+{
+    struct event_pool *event_pool = NULL;
+    int epfd = -1;
+
+    event_pool = GF_CALLOC(1, sizeof(*event_pool), gf_common_mt_event_pool);
+
+    if (!event_pool)
+        goto out;
+
+    epfd = epoll_create(count);
+
+    if (epfd == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_CREATE_FAILED,
+                NULL);
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        event_pool = NULL;
+        goto out;
+    }
+
+    event_pool->fd = epfd;
+
+    event_pool->count = count;
+    INIT_LIST_HEAD(&event_pool->poller_death);
+    event_pool->eventthreadcount = eventthreadcount;
+    event_pool->auto_thread_count = 0;
+
+    pthread_mutex_init(&event_pool->mutex, NULL);
+
+out:
+    return event_pool;
+}
+
+static void
+__slot_update_events(struct event_slot_epoll *slot, int poll_in, int poll_out)
+{
+    switch (poll_in) {
+        case 1:
+            slot->events |= EPOLLIN;
+            break;
+        case 0:
+            slot->events &= ~EPOLLIN;
+            break;
+        case -1:
+            /* do nothing */
+            break;
+        default:
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_IN,
+                    "value=%d", poll_in, NULL);
+            break;
+    }
+
+    switch (poll_out) {
+        case 1:
+            slot->events |= EPOLLOUT;
+            break;
+        case 0:
+            slot->events &= ~EPOLLOUT;
+            break;
+        case -1:
+            /* do nothing */
+            break;
+        default:
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_OUT,
+                    "value=%d", poll_out, NULL);
+            break;
+    }
+}
+
+int
+event_register_epoll(struct event_pool *event_pool, int fd,
+                     event_handler_t handler, void *data, int poll_in,
+                     int poll_out, char notify_poller_death)
+{
+    int idx = -1;
+    int ret = -1;
+    int destroy = 0;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+    struct event_slot_epoll *slot = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    /* TODO: Even with the below check, there is a possibility of race,
+     * What if the destroy mode is set after the check is done.
+     * Not sure of the best way to prevent this race, ref counting
+     * is one possibility.
+     * There is no harm in registering and unregistering the fd
+     * even after destroy mode is set, just that such fds will remain
+     * open until unregister is called, also the events on that fd will be
+     * notified, until one of the poller thread is alive.
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        destroy = event_pool->destroy;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    if (destroy == 1)
+        goto out;
+
+    idx = event_slot_alloc(event_pool, fd, notify_poller_death, &slot);
+    if (idx == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        /* make epoll 'singleshot', which
+           means we need to re-add the fd with
+           epoll_ctl(EPOLL_CTL_MOD) after delivery of every
+           single event. This assures us that while a poller
+           thread has picked up and is processing an event,
+           another poller will not try to pick this at the same
+           time as well.
+        */
+
+        slot->events = EPOLLPRI | EPOLLHUP | EPOLLERR | EPOLLONESHOT;
+        slot->handler = handler;
+        slot->data = data;
+
+        __slot_update_events(slot, poll_in, poll_out);
+
+        epoll_event.events = slot->events;
+        ev_data->idx = idx;
+        ev_data->gen = slot->gen;
+
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_ADD, fd, &epoll_event);
+        /* check ret after UNLOCK() to avoid deadlock in
+           event_slot_unref()
+        */
+    }
+    UNLOCK(&slot->lock);
+
+    if (ret == -1) {
+        gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_ADD_FAILED,
+                "fd=%d", fd, "epoll_fd=%d", event_pool->fd, NULL);
+        event_slot_unref(event_pool, slot, idx);
+        idx = -1;
+    }
+
+    /* keep slot->ref (do not event_slot_unref) if successful */
+out:
+    return idx;
+}
+
+static int
+event_unregister_epoll_common(struct event_pool *event_pool, int fd, int idx,
+                              int do_close)
+{
+    int ret = -1;
+    struct event_slot_epoll *slot = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    /* During shutdown, it may happen that a socket registration with
+     * the event sub-system may fail and an rpc_transport_unref() may
+     * be called for such an unregistered socket with idx == -1. This
+     * may cause the following assert(slot->fd == fd) to fail.
+     */
+    if (idx < 0)
+        goto out;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
+
+        if (ret == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_DEL_FAILED,
+                    "fd=%d", fd, "epoll_fd=%d", event_pool->fd, NULL);
+            goto unlock;
+        }
+
+        slot->do_close = do_close;
+        slot->gen++; /* detect unregister in dispatch_handler() */
+    }
+unlock:
+    UNLOCK(&slot->lock);
+
+    event_slot_unref(event_pool, slot, idx); /* one for event_register() */
+    event_slot_unref(event_pool, slot, idx); /* one for event_slot_get() */
+out:
+    return ret;
+}
+
+static int
+event_unregister_epoll(struct event_pool *event_pool, int fd, int idx_hint)
+{
+    int ret = -1;
+
+    ret = event_unregister_epoll_common(event_pool, fd, idx_hint, 0);
+
+    return ret;
+}
+
+static int
+event_unregister_close_epoll(struct event_pool *event_pool, int fd,
+                             int idx_hint)
+{
+    int ret = -1;
+
+    ret = event_unregister_epoll_common(event_pool, fd, idx_hint, 1);
+
+    return ret;
+}
+
+static int
+event_select_on_epoll(struct event_pool *event_pool, int fd, int idx,
+                      int poll_in, int poll_out)
+{
+    int ret = -1;
+    struct event_slot_epoll *slot = NULL;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        __slot_update_events(slot, poll_in, poll_out);
+
+        epoll_event.events = slot->events;
+        ev_data->idx = idx;
+        ev_data->gen = slot->gen;
+
+        if (slot->in_handler)
+            /*
+             * in_handler indicates at least one thread
+             * executing event_dispatch_epoll_handler()
+             * which will perform epoll_ctl(EPOLL_CTL_MOD)
+             * anyways (because of EPOLLET)
+             *
+             * This not only saves a system call, but also
+             * avoids possibility of another epoll thread
+             * picking up the next event while the ongoing
+             * handler is still in progress (and resulting
+             * in unnecessary contention on rpc_transport_t->mutex).
+             */
+            goto unlock;
+
+        ret = epoll_ctl(event_pool->fd, EPOLL_CTL_MOD, fd, &epoll_event);
+        if (ret == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, errno, LG_MSG_EPOLL_FD_MODIFY_FAILED,
+                    "fd=%d", fd, "events=%d", epoll_event.events, NULL);
+        }
+    }
+unlock:
+    UNLOCK(&slot->lock);
+
+    event_slot_unref(event_pool, slot, idx);
+
+out:
+    return idx;
+}
+
+static int
+event_dispatch_epoll_handler(struct event_pool *event_pool,
+                             struct epoll_event *event)
+{
+    struct event_data *ev_data = NULL;
+    struct event_slot_epoll *slot = NULL;
+    event_handler_t handler = NULL;
+    void *data = NULL;
+    int idx = -1;
+    int gen = -1;
+    int ret = -1;
+    int fd = -1;
+    gf_boolean_t handled_error_previously = _gf_false;
+
+    ev_data = (void *)&event->data;
+    handler = NULL;
+    data = NULL;
+
+    idx = ev_data->idx;
+    gen = ev_data->gen;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "idx=%d", idx,
+                NULL);
+        return -1;
+    }
+
+    LOCK(&slot->lock);
+    {
+        fd = slot->fd;
+        if (fd == -1) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_STALE_FD_FOUND, "idx=%d",
+                    idx, "gen=%d", gen, "events=%d", event->events,
+                    "slot->gen=%d", slot->gen, NULL);
+            /* fd got unregistered in another thread */
+            goto pre_unlock;
+        }
+
+        if (gen != slot->gen) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_GENERATION_MISMATCH,
+                    "idx=%d", idx, "gen=%d", gen, "slot->gen=%d", slot->gen,
+                    "slot->fd=%d", slot->fd, NULL);
+            /* slot was re-used and therefore is another fd! */
+            goto pre_unlock;
+        }
+
+        handler = slot->handler;
+        data = slot->data;
+
+        if (slot->in_handler > 0) {
+            /* Another handler is inprogress, skip this one. */
+            handler = NULL;
+            goto pre_unlock;
+        }
+
+        if (slot->handled_error) {
+            handled_error_previously = _gf_true;
+        } else {
+            slot->handled_error = (event->events & (EPOLLERR | EPOLLHUP));
+            slot->in_handler++;
+        }
+    }
+pre_unlock:
+    UNLOCK(&slot->lock);
+
+    ret = 0;
+
+    if (!handler)
+        goto out;
+
+    if (!handled_error_previously) {
+        handler(fd, idx, gen, data, (event->events & (EPOLLIN | EPOLLPRI)),
+                (event->events & (EPOLLOUT)),
+                (event->events & (EPOLLERR | EPOLLHUP)), 0);
+    }
+out:
+    event_slot_unref(event_pool, slot, idx);
+
+    return ret;
+}
+
+static void *
+event_dispatch_epoll_worker(void *data)
+{
+    struct epoll_event event;
+    int ret = -1;
+    struct event_thread_data *ev_data = data;
+    struct event_pool *event_pool;
+    int myindex = -1;
+    int timetodie = 0, gen = 0;
+    struct list_head poller_death_notify;
+    struct event_slot_epoll *slot = NULL, *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("event", ev_data, out);
+
+    event_pool = ev_data->event_pool;
+    myindex = ev_data->event_index;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    gf_smsg("epoll", GF_LOG_INFO, 0, LG_MSG_STARTED_EPOLL_THREAD, "index=%d",
+            myindex - 1, NULL);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount++;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    for (;;) {
+        if (event_pool->eventthreadcount < myindex) {
+            /* ...time to die, thread count was decreased below
+             * this threads index */
+            /* Start with extra safety at this point, reducing
+             * lock conention in normal case when threads are not
+             * reconfigured always */
+            pthread_mutex_lock(&event_pool->mutex);
+            {
+                if (event_pool->eventthreadcount < myindex) {
+                    while (event_pool->poller_death_sliced) {
+                        pthread_cond_wait(&event_pool->cond,
+                                          &event_pool->mutex);
+                    }
+
+                    INIT_LIST_HEAD(&poller_death_notify);
+                    /* if found true in critical section,
+                     * die */
+                    event_pool->pollers[myindex - 1] = 0;
+                    event_pool->activethreadcount--;
+                    timetodie = 1;
+                    gen = ++event_pool->poller_gen;
+                    list_for_each_entry(slot, &event_pool->poller_death,
+                                        poller_death)
+                    {
+                        event_slot_ref(slot);
+                    }
+
+                    list_splice_init(&event_pool->poller_death,
+                                     &poller_death_notify);
+                    event_pool->poller_death_sliced = 1;
+                    pthread_cond_broadcast(&event_pool->cond);
+                }
+            }
+            pthread_mutex_unlock(&event_pool->mutex);
+            if (timetodie) {
+                list_for_each_entry(slot, &poller_death_notify, poller_death)
+                {
+                    slot->handler(slot->fd, 0, gen, slot->data, 0, 0, 0, 1);
+                }
+
+                pthread_mutex_lock(&event_pool->mutex);
+                {
+                    list_for_each_entry_safe(slot, tmp, &poller_death_notify,
+                                             poller_death)
+                    {
+                        __event_slot_unref(event_pool, slot, slot->idx);
+                    }
+
+                    list_splice(&poller_death_notify,
+                                &event_pool->poller_death);
+                    event_pool->poller_death_sliced = 0;
+                    pthread_cond_broadcast(&event_pool->cond);
+                }
+                pthread_mutex_unlock(&event_pool->mutex);
+
+                gf_smsg("epoll", GF_LOG_INFO, 0, LG_MSG_EXITED_EPOLL_THREAD,
+                        "index=%d", myindex, NULL);
+
+                goto out;
+            }
+        }
+
+        ret = epoll_wait(event_pool->fd, &event, 1, -1);
+
+        if (ret == 0)
+            /* timeout */
+            continue;
+
+        if (ret == -1 && errno == EINTR)
+            /* sys call */
+            continue;
+
+        ret = event_dispatch_epoll_handler(event_pool, &event);
+        if (ret) {
+            gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_DISPATCH_HANDLER_FAILED,
+                    NULL);
+        }
+    }
+out:
+    if (ev_data)
+        GF_FREE(ev_data);
+    return NULL;
+}
+
+/* Attempts to start the # of configured pollers, ensuring at least the first
+ * is started in a joinable state */
+static int
+event_dispatch_epoll(struct event_pool *event_pool)
+{
+    int i = 0;
+    pthread_t t_id;
+    int pollercount = 0;
+    int ret = -1;
+    struct event_thread_data *ev_data = NULL;
+
+    /* Start the configured number of pollers */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        pollercount = event_pool->eventthreadcount;
+
+        /* Set to MAX if greater */
+        if (pollercount > EVENT_MAX_THREADS)
+            pollercount = EVENT_MAX_THREADS;
+
+        /* Default pollers to 1 in case this is incorrectly set */
+        if (pollercount <= 0)
+            pollercount = 1;
+
+        event_pool->activethreadcount++;
+
+        for (i = 0; i < pollercount; i++) {
+            ev_data = GF_CALLOC(1, sizeof(*ev_data), gf_common_mt_event_pool);
+            if (!ev_data) {
+                if (i == 0) {
+                    /* Need to succeed creating 0'th
+                     * thread, to joinable and wait */
+                    break;
+                } else {
+                    /* Inability to create other threads
+                     * are a lesser evil, and ignored */
+                    continue;
+                }
+            }
+
+            ev_data->event_pool = event_pool;
+            ev_data->event_index = i + 1;
+
+            ret = gf_thread_create(&t_id, NULL, event_dispatch_epoll_worker,
+                                   ev_data, "epoll%03hx", i & 0x3ff);
+            if (!ret) {
+                event_pool->pollers[i] = t_id;
+
+                /* mark all threads other than one in index 0
+                 * as detachable. Errors can be ignored, they
+                 * spend their time as zombies if not detched
+                 * and the thread counts are decreased */
+                if (i != 0)
+                    pthread_detach(event_pool->pollers[i]);
+            } else {
+                gf_smsg("epoll", GF_LOG_WARNING, 0,
+                        LG_MSG_START_EPOLL_THREAD_FAILED, "index=%d", i, NULL);
+                if (i == 0) {
+                    GF_FREE(ev_data);
+                    break;
+                } else {
+                    GF_FREE(ev_data);
+                    continue;
+                }
+            }
+        }
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    /* Just wait for the first thread, that is created in a joinable state
+     * and will never die, ensuring this function never returns */
+    if (event_pool->pollers[0] != 0)
+        pthread_join(event_pool->pollers[0], NULL);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount--;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return ret;
+}
+
+/**
+ * @param event_pool  event_pool on which fds of interest are registered for
+ *                     events.
+ *
+ * @return  1 if at least one epoll worker thread is spawned, 0 otherwise
+ *
+ * NB This function SHOULD be called under event_pool->mutex.
+ */
+
+static int
+event_pool_dispatched_unlocked(struct event_pool *event_pool)
+{
+    return (event_pool->pollers[0] != 0);
+}
+
+int
+event_reconfigure_threads_epoll(struct event_pool *event_pool, int value)
+{
+    int i;
+    int ret = 0;
+    pthread_t t_id;
+    int oldthreadcount;
+    struct event_thread_data *ev_data = NULL;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        /* Reconfigure to 0 threads is allowed only in destroy mode */
+        if (event_pool->destroy == 1) {
+            value = 0;
+        } else {
+            /* Set to MAX if greater */
+            if (value > EVENT_MAX_THREADS)
+                value = EVENT_MAX_THREADS;
+
+            /* Default pollers to 1 in case this is set incorrectly */
+            if (value <= 0)
+                value = 1;
+        }
+
+        oldthreadcount = event_pool->eventthreadcount;
+
+        /* Start 'worker' threads as necessary only if event_dispatch()
+         * was called before. If event_dispatch() was not called, there
+         * will be no epoll 'worker' threads running yet. */
+
+        if (event_pool_dispatched_unlocked(event_pool) &&
+            (oldthreadcount < value)) {
+            /* create more poll threads */
+            for (i = oldthreadcount; i < value; i++) {
+                /* Start a thread if the index at this location
+                 * is a 0, so that the older thread is confirmed
+                 * as dead */
+                if (event_pool->pollers[i] == 0) {
+                    ev_data = GF_CALLOC(1, sizeof(*ev_data),
+                                        gf_common_mt_event_pool);
+                    if (!ev_data) {
+                        continue;
+                    }
+
+                    ev_data->event_pool = event_pool;
+                    ev_data->event_index = i + 1;
+
+                    ret = gf_thread_create(&t_id, NULL,
+                                           event_dispatch_epoll_worker, ev_data,
+                                           "epoll%03hx", i & 0x3ff);
+                    if (ret) {
+                        gf_smsg("epoll", GF_LOG_WARNING, 0,
+                                LG_MSG_START_EPOLL_THREAD_FAILED, "index=%d", i,
+                                NULL);
+                        GF_FREE(ev_data);
+                    } else {
+                        pthread_detach(t_id);
+                        event_pool->pollers[i] = t_id;
+                    }
+                }
+            }
+        }
+
+        /* if value decreases, threads will terminate, themselves */
+        event_pool->eventthreadcount = value;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return 0;
+}
+
+/* This function is the destructor for the event_pool data structure
+ * Should be called only after poller_threads_destroy() is called,
+ * else will lead to crashes.
+ */
+static int
+event_pool_destroy_epoll(struct event_pool *event_pool)
+{
+    int ret = 0, i = 0, j = 0;
+    struct event_slot_epoll *table = NULL;
+
+    ret = sys_close(event_pool->fd);
+
+    for (i = 0; i < EVENT_EPOLL_TABLES; i++) {
+        if (event_pool->ereg[i]) {
+            table = event_pool->ereg[i];
+            event_pool->ereg[i] = NULL;
+            for (j = 0; j < EVENT_EPOLL_SLOTS; j++) {
+                LOCK_DESTROY(&table[j].lock);
+            }
+            GF_FREE(table);
+        }
+    }
+
+    pthread_mutex_destroy(&event_pool->mutex);
+    pthread_cond_destroy(&event_pool->cond);
+
+    GF_FREE(event_pool->evcache);
+    GF_FREE(event_pool->reg);
+    GF_FREE(event_pool);
+
+    return ret;
+}
+
+static int
+event_handled_epoll(struct event_pool *event_pool, int fd, int idx, int gen)
+{
+    struct event_slot_epoll *slot = NULL;
+    struct epoll_event epoll_event = {
+        0,
+    };
+    struct event_data *ev_data = (void *)&epoll_event.data;
+    int ret = 0;
+
+    slot = event_slot_get(event_pool, idx);
+    if (!slot) {
+        gf_smsg("epoll", GF_LOG_ERROR, 0, LG_MSG_SLOT_NOT_FOUND, "fd=%d", fd,
+                "idx=%d", idx, NULL);
+        return -1;
+    }
+
+    assert(slot->fd == fd);
+
+    LOCK(&slot->lock);
+    {
+        slot->in_handler--;
+
+        if (gen != slot->gen) {
+            /* event_unregister() happened while we were
+               in handler()
+            */
+            gf_msg_debug("epoll", 0,
+                         "generation bumped on idx=%d"
+                         " from gen=%d to slot->gen=%d, fd=%d, "
+                         "slot->fd=%d",
+                         idx, gen, slot->gen, fd, slot->fd);
+            goto unlock;
+        }
+
+        /* This call also picks up the changes made by another
+           thread calling event_select_on_epoll() while this
+           thread was busy in handler()
+        */
+        if (slot->in_handler == 0) {
+            epoll_event.events = slot->events;
+            ev_data->idx = idx;
+            ev_data->gen = gen;
+
+            ret = epoll_ctl(event_pool->fd, EPOLL_CTL_MOD, fd, &epoll_event);
+        }
+    }
+unlock:
+    UNLOCK(&slot->lock);
+
+    event_slot_unref(event_pool, slot, idx);
+
+    return ret;
+}
+
+struct event_ops event_ops_epoll = {
+    .new = event_pool_new_epoll,
+    .event_register = event_register_epoll,
+    .event_select_on = event_select_on_epoll,
+    .event_unregister = event_unregister_epoll,
+    .event_unregister_close = event_unregister_close_epoll,
+    .event_dispatch = event_dispatch_epoll,
+    .event_reconfigure_threads = event_reconfigure_threads_epoll,
+    .event_pool_destroy = event_pool_destroy_epoll,
+    .event_handled = event_handled_epoll,
+};
+
+#endif
diff --git a/libglusterfs/src/event-history.c b/libglusterfs/src/event-history.c
new file mode 100644
index 00000000000..379fed866be
--- /dev/null
+++ b/libglusterfs/src/event-history.c
@@ -0,0 +1,82 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/event-history.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+eh_t *
+eh_new(size_t buffer_size, gf_boolean_t use_buffer_once,
+       void (*destroy_buffer_data)(void *data))
+{
+    eh_t *history = NULL;
+    buffer_t *buffer = NULL;
+
+    history = GF_CALLOC(1, sizeof(eh_t), gf_common_mt_eh_t);
+    if (!history) {
+        goto out;
+    }
+
+    buffer = cb_buffer_new(buffer_size, use_buffer_once, destroy_buffer_data);
+    if (!buffer) {
+        GF_FREE(history);
+        history = NULL;
+        goto out;
+    }
+
+    history->buffer = buffer;
+
+    pthread_mutex_init(&history->lock, NULL);
+out:
+    return history;
+}
+
+void
+eh_dump(eh_t *history, void *data,
+        int(dump_fn)(circular_buffer_t *buffer, void *data))
+{
+    if (!history) {
+        gf_msg_debug("event-history", 0, "history is NULL");
+        goto out;
+    }
+
+    cb_buffer_dump(history->buffer, data, dump_fn);
+
+out:
+    return;
+}
+
+int
+eh_save_history(eh_t *history, void *data)
+{
+    int ret = -1;
+
+    ret = cb_add_entry_buffer(history->buffer, data);
+
+    return ret;
+}
+
+int
+eh_destroy(eh_t *history)
+{
+    if (!history) {
+        gf_msg("event-history", GF_LOG_INFO, 0, LG_MSG_INVALID_ARG,
+               "history for the xlator is NULL");
+        return -1;
+    }
+
+    cb_buffer_destroy(history->buffer);
+    history->buffer = NULL;
+
+    pthread_mutex_destroy(&history->lock);
+
+    GF_FREE(history);
+
+    return 0;
+}
diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c
new file mode 100644
index 00000000000..2cba963f096
--- /dev/null
+++ b/libglusterfs/src/event-poll.c
@@ -0,0 +1,513 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <sys/poll.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "glusterfs/logging.h"
+#include "glusterfs/gf-event.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+struct event_slot_poll {
+    int fd;
+    int events;
+    void *data;
+    event_handler_t handler;
+};
+
+static int
+event_register_poll(struct event_pool *event_pool, int fd,
+                    event_handler_t handler, void *data, int poll_in,
+                    int poll_out, char notify_poller_death);
+
+static void
+__flush_fd(int fd, int idx, int gen, void *data, int poll_in, int poll_out,
+           int poll_err, char event_thread_died)
+{
+    char buf[64];
+    int ret = -1;
+
+    if (!poll_in)
+        return;
+
+    do {
+        ret = sys_read(fd, buf, 64);
+        if (ret == -1 && errno != EAGAIN) {
+            gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_READ_FILE_FAILED,
+                    "fd=%d", fd, NULL);
+        }
+    } while (ret == 64);
+
+    return;
+}
+
+static int
+__event_getindex(struct event_pool *event_pool, int fd, int idx)
+{
+    int ret = -1;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    /* lookup in used space based on index provided */
+    if (idx > -1 && idx < event_pool->used) {
+        if (event_pool->reg[idx].fd == fd) {
+            ret = idx;
+            goto out;
+        }
+    }
+
+    /* search in used space, if lookup fails */
+    for (i = 0; i < event_pool->used; i++) {
+        if (event_pool->reg[i].fd == fd) {
+            ret = i;
+            break;
+        }
+    }
+
+out:
+    return ret;
+}
+
+static struct event_pool *
+event_pool_new_poll(int count, int eventthreadcount)
+{
+    struct event_pool *event_pool = NULL;
+    int ret = -1;
+
+    event_pool = GF_CALLOC(1, sizeof(*event_pool), gf_common_mt_event_pool);
+
+    if (!event_pool)
+        return NULL;
+
+    event_pool->count = count;
+    event_pool->reg = GF_CALLOC(event_pool->count, sizeof(*event_pool->reg),
+                                gf_common_mt_reg);
+
+    if (!event_pool->reg) {
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    pthread_mutex_init(&event_pool->mutex, NULL);
+
+    ret = pipe(event_pool->breaker);
+
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_PIPE_CREATE_FAILED, NULL);
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    ret = fcntl(event_pool->breaker[0], F_SETFL, O_NONBLOCK);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED, NULL);
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    ret = fcntl(event_pool->breaker[1], F_SETFL, O_NONBLOCK);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, errno, LG_MSG_SET_PIPE_FAILED, NULL);
+
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    ret = event_register_poll(event_pool, event_pool->breaker[0], __flush_fd,
+                              NULL, 1, 0, 0);
+    if (ret == -1) {
+        gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_REGISTER_PIPE_FAILED, NULL);
+        sys_close(event_pool->breaker[0]);
+        sys_close(event_pool->breaker[1]);
+        event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+        GF_FREE(event_pool->reg);
+        GF_FREE(event_pool);
+        return NULL;
+    }
+
+    if (eventthreadcount > 1) {
+        gf_smsg("poll", GF_LOG_INFO, 0, LG_MSG_POLL_IGNORE_MULTIPLE_THREADS,
+                "count=%d", eventthreadcount, NULL);
+    }
+
+    /* although, eventhreadcount for poll implementation is always
+     * going to be 1, eventthreadcount needs to be set to 1 so that
+     * rpcsvc_request_handler() thread scaling works flawlessly in
+     * both epoll and poll models
+     */
+    event_pool->eventthreadcount = 1;
+
+    return event_pool;
+}
+
+static int
+event_register_poll(struct event_pool *event_pool, int fd,
+                    event_handler_t handler, void *data, int poll_in,
+                    int poll_out, char notify_poller_death)
+{
+    int idx = -1;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        if (event_pool->count == event_pool->used) {
+            event_pool->count += 256;
+
+            event_pool->reg = GF_REALLOC(
+                event_pool->reg, event_pool->count * sizeof(*event_pool->reg));
+            if (!event_pool->reg)
+                goto unlock;
+        }
+
+        idx = event_pool->used++;
+
+        event_pool->reg[idx].fd = fd;
+        event_pool->reg[idx].events = POLLPRI;
+        event_pool->reg[idx].handler = handler;
+        event_pool->reg[idx].data = data;
+
+        switch (poll_in) {
+            case 1:
+                event_pool->reg[idx].events |= POLLIN;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLIN;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_IN,
+                        "value=%d", poll_in, NULL);
+                break;
+        }
+
+        switch (poll_out) {
+            case 1:
+                event_pool->reg[idx].events |= POLLOUT;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLOUT;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INVALID_POLL_OUT,
+                        "value=%d", poll_out, NULL);
+                break;
+        }
+
+        event_pool->changed = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&event_pool->mutex);
+
+out:
+    return idx;
+}
+
+static int
+event_unregister_poll(struct event_pool *event_pool, int fd, int idx_hint)
+{
+    int idx = -1;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, fd, idx_hint);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    fd, "idx_hint=%d", idx_hint, NULL);
+            errno = ENOENT;
+            goto unlock;
+        }
+
+        event_pool->reg[idx] = event_pool->reg[--event_pool->used];
+        event_pool->changed = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&event_pool->mutex);
+
+out:
+    return idx;
+}
+
+static int
+event_unregister_close_poll(struct event_pool *event_pool, int fd, int idx_hint)
+{
+    int ret = -1;
+
+    ret = event_unregister_poll(event_pool, fd, idx_hint);
+
+    sys_close(fd);
+
+    return ret;
+}
+
+static int
+event_select_on_poll(struct event_pool *event_pool, int fd, int idx_hint,
+                     int poll_in, int poll_out)
+{
+    int idx = -1;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, fd, idx_hint);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    fd, "idx_hint=%d", idx_hint, NULL);
+            errno = ENOENT;
+            goto unlock;
+        }
+
+        switch (poll_in) {
+            case 1:
+                event_pool->reg[idx].events |= POLLIN;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLIN;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                /* TODO: log error */
+                break;
+        }
+
+        switch (poll_out) {
+            case 1:
+                event_pool->reg[idx].events |= POLLOUT;
+                break;
+            case 0:
+                event_pool->reg[idx].events &= ~POLLOUT;
+                break;
+            case -1:
+                /* do nothing */
+                break;
+            default:
+                /* TODO: log error */
+                break;
+        }
+
+        if (poll_in + poll_out > -2)
+            event_pool->changed = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&event_pool->mutex);
+
+out:
+    return idx;
+}
+
+static int
+event_dispatch_poll_handler(struct event_pool *event_pool, struct pollfd *ufds,
+                            int i)
+{
+    event_handler_t handler = NULL;
+    void *data = NULL;
+    int idx = -1;
+    int ret = 0;
+
+    handler = NULL;
+    data = NULL;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        idx = __event_getindex(event_pool, ufds[i].fd, i);
+
+        if (idx == -1) {
+            gf_smsg("poll", GF_LOG_ERROR, 0, LG_MSG_INDEX_NOT_FOUND, "fd=%d",
+                    ufds[i].fd, "idx_hint=%d", i, NULL);
+            goto unlock;
+        }
+
+        handler = event_pool->reg[idx].handler;
+        data = event_pool->reg[idx].data;
+    }
+unlock:
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    if (handler)
+        handler(ufds[i].fd, idx, 0, data,
+                (ufds[i].revents & (POLLIN | POLLPRI)),
+                (ufds[i].revents & (POLLOUT)),
+                (ufds[i].revents & (POLLERR | POLLHUP | POLLNVAL)), 0);
+
+    return ret;
+}
+
+static int
+event_dispatch_poll_resize(struct event_pool *event_pool, struct pollfd *ufds,
+                           int size)
+{
+    int i = 0;
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        if (event_pool->changed == 0) {
+            goto unlock;
+        }
+
+        if (event_pool->used > event_pool->evcache_size) {
+            GF_FREE(event_pool->evcache);
+
+            event_pool->evcache = ufds = NULL;
+
+            event_pool->evcache_size = event_pool->used;
+
+            ufds = GF_CALLOC(sizeof(struct pollfd), event_pool->evcache_size,
+                             gf_common_mt_pollfd);
+            if (!ufds)
+                goto unlock;
+            event_pool->evcache = ufds;
+        }
+
+        if (ufds == NULL) {
+            goto unlock;
+        }
+
+        for (i = 0; i < event_pool->used; i++) {
+            ufds[i].fd = event_pool->reg[i].fd;
+            ufds[i].events = event_pool->reg[i].events;
+            ufds[i].revents = 0;
+        }
+
+        size = i;
+    }
+unlock:
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    return size;
+}
+
+static int
+event_dispatch_poll(struct event_pool *event_pool)
+{
+    struct pollfd *ufds = NULL;
+    int size = 0;
+    int i = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        event_pool->activethreadcount = 1;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    while (1) {
+        pthread_mutex_lock(&event_pool->mutex);
+        {
+            if (event_pool->destroy == 1) {
+                event_pool->activethreadcount = 0;
+                pthread_cond_broadcast(&event_pool->cond);
+                pthread_mutex_unlock(&event_pool->mutex);
+                return 0;
+            }
+        }
+        pthread_mutex_unlock(&event_pool->mutex);
+
+        size = event_dispatch_poll_resize(event_pool, ufds, size);
+        ufds = event_pool->evcache;
+
+        ret = poll(ufds, size, 1);
+
+        if (ret == 0)
+            /* timeout */
+            continue;
+
+        if (ret == -1 && errno == EINTR)
+            /* sys call */
+            continue;
+
+        for (i = 0; i < size; i++) {
+            if (!ufds[i].revents)
+                continue;
+
+            event_dispatch_poll_handler(event_pool, ufds, i);
+        }
+    }
+
+out:
+    return -1;
+}
+
+int
+event_reconfigure_threads_poll(struct event_pool *event_pool, int value)
+{
+    /* No-op for poll */
+
+    return 0;
+}
+
+/* This function is the destructor for the event_pool data structure
+ * Should be called only after poller_threads_destroy() is called,
+ * else will lead to crashes.
+ */
+static int
+event_pool_destroy_poll(struct event_pool *event_pool)
+{
+    int ret = 0;
+
+    ret = sys_close(event_pool->breaker[0]);
+    if (ret)
+        return ret;
+
+    ret = sys_close(event_pool->breaker[1]);
+    if (ret)
+        return ret;
+
+    event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+    GF_FREE(event_pool->reg);
+    GF_FREE(event_pool);
+
+    return ret;
+}
+
+struct event_ops event_ops_poll = {
+    .new = event_pool_new_poll,
+    .event_register = event_register_poll,
+    .event_select_on = event_select_on_poll,
+    .event_unregister = event_unregister_poll,
+    .event_unregister_close = event_unregister_close_poll,
+    .event_dispatch = event_dispatch_poll,
+    .event_reconfigure_threads = event_reconfigure_threads_poll,
+    .event_pool_destroy = event_pool_destroy_poll};
diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c
index cb6abd4180e..402c253ca25 100644
--- a/libglusterfs/src/event.c
+++ b/libglusterfs/src/event.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <sys/poll.h>
@@ -25,963 +16,290 @@
 #include <errno.h>
 #include <string.h>
 
-#include "logging.h"
-#include "event.h"
-#include "mem-pool.h"
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-static int
-event_register_poll (struct event_pool *event_pool, int fd,
-		     event_handler_t handler,
-		     void *data, int poll_in, int poll_out);
-
+#include "glusterfs/gf-event.h"
+#include "glusterfs/timespec.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/libglusterfs-messages.h"
+#include "glusterfs/syscall.h"
 
-static int
-__flush_fd (int fd, int idx, void *data,
-	    int poll_in, int poll_out, int poll_err)
-{
-	char buf[64];
-	int ret = -1;
-
-	if (!poll_in)
-		return ret;
-
-	do {
-		ret = read (fd, buf, 64);
-		if (ret == -1 && errno != EAGAIN) {
-			gf_log ("poll", GF_LOG_ERROR,
-				"read on %d returned error (%s)",
-				fd, strerror (errno));
-		}
-	} while (ret == 64);
-
-	return ret;
-}
-
-
-static int
-__event_getindex (struct event_pool *event_pool, int fd, int idx)
+struct event_pool *
+gf_event_pool_new(int count, int eventthreadcount)
 {
-	int  ret = -1;
-	int  i = 0;
-  
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	if (idx > -1 && idx < event_pool->used) {
-		if (event_pool->reg[idx].fd == fd)
-			ret = idx;
-	}
-
-	for (i=0; ret == -1 && i<event_pool->used; i++) {
-		if (event_pool->reg[i].fd == fd) {
-			ret = i;
-			break;
-		}
-	}
-
-	return ret;
-}
+    struct event_pool *event_pool = NULL;
+    extern struct event_ops event_ops_poll;
 
+#ifdef HAVE_SYS_EPOLL_H
+    extern struct event_ops event_ops_epoll;
 
-static struct event_pool *
-event_pool_new_poll (int count)
-{
-	struct event_pool *event_pool = NULL;
-	int                ret = -1;
-
-	event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
-
-	if (!event_pool)
-		return NULL;
-
-	event_pool->count = count;
-	event_pool->reg = GF_CALLOC (event_pool->count,
-				  sizeof (*event_pool->reg),
-                                  gf_common_mt_reg);
-
-	if (!event_pool->reg) {
-		gf_log ("poll", GF_LOG_CRITICAL,
-			"failed to allocate event registry");
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	pthread_mutex_init (&event_pool->mutex, NULL);
-
-	ret = pipe (event_pool->breaker);
-
-	if (ret == -1) {
-		gf_log ("poll", GF_LOG_ERROR,
-			"pipe creation failed (%s)", strerror (errno));
-		GF_FREE (event_pool->reg);
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	ret = fcntl (event_pool->breaker[0], F_SETFL, O_NONBLOCK);
-	if (ret == -1) {
-		gf_log ("poll", GF_LOG_ERROR,
-			"could not set pipe to non blocking mode (%s)",
-			strerror (errno));
-		close (event_pool->breaker[0]);
-		close (event_pool->breaker[1]);
-		event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-		GF_FREE (event_pool->reg);
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	ret = fcntl (event_pool->breaker[1], F_SETFL, O_NONBLOCK);
-	if (ret == -1) {
-		gf_log ("poll", GF_LOG_ERROR,
-			"could not set pipe to non blocking mode (%s)",
-			strerror (errno));
-
-		close (event_pool->breaker[0]);
-		close (event_pool->breaker[1]);
-		event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-		GF_FREE (event_pool->reg);
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	ret = event_register_poll (event_pool, event_pool->breaker[0],
-				   __flush_fd, NULL, 1, 0);
-	if (ret == -1) {
-		gf_log ("poll", GF_LOG_ERROR,
-			"could not register pipe fd with poll event loop");
-		close (event_pool->breaker[0]);
-		close (event_pool->breaker[1]);
-		event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-		GF_FREE (event_pool->reg);
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	return event_pool;
-}
+    event_pool = event_ops_epoll.new(count, eventthreadcount);
 
+    if (event_pool) {
+        event_pool->ops = &event_ops_epoll;
+    } else {
+        gf_msg("event", GF_LOG_WARNING, 0, LG_MSG_FALLBACK_TO_POLL,
+               "falling back to poll based event handling");
+    }
+#endif
 
-static int
-event_register_poll (struct event_pool *event_pool, int fd,
-		     event_handler_t handler,
-		     void *data, int poll_in, int poll_out)
-{
-	int idx = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		if (event_pool->count == event_pool->used)
-		{
-			event_pool->count += 256;
-
-			event_pool->reg = GF_REALLOC (event_pool->reg,
-					        event_pool->count *
-						sizeof (*event_pool->reg));
-                        if (!event_pool->reg)
-                                goto unlock;
-		}
-
-		idx = event_pool->used++;
-
-		event_pool->reg[idx].fd = fd;
-		event_pool->reg[idx].events = POLLPRI;
-		event_pool->reg[idx].handler = handler;
-		event_pool->reg[idx].data = data;
-
-		switch (poll_in) {
-		case 1:
-			event_pool->reg[idx].events |= POLLIN;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~POLLIN;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("poll", GF_LOG_ERROR,
-				"invalid poll_in value %d", poll_in);
-			break;
-		}
-
-		switch (poll_out) {
-		case 1:
-			event_pool->reg[idx].events |= POLLOUT;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~POLLOUT;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("poll", GF_LOG_ERROR,
-				"invalid poll_out value %d", poll_out);
-			break;
-		}
-
-		event_pool->changed = 1;
-
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return idx;
-}
+    if (!event_pool) {
+        event_pool = event_ops_poll.new(count, eventthreadcount);
 
+        if (event_pool)
+            event_pool->ops = &event_ops_poll;
+    }
 
-static int
-event_unregister_poll (struct event_pool *event_pool, int fd, int idx_hint)
-{
-	int idx = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, fd, idx_hint);
-
-		if (idx == -1) {
-			gf_log ("poll", GF_LOG_ERROR,
-				"index not found for fd=%d (idx_hint=%d)",
-				fd, idx_hint);
-			errno = ENOENT;
-			goto unlock;
-		}
-
-		event_pool->reg[idx] = 	event_pool->reg[--event_pool->used];
-		event_pool->changed = 1;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return idx;
+    return event_pool;
 }
 
-
-static int
-event_select_on_poll (struct event_pool *event_pool, int fd, int idx_hint,
-		      int poll_in, int poll_out)
-{
-	int idx = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, fd, idx_hint);
-
-		if (idx == -1) {
-			gf_log ("poll", GF_LOG_ERROR,
-				"index not found for fd=%d (idx_hint=%d)",
-				fd, idx_hint);
-			errno = ENOENT;
-			goto unlock;
-		}
-
-		switch (poll_in) {
-		case 1:
-			event_pool->reg[idx].events |= POLLIN;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~POLLIN;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			/* TODO: log error */
-			break;
-		}
-
-		switch (poll_out) {
-		case 1:
-			event_pool->reg[idx].events |= POLLOUT;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~POLLOUT;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			/* TODO: log error */
-			break;
-		}
-
-		if (poll_in + poll_out > -2)
-			event_pool->changed = 1;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return idx;
-}
-
-
-static int
-event_dispatch_poll_handler (struct event_pool *event_pool,
-			     struct pollfd *ufds, int i)
+int
+gf_event_register(struct event_pool *event_pool, int fd,
+                  event_handler_t handler, void *data, int poll_in,
+                  int poll_out, char notify_poller_death)
 {
-	event_handler_t  handler = NULL;
-	void            *data = NULL;
-	int              idx = -1;
-	int              ret = 0;
-
-	handler = NULL;
-	data    = NULL;
-
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, ufds[i].fd, i);
-
-		if (idx == -1) {
-			gf_log ("poll", GF_LOG_ERROR,
-				"index not found for fd=%d (idx_hint=%d)",
-				ufds[i].fd, i);
-			goto unlock;
-		}
-
-		handler = event_pool->reg[idx].handler;
-		data = event_pool->reg[idx].data;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	if (handler)
-		ret = handler (ufds[i].fd, idx, data,
-			       (ufds[i].revents & (POLLIN|POLLPRI)),
-			       (ufds[i].revents & (POLLOUT)),
-			       (ufds[i].revents & (POLLERR|POLLHUP|POLLNVAL)));
-
-	return ret;
-}
+    int ret = -1;
 
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-static int
-event_dispatch_poll_resize (struct event_pool *event_pool,
-			    struct pollfd *ufds, int size)
-{
-	int              i = 0;
-
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		if (event_pool->changed == 0) {
-			goto unlock;
-		}
-
-		if (event_pool->used > event_pool->evcache_size) {
-			if (event_pool->evcache)
-				GF_FREE (event_pool->evcache);
-
-			event_pool->evcache = ufds = NULL;
-
-			event_pool->evcache_size = event_pool->used;
-
-			ufds = GF_CALLOC (sizeof (struct pollfd),
-					  event_pool->evcache_size,
-                                          gf_common_mt_pollfd);
-                        if (!ufds)
-                                goto unlock;
-			event_pool->evcache = ufds;
-		}
-
-		for (i = 0; i < event_pool->used; i++) {
-			ufds[i].fd = event_pool->reg[i].fd;
-			ufds[i].events = event_pool->reg[i].events;
-			ufds[i].revents = 0;
-		}
-
-		size = i;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return size;
+    ret = event_pool->ops->event_register(
+        event_pool, fd, handler, data, poll_in, poll_out, notify_poller_death);
+out:
+    return ret;
 }
 
-
-static int
-event_dispatch_poll (struct event_pool *event_pool)
+int
+gf_event_unregister(struct event_pool *event_pool, int fd, int idx)
 {
-	struct pollfd   *ufds = NULL;
-	int              size = 0;
-	int              i = 0;
-	int              ret = -1;
-
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	while (1) {
-		size = event_dispatch_poll_resize (event_pool, ufds, size);
-		ufds = event_pool->evcache;
-
-		ret = poll (ufds, size, 1);
-
-		if (ret == 0)
-			/* timeout */
-			continue;
+    int ret = -1;
 
-		if (ret == -1 && errno == EINTR)
-			/* sys call */
-			continue;
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-		for (i = 0; i < size; i++) {
-			if (!ufds[i].revents)
-				continue;
+    ret = event_pool->ops->event_unregister(event_pool, fd, idx);
 
-			event_dispatch_poll_handler (event_pool, ufds, i);
-		}
-	}
-
-	return -1;
+out:
+    return ret;
 }
 
-
-static struct event_ops event_ops_poll = {
-	.new              = event_pool_new_poll,
-	.event_register   = event_register_poll,
-	.event_select_on  = event_select_on_poll,
-	.event_unregister = event_unregister_poll,
-	.event_dispatch   = event_dispatch_poll
-};
-
-
-
-#ifdef HAVE_SYS_EPOLL_H
-#include <sys/epoll.h>
-
-
-static struct event_pool *
-event_pool_new_epoll (int count)
+int
+gf_event_unregister_close(struct event_pool *event_pool, int fd, int idx)
 {
-	struct event_pool *event_pool = NULL;
-	int                epfd = -1;
-
-	event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
+    int ret = -1;
 
-	if (!event_pool)
-		return NULL;
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-	event_pool->count = count;
-	event_pool->reg = GF_CALLOC (event_pool->count,
-				  sizeof (*event_pool->reg),
-                                  gf_common_mt_reg);
+    ret = event_pool->ops->event_unregister_close(event_pool, fd, idx);
 
-	if (!event_pool->reg) {
-		gf_log ("epoll", GF_LOG_CRITICAL,
-			"event registry allocation failed");
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	epfd = epoll_create (count);
-
-	if (epfd == -1) {
-		gf_log ("epoll", GF_LOG_ERROR, "epoll fd creation failed (%s)",
-			strerror (errno));
-		GF_FREE (event_pool->reg);
-		GF_FREE (event_pool);
-		return NULL;
-	}
-
-	event_pool->fd = epfd;
-
-	event_pool->count = count;
-
-	pthread_mutex_init (&event_pool->mutex, NULL);
-	pthread_cond_init (&event_pool->cond, NULL);
-
-	return event_pool;
+out:
+    return ret;
 }
 
-
 int
-event_register_epoll (struct event_pool *event_pool, int fd,
-		      event_handler_t handler,
-		      void *data, int poll_in, int poll_out)
+gf_event_select_on(struct event_pool *event_pool, int fd, int idx_hint,
+                   int poll_in, int poll_out)
 {
-	int                 idx = -1;
-	int                 ret = -1;
-	struct epoll_event  epoll_event = {0, };
-	struct event_data  *ev_data = (void *)&epoll_event.data;
-
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		if (event_pool->count == event_pool->used) {
-			event_pool->count *= 2;
-
-			event_pool->reg = GF_REALLOC (event_pool->reg,
-						   event_pool->count *
-						   sizeof (*event_pool->reg));
-
-			if (!event_pool->reg) {
-				gf_log ("epoll", GF_LOG_ERROR,
-					"event registry re-allocation failed");
-				goto unlock;
-			}
-		}
-
-		idx = event_pool->used;
-		event_pool->used++;
-
-		event_pool->reg[idx].fd = fd;
-		event_pool->reg[idx].events = EPOLLPRI;
-		event_pool->reg[idx].handler = handler;
-		event_pool->reg[idx].data = data;
-
-		switch (poll_in) {
-		case 1:
-			event_pool->reg[idx].events |= EPOLLIN;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~EPOLLIN;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("epoll", GF_LOG_ERROR,
-				"invalid poll_in value %d", poll_in);
-			break;
-		}
-
-		switch (poll_out) {
-		case 1:
-			event_pool->reg[idx].events |= EPOLLOUT;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~EPOLLOUT;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("epoll", GF_LOG_ERROR,
-				"invalid poll_out value %d", poll_out);
-			break;
-		}
-
-		event_pool->changed = 1;
-
-		epoll_event.events = event_pool->reg[idx].events;
-		ev_data->fd = fd;
-		ev_data->idx = idx;
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_ADD, fd,
-				 &epoll_event);
-
-		if (ret == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"failed to add fd(=%d) to epoll fd(=%d) (%s)",
-				fd, event_pool->fd, strerror (errno));
-			goto unlock;
-		}
-
-		pthread_cond_broadcast (&event_pool->cond);
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return ret;
-}
+    int ret = -1;
 
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-static int
-event_unregister_epoll (struct event_pool *event_pool, int fd, int idx_hint)
-{
-	int  idx = -1;
-	int  ret = -1;
-
-	struct epoll_event epoll_event = {0, };
-	struct event_data *ev_data = (void *)&epoll_event.data;
-	int                lastidx = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, fd, idx_hint);
-
-		if (idx == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"index not found for fd=%d (idx_hint=%d)",
-				fd, idx_hint);
-			errno = ENOENT;
-			goto unlock;
-		}
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
-
-		/* if ret is -1, this array member should never be accessed */
-		/* if it is 0, the array member might be used by idx_cache
-		 * in which case the member should not be accessed till
-		 * it is reallocated
-		 */
-
-		event_pool->reg[idx].fd = -1;
-
-		if (ret == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"fail to del fd(=%d) from epoll fd(=%d) (%s)",
-				fd, event_pool->fd, strerror (errno));
-			goto unlock;
-		}
-
-		lastidx = event_pool->used - 1;
-		if (lastidx == idx) {
-			event_pool->used--;
-			goto unlock;
-		}
-
-		epoll_event.events = event_pool->reg[lastidx].events;
-		ev_data->fd = event_pool->reg[lastidx].fd;
-		ev_data->idx = idx;
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, ev_data->fd,
-				 &epoll_event);
-		if (ret == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"fail to modify fd(=%d) index %d to %d (%s)",
-				ev_data->fd, event_pool->used, idx,
-				strerror (errno));
-			goto unlock;
-		}
-
-		/* just replace the unregistered idx by last one */
-		event_pool->reg[idx] = event_pool->reg[lastidx];
-		event_pool->used--;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return ret;
+    ret = event_pool->ops->event_select_on(event_pool, fd, idx_hint, poll_in,
+                                           poll_out);
+out:
+    return ret;
 }
 
-
-static int
-event_select_on_epoll (struct event_pool *event_pool, int fd, int idx_hint,
-		       int poll_in, int poll_out)
+int
+gf_event_dispatch(struct event_pool *event_pool)
 {
-	int idx = -1;
-	int ret = -1;
-
-	struct epoll_event epoll_event = {0, };
-	struct event_data *ev_data = (void *)&epoll_event.data;
-
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, fd, idx_hint);
-
-		if (idx == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"index not found for fd=%d (idx_hint=%d)",
-				fd, idx_hint);
-			errno = ENOENT;
-			goto unlock;
-		}
-
-		switch (poll_in) {
-		case 1:
-			event_pool->reg[idx].events |= EPOLLIN;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~EPOLLIN;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("epoll", GF_LOG_ERROR,
-				"invalid poll_in value %d", poll_in);
-			break;
-		}
-
-		switch (poll_out) {
-		case 1:
-			event_pool->reg[idx].events |= EPOLLOUT;
-			break;
-		case 0:
-			event_pool->reg[idx].events &= ~EPOLLOUT;
-			break;
-		case -1:
-			/* do nothing */
-			break;
-		default:
-			gf_log ("epoll", GF_LOG_ERROR,
-				"invalid poll_out value %d", poll_out);
-			break;
-		}
-
-		epoll_event.events = event_pool->reg[idx].events;
-		ev_data->fd = fd;
-		ev_data->idx = idx;
-
-		ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, fd,
-				 &epoll_event);
-		if (ret == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"failed to modify fd(=%d) events to %d",
-				fd, epoll_event.events);
-		}
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	return ret;
-}
+    int ret = -1;
 
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-static int
-event_dispatch_epoll_handler (struct event_pool *event_pool,
-			      struct epoll_event *events, int i)
-{
-	struct event_data  *event_data = NULL;
-	event_handler_t     handler = NULL;
-	void               *data = NULL;
-	int                 idx = -1;
-	int                 ret = -1;
-
-
-	event_data = (void *)&events[i].data;
-	handler = NULL;
-	data = NULL;
-
-	pthread_mutex_lock (&event_pool->mutex);
-	{
-		idx = __event_getindex (event_pool, event_data->fd,
-					event_data->idx);
-
-		if (idx == -1) {
-			gf_log ("epoll", GF_LOG_ERROR,
-				"index not found for fd(=%d) (idx_hint=%d)",
-				event_data->fd, event_data->idx);
-			goto unlock;
-		}
-
-		handler = event_pool->reg[idx].handler;
-		data = event_pool->reg[idx].data;
-	}
-unlock:
-	pthread_mutex_unlock (&event_pool->mutex);
-
-	if (handler)
-		ret = handler (event_data->fd, event_data->idx, data,
-			       (events[i].events & (EPOLLIN|EPOLLPRI)),
-			       (events[i].events & (EPOLLOUT)),
-			       (events[i].events & (EPOLLERR|EPOLLHUP)));
-	return ret;
-}
+    ret = event_pool->ops->event_dispatch(event_pool);
+    if (ret)
+        goto out;
 
+out:
+    return ret;
+}
 
-static int
-event_dispatch_epoll (struct event_pool *event_pool)
+int
+gf_event_reconfigure_threads(struct event_pool *event_pool, int value)
 {
-	struct epoll_event *events = NULL;
-	int                 size = 0;
-	int                 i = 0;
-	int                 ret = -1;
-
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	while (1) {
-		pthread_mutex_lock (&event_pool->mutex);
-		{
-			while (event_pool->used == 0)
-				pthread_cond_wait (&event_pool->cond,
-						   &event_pool->mutex);
-
-			if (event_pool->used > event_pool->evcache_size) {
-				if (event_pool->evcache)
-					GF_FREE (event_pool->evcache);
-
-				event_pool->evcache = events = NULL;
-
-				event_pool->evcache_size =
-					event_pool->used + 256;
-
-				events = GF_CALLOC (event_pool->evcache_size,
-					            sizeof (struct epoll_event),
-                                                    gf_common_mt_epoll_event);
+    int ret = -1;
 
-				event_pool->evcache = events;
-			}
-		}
-		pthread_mutex_unlock (&event_pool->mutex);
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-		ret = epoll_wait (event_pool->fd, event_pool->evcache,
-				  event_pool->evcache_size, -1);
+    /* call event refresh function */
+    ret = event_pool->ops->event_reconfigure_threads(event_pool, value);
 
-		if (ret == 0)
-			/* timeout */
-			continue;
-
-		if (ret == -1 && errno == EINTR)
-			/* sys call */
-			continue;
-
-		size = ret;
-
-		for (i = 0; i < size; i++) {
-			if (!events || !events[i].events)
-				continue;
-
-			ret = event_dispatch_epoll_handler (event_pool,
-							    events, i);
-		}
-	}
-
-	return -1;
+out:
+    return ret;
 }
 
-
-static struct event_ops event_ops_epoll = {
-	.new              = event_pool_new_epoll,
-	.event_register   = event_register_epoll,
-	.event_select_on  = event_select_on_epoll,
-	.event_unregister = event_unregister_epoll,
-	.event_dispatch   = event_dispatch_epoll
-};
-
-#endif
-
-
-struct event_pool *
-event_pool_new (int count)
+int
+gf_event_pool_destroy(struct event_pool *event_pool)
 {
-	struct event_pool *event_pool = NULL;
+    int ret = -1;
+    int destroy = 0, activethreadcount = 0;
 
-#ifdef HAVE_SYS_EPOLL_H
-	event_pool = event_ops_epoll.new (count);
-
-	if (event_pool) {
-		event_pool->ops = &event_ops_epoll;
-	} else {
-		gf_log ("event", GF_LOG_WARNING,
-			"failing back to poll based event handling");
-	}
-#endif
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
 
-	if (!event_pool) {
-		event_pool = event_ops_poll.new (count);
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        destroy = event_pool->destroy;
+        activethreadcount = event_pool->activethreadcount;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
 
-		if (event_pool)
-			event_pool->ops = &event_ops_poll;
-	}
+    if (!destroy || (activethreadcount > 0)) {
+        goto out;
+    }
 
-	return event_pool;
+    ret = event_pool->ops->event_pool_destroy(event_pool);
+out:
+    return ret;
 }
 
-
-int
-event_register (struct event_pool *event_pool, int fd,
-		event_handler_t handler,
-		void *data, int poll_in, int poll_out)
+void
+poller_destroy_handler(int fd, int idx, int gen, void *data, int poll_out,
+                       int poll_in, int poll_err, char event_thread_exit)
 {
-	int ret = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	ret = event_pool->ops->event_register (event_pool, fd, handler, data,
-					       poll_in, poll_out);
-	return ret;
-}
+    struct event_destroy_data *destroy = NULL;
+    int readfd = -1;
+    char buf = '\0';
 
+    destroy = data;
+    readfd = destroy->readfd;
+    if (readfd < 0) {
+        goto out;
+    }
 
-int
-event_unregister (struct event_pool *event_pool, int fd, int idx)
-{
-	int ret = -1;
+    while (sys_read(readfd, &buf, 1) > 0) {
+    }
 
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	ret = event_pool->ops->event_unregister (event_pool, fd, idx);
+out:
+    gf_event_handled(destroy->pool, fd, idx, gen);
 
-	return ret;
+    return;
 }
 
-
+/* This function destroys all the poller threads.
+ * Note: to be called before gf_event_pool_destroy is called.
+ * The order in which cleaning is performed:
+ * - Register a pipe fd(this is for waking threads in poll()/epoll_wait())
+ * - Set the destroy mode, which this no new event registration will succeed
+ * - Reconfigure the thread count to 0(this will succeed only in destroy mode)
+ * - Wake up all the threads in poll() or epoll_wait(), so that they can
+ *   destroy themselves.
+ * - Wait for the thread to join(which will happen only after all the other
+ *   threads are destroyed)
+ */
 int
-event_select_on (struct event_pool *event_pool, int fd, int idx_hint,
-		 int poll_in, int poll_out)
+gf_event_dispatch_destroy(struct event_pool *event_pool)
 {
-	int ret = -1;
-
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	ret = event_pool->ops->event_select_on (event_pool, fd, idx_hint,
-						poll_in, poll_out);
-	return ret;
+    int ret = -1, threadcount = 0;
+    int fd[2] = {-1};
+    int idx = -1;
+    int flags = 0;
+    struct timespec sleep_till = {
+        0,
+    };
+    struct event_destroy_data data = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("event", event_pool, out);
+
+    ret = pipe(fd);
+    if (ret < 0)
+        goto out;
+
+    /* Make the read end of the pipe nonblocking */
+    flags = fcntl(fd[0], F_GETFL);
+    flags |= O_NONBLOCK;
+    ret = fcntl(fd[0], F_SETFL, flags);
+    if (ret < 0)
+        goto out;
+
+    /* Make the write end of the pipe nonblocking */
+    flags = fcntl(fd[1], F_GETFL);
+    flags |= O_NONBLOCK;
+    ret = fcntl(fd[1], F_SETFL, flags);
+    if (ret < 0)
+        goto out;
+
+    data.pool = event_pool;
+    data.readfd = fd[1];
+
+    /* From the main thread register an event on the pipe fd[0],
+     */
+    idx = gf_event_register(event_pool, fd[0], poller_destroy_handler, &data, 1,
+                            0, 0);
+    if (idx < 0)
+        goto out;
+
+    /* Enter the destroy mode first, set this before reconfiguring to 0
+     * threads, to prevent further reconfigure to thread count > 0.
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        threadcount = event_pool->eventthreadcount;
+        event_pool->destroy = 1;
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    ret = gf_event_reconfigure_threads(event_pool, 0);
+    if (ret < 0)
+        goto out;
+
+    /* Write something onto the write end of the pipe(fd[1]) so that
+     * poll wakes up and calls the handler, poller_destroy_handler()
+     */
+    pthread_mutex_lock(&event_pool->mutex);
+    {
+        /* Write to pipe(fd[1]) and then wait for 1 second or until
+         * a poller thread that is dying, broadcasts. Make sure we
+         * do not loop forever by limiting to 10 retries
+         */
+        int retry = 0;
+
+        while (event_pool->activethreadcount > 0 &&
+               (retry++ < (threadcount + 10))) {
+            if (sys_write(fd[1], "dummy", 6) == -1) {
+                break;
+            }
+            timespec_now_realtime(&sleep_till);
+            sleep_till.tv_sec += 1;
+            ret = pthread_cond_timedwait(&event_pool->cond, &event_pool->mutex,
+                                         &sleep_till);
+            if (ret) {
+                gf_msg_debug("event", 0,
+                             "thread cond-timedwait failed "
+                             "active-thread-count: %d, "
+                             "retry: %d",
+                             event_pool->activethreadcount, retry);
+            }
+        }
+    }
+    pthread_mutex_unlock(&event_pool->mutex);
+
+    ret = gf_event_unregister(event_pool, fd[0], idx);
+
+out:
+    if (fd[0] != -1)
+        sys_close(fd[0]);
+    if (fd[1] != -1)
+        sys_close(fd[1]);
+
+    return ret;
 }
 
-
 int
-event_dispatch (struct event_pool *event_pool)
+gf_event_handled(struct event_pool *event_pool, int fd, int idx, int gen)
 {
-	int ret = -1;
+    int ret = 0;
 
-	if (event_pool == NULL) {
-		gf_log ("event", GF_LOG_ERROR, "invalid argument");
-		return -1;
-	}
-  
-	ret = event_pool->ops->event_dispatch (event_pool);
+    if (event_pool->ops->event_handled)
+        ret = event_pool->ops->event_handled(event_pool, fd, idx, gen);
 
-	return ret;
+    return ret;
 }
diff --git a/libglusterfs/src/event.h b/libglusterfs/src/event.h
deleted file mode 100644
index 25396325c2a..00000000000
--- a/libglusterfs/src/event.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _EVENT_H_
-#define _EVENT_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <pthread.h>
-
-struct event_pool;
-struct event_ops;
-struct event_data {
-  int fd;
-  int idx;
-} __attribute__ ((__packed__));
-
-
-typedef int (*event_handler_t) (int fd, int idx, void *data,
-				int poll_in, int poll_out, int poll_err);
-
-struct event_pool {
-  struct event_ops *ops;
-
-  int fd;
-  int breaker[2];
-
-  int count;
-  struct {
-    int fd;
-    int events;
-    void *data;
-    event_handler_t handler;
-  } *reg;
-
-  int used;
-  int idx_cache;
-  int changed;
-
-  pthread_mutex_t mutex;
-  pthread_cond_t cond;
-
-  void *evcache;
-  int evcache_size;
-};
-
-struct event_ops {
-  struct event_pool * (*new) (int count);
-
-  int (*event_register) (struct event_pool *event_pool, int fd,
-			 event_handler_t handler,
-			 void *data, int poll_in, int poll_out);
-
-  int (*event_select_on) (struct event_pool *event_pool, int fd, int idx,
-			  int poll_in, int poll_out);
-
-  int (*event_unregister) (struct event_pool *event_pool, int fd, int idx);
-
-  int (*event_dispatch) (struct event_pool *event_pool);
-};
-
-struct event_pool * event_pool_new (int count);
-int event_select_on (struct event_pool *event_pool, int fd, int idx,
-		     int poll_in, int poll_out);
-int event_register (struct event_pool *event_pool, int fd,
-		    event_handler_t handler,
-		    void *data, int poll_in, int poll_out);
-int event_unregister (struct event_pool *event_pool, int fd, int idx);
-int event_dispatch (struct event_pool *event_pool);
-
-#endif /* _EVENT_H_ */
diff --git a/libglusterfs/src/events.c b/libglusterfs/src/events.c
new file mode 100644
index 00000000000..33157549897
--- /dev/null
+++ b/libglusterfs/src/events.c
@@ -0,0 +1,136 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <time.h>
+#include <stdarg.h>
+#include <string.h>
+#include <netinet/in.h>
+#include <netdb.h>
+
+#include "glusterfs/syscall.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/events.h"
+
+#define EVENT_HOST "127.0.0.1"
+#define EVENT_PORT 24009
+
+int
+_gf_event(eventtypes_t event, const char *fmt, ...)
+{
+    int ret = 0;
+    int sock = -1;
+    char *eventstr = NULL;
+    va_list arguments;
+    char *msg = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    char *host = NULL;
+    struct addrinfo hints;
+    struct addrinfo *result = NULL;
+    struct addrinfo *iter_result_ptr = NULL;
+    xlator_t *this = THIS;
+    char *volfile_server_transport = NULL;
+
+    /* Global context */
+    ctx = this->ctx;
+
+    if (event < 0 || event >= EVENT_LAST) {
+        ret = EVENT_ERROR_INVALID_INPUTS;
+        goto out;
+    }
+
+    if (ctx) {
+        volfile_server_transport = ctx->cmd_args.volfile_server_transport;
+    }
+    if (!volfile_server_transport) {
+        volfile_server_transport = "tcp";
+    }
+
+    /* host = NULL returns localhost */
+    if (ctx && ctx->cmd_args.volfile_server &&
+        (strcmp(volfile_server_transport, "unix"))) {
+        /* If it is client code then volfile_server is set
+           use that information to push the events. */
+        host = ctx->cmd_args.volfile_server;
+    }
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
+    hints.ai_socktype = SOCK_DGRAM;
+    hints.ai_flags = AI_ADDRCONFIG;
+
+    if ((getaddrinfo(host, TOSTRING(EVENT_PORT), &hints, &result)) != 0) {
+        ret = EVENT_ERROR_RESOLVE;
+        goto out;
+    }
+
+    // iterate over the result and break when socket creation is success.
+    for (iter_result_ptr = result; iter_result_ptr != NULL;
+         iter_result_ptr = iter_result_ptr->ai_next) {
+        sock = socket(iter_result_ptr->ai_family, iter_result_ptr->ai_socktype,
+                      iter_result_ptr->ai_protocol);
+        if (sock != -1) {
+            break;
+        }
+    }
+    /*
+     * If none of the addrinfo structures lead to a successful socket
+     * creation, socket creation has failed.
+     */
+    if (sock < 0) {
+        ret = EVENT_ERROR_SOCKET;
+        goto out;
+    }
+
+    va_start(arguments, fmt);
+    ret = gf_vasprintf(&msg, fmt, arguments);
+    va_end(arguments);
+
+    if (ret < 0) {
+        ret = EVENT_ERROR_INVALID_INPUTS;
+        goto out;
+    }
+
+    ret = gf_asprintf(&eventstr, "%u %d %s", (unsigned)gf_time(), event, msg);
+    GF_FREE(msg);
+    if (ret <= 0) {
+        ret = EVENT_ERROR_MSG_FORMAT;
+        goto out;
+    }
+
+    /* Send Message */
+    if (sendto(sock, eventstr, strlen(eventstr), 0, result->ai_addr,
+               result->ai_addrlen) <= 0) {
+        ret = EVENT_ERROR_SEND;
+        goto out;
+    }
+
+    ret = EVENT_SEND_OK;
+
+out:
+    if (sock >= 0) {
+        sys_close(sock);
+    }
+
+    /* Allocated by gf_asprintf */
+    if (eventstr)
+        GF_FREE(eventstr);
+
+    if (result)
+        freeaddrinfo(result);
+
+    return ret;
+}
diff --git a/libglusterfs/src/fd-lk.c b/libglusterfs/src/fd-lk.c
new file mode 100644
index 00000000000..c2d34f81c9c
--- /dev/null
+++ b/libglusterfs/src/fd-lk.c
@@ -0,0 +1,433 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/fd-lk.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+int32_t
+_fd_lk_delete_lock(fd_lk_ctx_node_t *lock)
+{
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lock, out);
+
+    list_del_init(&lock->next);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int32_t
+_fd_lk_destroy_lock(fd_lk_ctx_node_t *lock)
+{
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lock, out);
+
+    GF_FREE(lock);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+_fd_lk_destroy_lock_list(fd_lk_ctx_t *lk_ctx)
+{
+    int ret = -1;
+    fd_lk_ctx_node_t *lk = NULL;
+    fd_lk_ctx_node_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, out);
+
+    list_for_each_entry_safe(lk, tmp, &lk_ctx->lk_list, next)
+    {
+        _fd_lk_delete_lock(lk);
+        _fd_lk_destroy_lock(lk);
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+fd_lk_ctx_unref(fd_lk_ctx_t *lk_ctx)
+{
+    int ref = -1;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, err);
+
+    ref = GF_ATOMIC_DEC(lk_ctx->ref);
+    if (ref < 0)
+        GF_ASSERT(!ref);
+    if (ref == 0)
+        _fd_lk_destroy_lock_list(lk_ctx);
+
+    if (ref == 0) {
+        LOCK_DESTROY(&lk_ctx->lock);
+        GF_FREE(lk_ctx);
+    }
+
+    return 0;
+err:
+    return -1;
+}
+
+fd_lk_ctx_t *
+fd_lk_ctx_ref(fd_lk_ctx_t *lk_ctx)
+{
+    if (!lk_ctx) {
+        gf_msg_callingfn("fd-lk", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return NULL;
+    }
+
+    GF_ATOMIC_INC(lk_ctx->ref);
+
+    return lk_ctx;
+}
+
+fd_lk_ctx_t *
+fd_lk_ctx_create()
+{
+    fd_lk_ctx_t *fd_lk_ctx = NULL;
+
+    fd_lk_ctx = GF_CALLOC(1, sizeof(fd_lk_ctx_t), gf_common_mt_fd_lk_ctx_t);
+    if (!fd_lk_ctx)
+        goto out;
+
+    INIT_LIST_HEAD(&fd_lk_ctx->lk_list);
+
+    LOCK_INIT(&fd_lk_ctx->lock);
+
+    fd_lk_ctx = fd_lk_ctx_ref(fd_lk_ctx);
+out:
+    return fd_lk_ctx;
+}
+
+int
+_fd_lk_insert_lock(fd_lk_ctx_t *lk_ctx, fd_lk_ctx_node_t *lock)
+{
+    list_add_tail(&lock->next, &lk_ctx->lk_list);
+    return 0;
+}
+
+static off_t
+_fd_lk_get_lock_len(off_t start, off_t end)
+{
+    if (end == LLONG_MAX)
+        return 0;
+    else
+        return (end - start + 1);
+}
+
+fd_lk_ctx_node_t *
+fd_lk_ctx_node_new(int32_t cmd, struct gf_flock *flock)
+{
+    fd_lk_ctx_node_t *new_lock = NULL;
+
+    /* TODO: get from mem-pool */
+    new_lock = GF_CALLOC(1, sizeof(fd_lk_ctx_node_t),
+                         gf_common_mt_fd_lk_ctx_node_t);
+    if (!new_lock)
+        goto out;
+
+    new_lock->cmd = cmd;
+
+    if (flock) {
+        new_lock->fl_type = flock->l_type;
+        new_lock->fl_start = flock->l_start;
+
+        if (flock->l_len == 0)
+            new_lock->fl_end = LLONG_MAX;
+        else
+            new_lock->fl_end = flock->l_start + flock->l_len - 1;
+
+        memcpy(&new_lock->user_flock, flock, sizeof(struct gf_flock));
+    }
+
+    INIT_LIST_HEAD(&new_lock->next);
+out:
+    return new_lock;
+}
+
+int32_t
+_fd_lk_delete_unlck_locks(fd_lk_ctx_t *lk_ctx)
+{
+    int32_t ret = -1;
+    fd_lk_ctx_node_t *tmp = NULL;
+    fd_lk_ctx_node_t *lk = NULL;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", lk_ctx, out);
+
+    list_for_each_entry_safe(lk, tmp, &lk_ctx->lk_list, next)
+    {
+        if (lk->fl_type == F_UNLCK) {
+            _fd_lk_delete_lock(lk);
+            _fd_lk_destroy_lock(lk);
+        }
+    }
+out:
+    return ret;
+}
+
+int
+fd_lk_overlap(fd_lk_ctx_node_t *l1, fd_lk_ctx_node_t *l2)
+{
+    if (l1->fl_end >= l2->fl_start && l2->fl_end >= l1->fl_start)
+        return 1;
+
+    return 0;
+}
+
+fd_lk_ctx_node_t *
+_fd_lk_add_locks(fd_lk_ctx_node_t *l1, fd_lk_ctx_node_t *l2)
+{
+    fd_lk_ctx_node_t *sum = NULL;
+
+    sum = fd_lk_ctx_node_new(0, NULL);
+    if (!sum)
+        goto out;
+
+    sum->fl_start = min(l1->fl_start, l2->fl_start);
+    sum->fl_end = max(l1->fl_end, l2->fl_end);
+
+    sum->user_flock.l_start = sum->fl_start;
+    sum->user_flock.l_len = _fd_lk_get_lock_len(sum->fl_start, sum->fl_end);
+out:
+    return sum;
+}
+
+/* Subtract two locks */
+struct _values {
+    fd_lk_ctx_node_t *locks[3];
+};
+
+int32_t
+_fd_lk_sub_locks(struct _values *v, fd_lk_ctx_node_t *big,
+                 fd_lk_ctx_node_t *small)
+{
+    int32_t ret = -1;
+
+    if ((big->fl_start == small->fl_start) && (big->fl_end == small->fl_end)) {
+        /* both edges coincide with big */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+
+        v->locks[0]->fl_type = small->fl_type;
+        v->locks[0]->user_flock.l_type = small->fl_type;
+    } else if ((small->fl_start > big->fl_start) &&
+               (small->fl_end < big->fl_end)) {
+        /* small lock is completely inside big lock,
+           break it down into 3 different locks. */
+        v->locks[0] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        v->locks[2] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[2])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[0]->fl_end = small->fl_start - 1;
+        v->locks[0]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[0]->fl_start, v->locks[0]->fl_end);
+
+        memcpy(v->locks[1], small, sizeof(fd_lk_ctx_node_t));
+
+        memcpy(v->locks[2], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[2]->fl_start = small->fl_end + 1;
+        v->locks[2]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[2]->fl_start, v->locks[2]->fl_end);
+    } else if (small->fl_start == big->fl_start) {
+        /* One of the ends co-incide, break the
+           locks into two separate parts */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        memcpy(v->locks[0], small, sizeof(fd_lk_ctx_node_t));
+
+        memcpy(v->locks[1], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[1]->fl_start = small->fl_end + 1;
+        v->locks[1]->user_flock.l_start = small->fl_end + 1;
+    } else if (small->fl_end == big->fl_end) {
+        /* One of the ends co-incide, break the
+           locks into two separate parts */
+        v->locks[0] = fd_lk_ctx_node_new(small->cmd, NULL);
+        if (!v->locks[0])
+            goto out;
+
+        v->locks[1] = fd_lk_ctx_node_new(big->cmd, NULL);
+        if (!v->locks[1])
+            goto out;
+
+        memcpy(v->locks[0], big, sizeof(fd_lk_ctx_node_t));
+        v->locks[0]->fl_end = small->fl_start - 1;
+        v->locks[0]->user_flock.l_len = _fd_lk_get_lock_len(
+            v->locks[0]->fl_start, v->locks[0]->fl_end);
+
+        memcpy(v->locks[1], small, sizeof(fd_lk_ctx_node_t));
+    } else {
+        /* We should never come to this case */
+        GF_ASSERT(!"Invalid case");
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+_fd_lk_insert_and_merge(fd_lk_ctx_t *lk_ctx, fd_lk_ctx_node_t *lock)
+{
+    int32_t ret = -1;
+    int32_t i = 0;
+    fd_lk_ctx_node_t *entry = NULL;
+    fd_lk_ctx_node_t *t = NULL;
+    fd_lk_ctx_node_t *sum = NULL;
+    struct _values v = {.locks = {0, 0, 0}};
+
+    list_for_each_entry_safe(entry, t, &lk_ctx->lk_list, next)
+    {
+        if (!fd_lk_overlap(entry, lock))
+            continue;
+
+        if (entry->fl_type == lock->fl_type) {
+            sum = _fd_lk_add_locks(entry, lock);
+            if (!sum)
+                return;
+            sum->fl_type = entry->fl_type;
+            sum->user_flock.l_type = entry->fl_type;
+            _fd_lk_delete_lock(entry);
+            _fd_lk_destroy_lock(entry);
+            _fd_lk_destroy_lock(lock);
+            _fd_lk_insert_and_merge(lk_ctx, sum);
+            return;
+        } else {
+            sum = _fd_lk_add_locks(entry, lock);
+            sum->fl_type = lock->fl_type;
+            sum->user_flock.l_type = lock->fl_type;
+            ret = _fd_lk_sub_locks(&v, sum, lock);
+            if (ret)
+                return;
+            _fd_lk_delete_lock(entry);
+            _fd_lk_destroy_lock(entry);
+
+            _fd_lk_delete_lock(lock);
+            _fd_lk_destroy_lock(lock);
+
+            _fd_lk_destroy_lock(sum);
+
+            for (i = 0; i < 3; i++) {
+                if (!v.locks[i])
+                    continue;
+
+                INIT_LIST_HEAD(&v.locks[i]->next);
+                _fd_lk_insert_and_merge(lk_ctx, v.locks[i]);
+            }
+            _fd_lk_delete_unlck_locks(lk_ctx);
+            return;
+        }
+    }
+
+    /* no conflicts, so just insert */
+    if (lock->fl_type != F_UNLCK) {
+        _fd_lk_insert_lock(lk_ctx, lock);
+    } else {
+        _fd_lk_destroy_lock(lock);
+    }
+}
+
+static void
+print_lock_list(fd_lk_ctx_t *lk_ctx)
+{
+    fd_lk_ctx_node_t *lk = NULL;
+
+    gf_msg_debug("fd-lk", 0, "lock list:");
+
+    list_for_each_entry(lk, &lk_ctx->lk_list, next)
+        gf_msg_debug("fd-lk", 0,
+                     "owner = %s, cmd = %s fl_type = %s,"
+                     " fs_start = %" PRId64 ", fs_end = %" PRId64
+                     ", "
+                     "user_flock: l_type = %s, l_start = %" PRId64
+                     ", "
+                     "l_len = %" PRId64 ", ",
+                     lkowner_utoa(&lk->user_flock.l_owner), get_lk_cmd(lk->cmd),
+                     get_lk_type(lk->fl_type), lk->fl_start, lk->fl_end,
+                     get_lk_type(lk->user_flock.l_type), lk->user_flock.l_start,
+                     lk->user_flock.l_len);
+}
+
+int
+fd_lk_insert_and_merge(fd_t *fd, int32_t cmd, struct gf_flock *flock)
+{
+    int32_t ret = -1;
+    fd_lk_ctx_t *lk_ctx = NULL;
+    fd_lk_ctx_node_t *lk = NULL;
+
+    GF_VALIDATE_OR_GOTO("fd-lk", fd, out);
+    GF_VALIDATE_OR_GOTO("fd-lk", flock, out);
+
+    lk_ctx = fd_lk_ctx_ref(fd->lk_ctx);
+    lk = fd_lk_ctx_node_new(cmd, flock);
+
+    gf_msg_debug("fd-lk", 0,
+                 "new lock request: owner = %s, fl_type = %s"
+                 ", fs_start = %" PRId64 ", fs_end = %" PRId64
+                 ", user_flock:"
+                 " l_type = %s, l_start = %" PRId64 ", l_len = %" PRId64,
+                 lkowner_utoa(&flock->l_owner), get_lk_type(lk->fl_type),
+                 lk->fl_start, lk->fl_end, get_lk_type(lk->user_flock.l_type),
+                 lk->user_flock.l_start, lk->user_flock.l_len);
+
+    LOCK(&lk_ctx->lock);
+    {
+        _fd_lk_insert_and_merge(lk_ctx, lk);
+        print_lock_list(lk_ctx);
+    }
+    UNLOCK(&lk_ctx->lock);
+
+    fd_lk_ctx_unref(lk_ctx);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+gf_boolean_t
+fd_lk_ctx_empty(fd_lk_ctx_t *lk_ctx)
+{
+    gf_boolean_t verdict = _gf_true;
+
+    if (!lk_ctx)
+        return _gf_true;
+
+    LOCK(&lk_ctx->lock);
+    {
+        verdict = list_empty(&lk_ctx->lk_list);
+    }
+    UNLOCK(&lk_ctx->lock);
+
+    return verdict;
+}
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index def15255728..62606e91164 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -1,862 +1,1203 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include "fd.h"
-#include "glusterfs.h"
-#include "inode.h"
-#include "dict.h"
-#include "statedump.h"
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
+#include "glusterfs/fd.h"
+#include <errno.h>     // for EINVAL, errno, ENOMEM
+#include <inttypes.h>  // for PRIu64
+#include <stdint.h>    // for UINT32_MAX
+#include <string.h>    // for NULL, memcpy, memset, size_t
+#include "glusterfs/statedump.h"
 
 static int
-gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr);
-
+gf_fd_fdtable_expand(fdtable_t *fdtable, uint32_t nr);
 
 fd_t *
-_fd_ref (fd_t *fd);
-
-/*
-   Allocate in memory chunks of power of 2 starting from 1024B
-   Assumes fdtable->lock is held
-*/
-static inline int
-gf_roundup_power_of_two (uint32_t nr)
-{
-        uint32_t result = 1;
-
-        if (nr < 0) {
-                gf_log ("server-protocol/fd",
-                                GF_LOG_ERROR,
-                                "Negative number passed");
-                return -1;
-        }
-
-        while (result <= nr)
-                result *= 2;
-
-        return result;
-}
-
+__fd_ref(fd_t *fd);
 
 static int
-gf_fd_chain_fd_entries (fdentry_t *entries, uint32_t startidx,
-                        uint32_t endcount)
+gf_fd_chain_fd_entries(fdentry_t *entries, uint32_t startidx, uint32_t endcount)
 {
-        uint32_t        i = 0;
+    uint32_t i = 0;
 
-        if (!entries)
-                return -1;
+    if (!entries) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!entries");
+        return -1;
+    }
 
-        /* Chain only till the second to last entry because we want to
-         * ensure that the last entry has GF_FDTABLE_END.
-         */
-        for (i = startidx; i < (endcount - 1); i++)
-                entries[i].next_free = i + 1;
+    /* Chain only till the second to last entry because we want to
+     * ensure that the last entry has GF_FDTABLE_END.
+     */
+    for (i = startidx; i < (endcount - 1); i++)
+        entries[i].next_free = i + 1;
 
-        /* i has already been incremented upto the last entry. */
-        entries[i].next_free = GF_FDTABLE_END;
+    /* i has already been incremented up to the last entry. */
+    entries[i].next_free = GF_FDTABLE_END;
 
-        return 0;
+    return 0;
 }
 
-
 static int
-gf_fd_fdtable_expand (fdtable_t *fdtable, uint32_t nr)
+gf_fd_fdtable_expand(fdtable_t *fdtable, uint32_t nr)
 {
-	fdentry_t   *oldfds = NULL;
-	uint32_t     oldmax_fds = -1;
-        int          ret = -1;
-
-	if (fdtable == NULL || nr < 0) {
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-                ret = EINVAL;
-                goto out;
-	}
-
-	nr /= (1024 / sizeof (fdentry_t));
-	nr = gf_roundup_power_of_two (nr + 1);
-	nr *= (1024 / sizeof (fdentry_t));
+    fdentry_t *oldfds = NULL;
+    uint32_t oldmax_fds = -1;
+    int ret = -1;
+
+    if (fdtable == NULL || nr > UINT32_MAX) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        ret = EINVAL;
+        goto out;
+    }
+
+    nr /= (1024 / sizeof(fdentry_t));
+    nr = gf_roundup_next_power_of_two(nr + 1);
+    nr *= (1024 / sizeof(fdentry_t));
+
+    oldfds = fdtable->fdentries;
+    oldmax_fds = fdtable->max_fds;
+
+    fdtable->fdentries = GF_CALLOC(nr, sizeof(fdentry_t),
+                                   gf_common_mt_fdentry_t);
+    if (!fdtable->fdentries) {
+        ret = ENOMEM;
+        goto out;
+    }
+    fdtable->max_fds = nr;
+
+    if (oldfds) {
+        uint32_t cpy = oldmax_fds * sizeof(fdentry_t);
+        memcpy(fdtable->fdentries, oldfds, cpy);
+    }
+
+    gf_fd_chain_fd_entries(fdtable->fdentries, oldmax_fds, fdtable->max_fds);
+
+    /* Now that expansion is done, we must update the fd list
+     * head pointer so that the fd allocation functions can continue
+     * using the expanded table.
+     */
+    fdtable->first_free = oldmax_fds;
+    GF_FREE(oldfds);
+    ret = 0;
+out:
+    return ret;
+}
 
-	oldfds = fdtable->fdentries;
-	oldmax_fds = fdtable->max_fds;
+fdtable_t *
+gf_fd_fdtable_alloc(void)
+{
+    fdtable_t *fdtable = NULL;
 
-	fdtable->fdentries = GF_CALLOC (nr, sizeof (fdentry_t),
-                                        gf_common_mt_fdentry_t);
-	if (!fdtable->fdentries) {
-                ret = ENOMEM;
-                goto out;
-        }
-	fdtable->max_fds = nr;
+    fdtable = GF_CALLOC(1, sizeof(*fdtable), gf_common_mt_fdtable_t);
+    if (!fdtable)
+        return NULL;
 
-	if (oldfds) {
-		uint32_t cpy = oldmax_fds * sizeof (fdentry_t);
-		memcpy (fdtable->fdentries, oldfds, cpy);
-	}
+    pthread_rwlock_init(&fdtable->lock, NULL);
 
-        gf_fd_chain_fd_entries (fdtable->fdentries, oldmax_fds,
-                                fdtable->max_fds);
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        gf_fd_fdtable_expand(fdtable, 0);
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        /* Now that expansion is done, we must update the fd list
-         * head pointer so that the fd allocation functions can continue
-         * using the expanded table.
-         */
-        fdtable->first_free = oldmax_fds;
-	GF_FREE (oldfds);
-        ret = 0;
-out:
-	return ret;
+    return fdtable;
 }
 
-
-fdtable_t *
-gf_fd_fdtable_alloc (void)
+static fdentry_t *
+__gf_fd_fdtable_get_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-	fdtable_t *fdtable = NULL;
+    fdentry_t *fdentries = NULL;
 
-	fdtable = GF_CALLOC (1, sizeof (*fdtable), gf_common_mt_fdtable_t);
-	if (!fdtable) 
-		return NULL;
+    if (count == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!count");
+        goto out;
+    }
 
-	pthread_mutex_init (&fdtable->lock, NULL);
+    fdentries = fdtable->fdentries;
+    fdtable->fdentries = GF_CALLOC(fdtable->max_fds, sizeof(fdentry_t),
+                                   gf_common_mt_fdentry_t);
+    gf_fd_chain_fd_entries(fdtable->fdentries, 0, fdtable->max_fds);
+    *count = fdtable->max_fds;
 
-	pthread_mutex_lock (&fdtable->lock);
-	{
-		gf_fd_fdtable_expand (fdtable, 0);
-	}
-	pthread_mutex_unlock (&fdtable->lock);
-
-	return fdtable;
+out:
+    return fdentries;
 }
 
-
 fdentry_t *
-__gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+gf_fd_fdtable_get_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t       *fdentries = NULL;
+    fdentry_t *entries = NULL;
 
-        if (count == NULL) {
-                goto out;
+    if (fdtable) {
+        pthread_rwlock_wrlock(&fdtable->lock);
+        {
+            entries = __gf_fd_fdtable_get_all_fds(fdtable, count);
         }
+        pthread_rwlock_unlock(&fdtable->lock);
+    }
 
-        fdentries = fdtable->fdentries;
-        fdtable->fdentries = GF_CALLOC (fdtable->max_fds, sizeof (fdentry_t),
-                                        gf_common_mt_fdentry_t);
-        gf_fd_chain_fd_entries (fdtable->fdentries, 0, fdtable->max_fds);
-        *count = fdtable->max_fds;
+    return entries;
+}
+
+static fdentry_t *
+__gf_fd_fdtable_copy_all_fds(fdtable_t *fdtable, uint32_t *count)
+{
+    fdentry_t *fdentries = NULL;
+    int i = 0;
+
+    if (count == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!count");
+        goto out;
+    }
+
+    fdentries = GF_CALLOC(fdtable->max_fds, sizeof(fdentry_t),
+                          gf_common_mt_fdentry_t);
+    if (fdentries == NULL) {
+        goto out;
+    }
+
+    *count = fdtable->max_fds;
+
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (fdtable->fdentries[i].fd != NULL) {
+            fdentries[i].fd = fd_ref(fdtable->fdentries[i].fd);
+        }
+    }
 
 out:
-        return fdentries;
+    return fdentries;
 }
 
-
 fdentry_t *
-gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+gf_fd_fdtable_copy_all_fds(fdtable_t *fdtable, uint32_t *count)
 {
-        fdentry_t       *entries = NULL;
+    fdentry_t *entries = NULL;
 
-        if (fdtable) {
-                pthread_mutex_lock (&fdtable->lock);
-                {
-                        entries = __gf_fd_fdtable_get_all_fds (fdtable, count);
-                }
-                pthread_mutex_unlock (&fdtable->lock);
+    if (fdtable) {
+        pthread_rwlock_rdlock(&fdtable->lock);
+        {
+            entries = __gf_fd_fdtable_copy_all_fds(fdtable, count);
         }
+        pthread_rwlock_unlock(&fdtable->lock);
+    }
 
-        return entries;
+    return entries;
 }
 
-
 void
-gf_fd_fdtable_destroy (fdtable_t *fdtable)
-{
-        struct list_head  list = {0, };
-        fd_t             *fd = NULL;
-        fdentry_t        *fdentries = NULL;
-        uint32_t          fd_count = 0;
-        int32_t           i = 0;
-
-        INIT_LIST_HEAD (&list);
-
-	if (!fdtable)
-                return;
-
-	pthread_mutex_lock (&fdtable->lock);
-	{
-                fdentries = __gf_fd_fdtable_get_all_fds (fdtable, &fd_count);
-		GF_FREE (fdtable->fdentries);
-	}
-	pthread_mutex_unlock (&fdtable->lock);
-
-        if (fdentries != NULL) {
-                for (i = 0; i < fd_count; i++) {
-                        fd = fdentries[i].fd;
-                        if (fd != NULL) {
-                                fd_unref (fd);
-                        }
-                }
+gf_fd_fdtable_destroy(fdtable_t *fdtable)
+{
+    struct list_head list = {
+        0,
+    };
+    fd_t *fd = NULL;
+    fdentry_t *fdentries = NULL;
+    uint32_t fd_count = 0;
+    int32_t i = 0;
+
+    INIT_LIST_HEAD(&list);
+
+    if (!fdtable) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!fdtable");
+        return;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        fdentries = __gf_fd_fdtable_get_all_fds(fdtable, &fd_count);
+        GF_FREE(fdtable->fdentries);
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
+
+    if (fdentries != NULL) {
+        for (i = 0; i < fd_count; i++) {
+            fd = fdentries[i].fd;
+            if (fd != NULL) {
+                fd_unref(fd);
+            }
+        }
 
-                GF_FREE (fdentries);
-		pthread_mutex_destroy (&fdtable->lock);
-		GF_FREE (fdtable);
-	}
+        GF_FREE(fdentries);
+        pthread_rwlock_destroy(&fdtable->lock);
+        GF_FREE(fdtable);
+    }
 }
 
-
 int
-gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr)
-{
-	int32_t         fd = -1;
-        fdentry_t       *fde = NULL;
-	int             error;
-        int             alloc_attempts = 0;
-
-	if (fdtable == NULL || fdptr == NULL)
-	{
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-		return EINVAL;
-	}
-
-	pthread_mutex_lock (&fdtable->lock);
-	{
-fd_alloc_try_again:
-                if (fdtable->first_free != GF_FDTABLE_END) {
-                        fde = &fdtable->fdentries[fdtable->first_free];
-                        fd = fdtable->first_free;
-                        fdtable->first_free = fde->next_free;
-                        fde->next_free = GF_FDENTRY_ALLOCATED;
-                        fde->fd = fdptr;
-		} else {
-                        /* If this is true, there is something
-                         * seriously wrong with our data structures.
-                         */
-                        if (alloc_attempts >= 2) {
-                                gf_log ("server-protocol.c", GF_LOG_ERROR,
-                                        "Multiple attempts to expand fd table"
-                                        " have failed.");
-                                goto out;
-                        }
-                        error = gf_fd_fdtable_expand (fdtable,
-                                                      fdtable->max_fds + 1);
-			if (error) {
-				gf_log ("server-protocol.c",
-					GF_LOG_ERROR,
-					"Cannot expand fdtable:%s", strerror (error));
-                                goto out;
-			}
-                        ++alloc_attempts;
-                        /* At this point, the table stands expanded
-                         * with the first_free referring to the first
-                         * free entry in the new set of fdentries that
-                         * have just been allocated. That means, the
-                         * above logic should just work.
-                         */
-                        goto fd_alloc_try_again;
-		}
-	}
+gf_fd_unused_get(fdtable_t *fdtable, fd_t *fdptr)
+{
+    int32_t fd = -1;
+    fdentry_t *fde = NULL;
+    int error;
+    int alloc_attempts = 0;
+
+    if (fdtable == NULL || fdptr == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return EINVAL;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+    fd_alloc_try_again:
+        if (fdtable->first_free != GF_FDTABLE_END) {
+            fde = &fdtable->fdentries[fdtable->first_free];
+            fd = fdtable->first_free;
+            fdtable->first_free = fde->next_free;
+            fde->next_free = GF_FDENTRY_ALLOCATED;
+            fde->fd = fdptr;
+        } else {
+            /* If this is true, there is something
+             * seriously wrong with our data structures.
+             */
+            if (alloc_attempts >= 2) {
+                gf_msg("fd", GF_LOG_ERROR, 0, LG_MSG_EXPAND_FD_TABLE_FAILED,
+                       "multiple attempts to expand fd table"
+                       " have failed.");
+                goto out;
+            }
+            error = gf_fd_fdtable_expand(fdtable, fdtable->max_fds + 1);
+            if (error) {
+                gf_msg("fd", GF_LOG_ERROR, error, LG_MSG_EXPAND_FD_TABLE_FAILED,
+                       "Cannot expand fdtable");
+                goto out;
+            }
+            ++alloc_attempts;
+            /* At this point, the table stands expanded
+             * with the first_free referring to the first
+             * free entry in the new set of fdentries that
+             * have just been allocated. That means, the
+             * above logic should just work.
+             */
+            goto fd_alloc_try_again;
+        }
+    }
 out:
-	pthread_mutex_unlock (&fdtable->lock);
-
-	return fd;
-}
-
-
-inline void
-gf_fd_put (fdtable_t *fdtable, int32_t fd)
-{
-	fd_t *fdptr = NULL;
-        fdentry_t *fde = NULL;
-
-	if (fdtable == NULL || fd < 0) {
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-		return;
-	}
-
-	if (!(fd < fdtable->max_fds)) {
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-		return;
-	}
-
-	pthread_mutex_lock (&fdtable->lock);
-	{
-                fde = &fdtable->fdentries[fd];
-                /* If the entry is not allocated, put operation must return
-                 * without doing anything.
-                 * This has the potential of masking out any bugs in a user of
-                 * fd that ends up calling gf_fd_put twice for the same fd or
-                 * for an unallocated fd, but thats a price we have to pay for
-                 * ensuring sanity of our fd-table.
-                 */
-                if (fde->next_free != GF_FDENTRY_ALLOCATED)
-                        goto unlock_out;
-                fdptr = fde->fd;
-                fde->fd = NULL;
-                fde->next_free = fdtable->first_free;
-                fdtable->first_free = fd;
-	}
-unlock_out:
-	pthread_mutex_unlock (&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
-	if (fdptr) {
-		fd_unref (fdptr);
-	}
+    return fd;
 }
 
-
-fd_t *
-gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd)
+void
+gf_fd_put(fdtable_t *fdtable, int32_t fd)
 {
-	fd_t *fdptr = NULL;
+    fd_t *fdptr = NULL;
+    fdentry_t *fde = NULL;
 
-	if (fdtable == NULL || fd < 0) {
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-		errno = EINVAL;
-		return NULL;
-	}
+    if (fd == GF_ANON_FD_NO)
+        return;
 
-	if (!(fd < fdtable->max_fds)) {
-		gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-		errno = EINVAL;
-		return NULL;
-	}
+    if (fdtable == NULL || fd < 0) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
 
-	pthread_mutex_lock (&fdtable->lock);
-	{
-		fdptr = fdtable->fdentries[fd].fd;
-		if (fdptr) {
-			fd_ref (fdptr);
-		}
-	}
-	pthread_mutex_unlock (&fdtable->lock);
+    if (!(fd < fdtable->max_fds)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        fde = &fdtable->fdentries[fd];
+        /* If the entry is not allocated, put operation must return
+         * without doing anything.
+         * This has the potential of masking out any bugs in a user of
+         * fd that ends up calling gf_fd_put twice for the same fd or
+         * for an unallocated fd, but it is a price we have to pay for
+         * ensuring sanity of our fd-table.
+         */
+        if (fde->next_free != GF_FDENTRY_ALLOCATED)
+            goto unlock_out;
+        fdptr = fde->fd;
+        fde->fd = NULL;
+        fde->next_free = fdtable->first_free;
+        fdtable->first_free = fd;
+    }
+unlock_out:
+    pthread_rwlock_unlock(&fdtable->lock);
 
-	return fdptr;
+    if (fdptr) {
+        fd_unref(fdptr);
+    }
 }
 
-
-fd_t *
-_fd_ref (fd_t *fd)
+void
+gf_fdptr_put(fdtable_t *fdtable, fd_t *fd)
 {
-	++fd->refcount;
+    fdentry_t *fde = NULL;
+    int32_t i = 0;
 
-	return fd;
-}
+    if ((fdtable == NULL) || (fd == NULL)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return;
+    }
+
+    pthread_rwlock_wrlock(&fdtable->lock);
+    {
+        for (i = 0; i < fdtable->max_fds; i++) {
+            if (fdtable->fdentries[i].fd == fd) {
+                fde = &fdtable->fdentries[i];
+                break;
+            }
+        }
 
+        if (fde == NULL) {
+            gf_msg_callingfn("fd", GF_LOG_WARNING, 0,
+                             LG_MSG_FD_NOT_FOUND_IN_FDTABLE,
+                             "fd (%p) is not present in fdtable", fd);
+            goto unlock_out;
+        }
+
+        /* If the entry is not allocated, put operation must return
+         * without doing anything.
+         * This has the potential of masking out any bugs in a user of
+         * fd that ends up calling gf_fd_put twice for the same fd or
+         * for an unallocated fd, but it is a price we have to pay for
+         * ensuring sanity of our fd-table.
+         */
+        if (fde->next_free != GF_FDENTRY_ALLOCATED)
+            goto unlock_out;
+        fde->fd = NULL;
+        fde->next_free = fdtable->first_free;
+        fdtable->first_free = i;
+    }
+unlock_out:
+    pthread_rwlock_unlock(&fdtable->lock);
+
+    if ((fd != NULL) && (fde != NULL)) {
+        fd_unref(fd);
+    }
+}
 
 fd_t *
-fd_ref (fd_t *fd)
+gf_fd_fdptr_get(fdtable_t *fdtable, int64_t fd)
 {
-	fd_t *refed_fd = NULL;
+    fd_t *fdptr = NULL;
+
+    if (fdtable == NULL || fd < 0) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (!(fd < fdtable->max_fds)) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        errno = EINVAL;
+        return NULL;
+    }
+
+    pthread_rwlock_rdlock(&fdtable->lock);
+    {
+        fdptr = fdtable->fdentries[fd].fd;
+        if (fdptr) {
+            fd_ref(fdptr);
+        }
+    }
+    pthread_rwlock_unlock(&fdtable->lock);
 
-	if (!fd) {
-		gf_log ("fd", GF_LOG_ERROR, "@fd=%p", fd);
-		return NULL;
-	}
+    return fdptr;
+}
 
-	LOCK (&fd->inode->lock);
-	refed_fd = _fd_ref (fd);
-	UNLOCK (&fd->inode->lock);
+fd_t *
+__fd_ref(fd_t *fd)
+{
+    GF_ATOMIC_INC(fd->refcount);
 
-	return refed_fd;
+    return fd;
 }
 
-
 fd_t *
-_fd_unref (fd_t *fd)
+fd_ref(fd_t *fd)
 {
-	GF_ASSERT (fd->refcount);
-
-	--fd->refcount;
+    if (!fd) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "null fd");
+        return NULL;
+    }
 
-	if (fd->refcount == 0) {
-		list_del_init (&fd->inode_list);
-	}
+    GF_ATOMIC_INC(fd->refcount);
 
-	return fd;
+    return fd;
 }
 
-
 static void
-fd_destroy (fd_t *fd)
+fd_destroy(fd_t *fd, gf_boolean_t bound)
 {
-        xlator_t    *xl = NULL;
-	int          i = 0;
-        xlator_t    *old_THIS = NULL;
-	 struct mem_pool *tmp_pool = NULL;
-
-        if (fd == NULL){
-                gf_log ("xlator", GF_LOG_ERROR, "invalid arugument");
-                goto out;
+    xlator_t *xl = NULL;
+    int i = 0;
+    xlator_t *old_THIS = NULL;
+
+    if (fd == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        goto out;
+    }
+
+    if (fd->inode == NULL) {
+        gf_msg_callingfn("xlator", GF_LOG_ERROR, 0, LG_MSG_FD_INODE_NULL,
+                         "fd->inode is NULL");
+        goto out;
+    }
+    if (!fd->_ctx)
+        goto out;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        for (i = 0; i < fd->xl_count; i++) {
+            if (fd->_ctx[i].key) {
+                xl = fd->_ctx[i].xl_key;
+                old_THIS = THIS;
+                THIS = xl;
+                if (!xl->call_cleanup && xl->cbks->releasedir)
+                    xl->cbks->releasedir(xl, fd);
+                THIS = old_THIS;
+            }
         }
-
-        if (fd->inode == NULL){
-                gf_log ("xlator", GF_LOG_ERROR, "fd->inode is NULL");
-                goto out;
-        }
-	if (!fd->_ctx)
-		goto out;
-
-	 tmp_pool = fd->inode->table->fd_mem_pool;
-
-        if (IA_ISDIR (fd->inode->ia_type)) {
-		for (i = 0; i < fd->xl_count; i++) {
-			if (fd->_ctx[i].key) {
-				xl = fd->_ctx[i].xl_key;
-                                old_THIS = THIS;
-                                THIS = xl;
-				if (xl->cbks->releasedir)
-					xl->cbks->releasedir (xl, fd);
-                                THIS = old_THIS;
-			}
-		}
-        } else {
-		for (i = 0; i < fd->xl_count; i++) {
-			if (fd->_ctx[i].key) {
-				xl = fd->_ctx[i].xl_key;
-                                old_THIS = THIS;
-                                THIS = xl;
-				if (xl->cbks->release)
-					xl->cbks->release (xl, fd);
-                                THIS = old_THIS;
-			}
-		}
+    } else {
+        for (i = 0; i < fd->xl_count; i++) {
+            if (fd->_ctx[i].key) {
+                xl = fd->_ctx[i].xl_key;
+                old_THIS = THIS;
+                THIS = xl;
+                if (!xl->call_cleanup && xl->cbks->release)
+                    xl->cbks->release(xl, fd);
+                THIS = old_THIS;
+            }
         }
+    }
 
-        LOCK_DESTROY (&fd->lock);
+    LOCK_DESTROY(&fd->lock);
 
-	GF_FREE (fd->_ctx);
-        inode_unref (fd->inode);
-        fd->inode = (inode_t *)0xaaaaaaaa;
-        mem_put (tmp_pool,fd);
-	 tmp_pool = NULL;
+    GF_FREE(fd->_ctx);
+    if (bound) {
+        /*Decrease the count only after close happens on file*/
+        LOCK(&fd->inode->lock);
+        {
+            fd->inode->fd_count--;
+        }
+        UNLOCK(&fd->inode->lock);
+    }
+    inode_unref(fd->inode);
+    fd->inode = NULL;
+    fd_lk_ctx_unref(fd->lk_ctx);
+    mem_put(fd);
 out:
-        return;
+    return;
 }
 
-
 void
-fd_unref (fd_t *fd)
+fd_close(fd_t *fd)
 {
-        int32_t refcount = 0;
+    xlator_t *xl, *old_THIS;
 
-        if (!fd) {
-                gf_log ("fd.c", GF_LOG_ERROR, "fd is NULL");
-                return;
-        }
+    old_THIS = THIS;
 
-        LOCK (&fd->inode->lock);
-        {
-                _fd_unref (fd);
-                refcount = fd->refcount;
+    for (xl = fd->inode->table->xl->graph->first; xl != NULL; xl = xl->next) {
+        if (!xl->call_cleanup) {
+            THIS = xl;
+
+            if (IA_ISDIR(fd->inode->ia_type)) {
+                if (xl->cbks->fdclosedir != NULL) {
+                    xl->cbks->fdclosedir(xl, fd);
+                }
+            } else {
+                if (xl->cbks->fdclose != NULL) {
+                    xl->cbks->fdclose(xl, fd);
+                }
+            }
         }
-        UNLOCK (&fd->inode->lock);
+    }
+
+    THIS = old_THIS;
+}
 
+void
+fd_unref(fd_t *fd)
+{
+    int32_t refcount = 0;
+    gf_boolean_t bound = _gf_false;
+
+    if (!fd) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "fd is NULL");
+        return;
+    }
+
+    LOCK(&fd->inode->lock);
+    {
+        refcount = GF_ATOMIC_DEC(fd->refcount);
         if (refcount == 0) {
-                fd_destroy (fd);
+            if (!list_empty(&fd->inode_list)) {
+                list_del_init(&fd->inode_list);
+                fd->inode->active_fd_count--;
+                bound = _gf_true;
+            }
         }
+    }
+    UNLOCK(&fd->inode->lock);
 
-        return ;
+    if (refcount == 0) {
+        fd_destroy(fd, bound);
+    }
+
+    return;
 }
 
+static fd_t *
+__fd_bind(fd_t *fd)
+{
+    list_del_init(&fd->inode_list);
+    list_add(&fd->inode_list, &fd->inode->fd_list);
+    fd->inode->fd_count++;
+    fd->inode->active_fd_count++;
+
+    return fd;
+}
 
 fd_t *
-fd_bind (fd_t *fd)
+fd_bind(fd_t *fd)
 {
-        inode_t *inode = NULL;
+    if (!fd || !fd->inode) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "!fd || !fd->inode");
+        return NULL;
+    }
+
+    LOCK(&fd->inode->lock);
+    {
+        fd = __fd_bind(fd);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    return fd;
+}
 
-        if (!fd || !fd->inode) {
-                gf_log_callingfn ("fd", GF_LOG_ERROR, "!fd || !fd->inode");
-                return NULL;
-        }
-        inode = fd->inode;
+static fd_t *
+fd_allocate(inode_t *inode, uint64_t pid)
+{
+    fd_t *fd;
+
+    if (inode == NULL) {
+        gf_msg_callingfn("fd", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid argument");
+        return NULL;
+    }
+
+    fd = mem_get0(inode->table->fd_mem_pool);
+    if (fd == NULL) {
+        return NULL;
+    }
+
+    fd->xl_count = inode->table->xl->graph->xl_count + 1;
+
+    fd->_ctx = GF_CALLOC(1, (sizeof(struct _fd_ctx) * fd->xl_count),
+                         gf_common_mt_fd_ctx);
+    if (fd->_ctx == NULL) {
+        goto failed;
+    }
+
+    fd->lk_ctx = fd_lk_ctx_create();
+    if (fd->lk_ctx != NULL) {
+        /* We need to take a reference from the inode, but we cannot do it
+         * here because this function can be called with the inode lock taken
+         * and inode_ref() takes the inode's table lock. This is the reverse
+         * of the logical lock acquisition order and can cause a deadlock. So
+         * we simply assign the inode here and we delefate the inode reference
+         * responsibility to the caller (when this function succeeds and the
+         * inode lock is released). This is safe because the caller must hold
+         * a reference of the inode to use it, so it's guaranteed that the
+         * number of references won't reach 0 before the caller finishes.
+         *
+         * TODO: minimize use of locks in favor of atomic operations to avoid
+         *       these dependencies. */
+        fd->inode = inode;
+        fd->pid = pid;
+        INIT_LIST_HEAD(&fd->inode_list);
+        LOCK_INIT(&fd->lock);
+        GF_ATOMIC_INIT(fd->refcount, 1);
+        return fd;
+    }
 
-        LOCK (&inode->lock);
-        {
-                list_add (&fd->inode_list, &inode->fd_list);
-        }
-        UNLOCK (&inode->lock);
+    GF_FREE(fd->_ctx);
 
-        return fd;
+failed:
+    mem_put(fd);
+
+    return NULL;
 }
 
+fd_t *
+fd_create_uint64(inode_t *inode, uint64_t pid)
+{
+    fd_t *fd;
+
+    fd = fd_allocate(inode, pid);
+    if (fd != NULL) {
+        /* fd_allocate() doesn't get a reference from the inode. We need to
+         * take it here in case of success. */
+        inode_ref(inode);
+    }
+
+    return fd;
+}
 
 fd_t *
-fd_create (inode_t *inode, pid_t pid)
+fd_create(inode_t *inode, pid_t pid)
 {
-        fd_t *fd = NULL;
+    return fd_create_uint64(inode, (uint64_t)pid);
+}
 
-        if (inode == NULL) {
-                gf_log ("fd", GF_LOG_ERROR, "invalid argument");
-                return NULL;
+static fd_t *
+__fd_lookup(inode_t *inode, uint64_t pid)
+{
+    fd_t *iter_fd = NULL;
+    fd_t *fd = NULL;
+
+    if (list_empty(&inode->fd_list))
+        return NULL;
+
+    list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+    {
+        if (iter_fd->anonymous)
+            /* If someone was interested in getting an
+               anonymous fd (or was OK getting an anonymous fd),
+               they can as well call fd_anonymous() directly */
+            continue;
+
+        if (!pid || iter_fd->pid == pid) {
+            fd = __fd_ref(iter_fd);
+            break;
         }
+    }
 
-        fd = mem_get0 (inode->table->fd_mem_pool);
-        if (!fd)
-                goto out;
+    return fd;
+}
 
-        fd->xl_count = inode->table->xl->graph->xl_count + 1;
+fd_t *
+fd_lookup(inode_t *inode, pid_t pid)
+{
+    fd_t *fd = NULL;
 
-        fd->_ctx = GF_CALLOC (1, (sizeof (struct _fd_ctx) * fd->xl_count),
-                              gf_common_mt_fd_ctx);
-        if (!fd->_ctx) {
-                GF_FREE (fd);
-                fd = NULL;
-                goto out;
-        }
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
 
-        fd->inode = inode_ref (inode);
-        fd->pid = pid;
-        INIT_LIST_HEAD (&fd->inode_list);
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup(inode, (uint64_t)pid);
+    }
+    UNLOCK(&inode->lock);
 
-        LOCK_INIT (&fd->lock);
+    return fd;
+}
 
-        LOCK (&inode->lock);
-        {
-                fd = _fd_ref (fd);
-        }
-        UNLOCK (&inode->lock);
-out:
-        return fd;
+fd_t *
+fd_lookup_uint64(inode_t *inode, uint64_t pid)
+{
+    fd_t *fd = NULL;
+
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
+
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup(inode, pid);
+    }
+    UNLOCK(&inode->lock);
+
+    return fd;
 }
 
+static fd_t *
+__fd_lookup_anonymous(inode_t *inode, int32_t flags)
+{
+    fd_t *iter_fd = NULL;
+    fd_t *fd = NULL;
+
+    if (list_empty(&inode->fd_list))
+        return NULL;
+
+    list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+    {
+        if ((iter_fd->anonymous) && (flags == iter_fd->flags)) {
+            fd = __fd_ref(iter_fd);
+            break;
+        }
+    }
+
+    return fd;
+}
 
 fd_t *
-fd_lookup (inode_t *inode, pid_t pid)
+fd_anonymous_with_flags(inode_t *inode, int32_t flags)
 {
-        fd_t *fd = NULL;
-        fd_t *iter_fd = NULL;
+    fd_t *fd = NULL;
+    bool ref = false;
 
-        if (!inode)
-                return NULL;
+    LOCK(&inode->lock);
 
-        LOCK (&inode->lock);
-        {
-                if (list_empty (&inode->fd_list)) {
-                        fd = NULL;
-                } else {
-                        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                                if (pid) {
-                                        if (iter_fd->pid == pid) {
-                                                fd = _fd_ref (iter_fd);
-                                                break;
-                                        }
-                                } else {
-                                        fd = _fd_ref (iter_fd);
-                                        break;
-                                }
-                        }
-                }
+    fd = __fd_lookup_anonymous(inode, flags);
+
+    /* if (fd); then we already have increased the refcount in
+       __fd_lookup_anonymous(), so no need of one more fd_ref().
+       if (!fd); then both create and bind won't bump up the ref
+       count, so we have to call fd_ref() after bind. */
+    if (fd == NULL) {
+        fd = fd_allocate(inode, 0);
+        if (fd != NULL) {
+            fd->anonymous = _gf_true;
+            fd->flags = GF_ANON_FD_FLAGS | (flags & O_DIRECT);
+
+            __fd_bind(fd);
+
+            ref = true;
         }
-        UNLOCK (&inode->lock);
+    }
 
-        return fd;
+    UNLOCK(&inode->lock);
+
+    if (ref) {
+        /* fd_allocate() doesn't get a reference from the inode. We need to
+         * take it here in case of success. */
+        inode_ref(inode);
+    }
+
+    return fd;
 }
 
+fd_t *
+fd_anonymous(inode_t *inode)
+{
+    return fd_anonymous_with_flags(inode, 0);
+}
+
+fd_t *
+fd_lookup_anonymous(inode_t *inode, int32_t flags)
+{
+    fd_t *fd = NULL;
+
+    if (!inode) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "!inode");
+        return NULL;
+    }
+
+    LOCK(&inode->lock);
+    {
+        fd = __fd_lookup_anonymous(inode, flags);
+    }
+    UNLOCK(&inode->lock);
+    return fd;
+}
+
+gf_boolean_t
+fd_is_anonymous(fd_t *fd)
+{
+    return (fd && fd->anonymous);
+}
 
 uint8_t
-fd_list_empty (inode_t *inode)
+fd_list_empty(inode_t *inode)
 {
-        uint8_t empty = 0;
+    uint8_t empty = 0;
 
-        LOCK (&inode->lock);
-        {
-                empty = list_empty (&inode->fd_list);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        empty = list_empty(&inode->fd_list);
+    }
+    UNLOCK(&inode->lock);
 
-        return empty;
+    return empty;
 }
 
-
 int
-__fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
+__fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value)
 {
-	int index = 0;
-        int ret = 0;
-        int set_idx = -1;
+    int index = 0, new_xl_count = 0;
+    int ret = 0;
+    int set_idx = -1;
+    void *begin = NULL;
+    size_t diff = 0;
+    struct _fd_ctx *tmp = NULL;
+
+    if (!fd || !xlator)
+        return -1;
+
+    for (index = 0; index < fd->xl_count; index++) {
+        if (!fd->_ctx[index].key) {
+            if (set_idx == -1)
+                set_idx = index;
+            /* don't break, to check if key already exists
+               further on */
+        }
+        if (fd->_ctx[index].xl_key == xlator) {
+            set_idx = index;
+            break;
+        }
+    }
 
-	if (!fd || !xlator)
-		return -1;
+    if (set_idx == -1) {
+        set_idx = fd->xl_count;
 
-        for (index = 0; index < fd->xl_count; index++) {
-                if (!fd->_ctx[index].key) {
-                        if (set_idx == -1)
-                                set_idx = index;
-                        /* dont break, to check if key already exists
-                           further on */
-                }
-                if (fd->_ctx[index].xl_key == xlator) {
-                        set_idx = index;
-                        break;
-                }
-        }
+        new_xl_count = fd->xl_count + xlator->graph->xl_count;
 
-        if (set_idx == -1) {
-                ret = -1;
-                goto out;
+        tmp = GF_REALLOC(fd->_ctx, (sizeof(struct _fd_ctx) * new_xl_count));
+        if (tmp == NULL) {
+            ret = -1;
+            goto out;
         }
 
-        fd->_ctx[set_idx].xl_key = xlator;
-        fd->_ctx[set_idx].value1  = value;
+        fd->_ctx = tmp;
+
+        begin = fd->_ctx;
+        begin += (fd->xl_count * sizeof(struct _fd_ctx));
+
+        diff = (new_xl_count - fd->xl_count) * sizeof(struct _fd_ctx);
+
+        memset(begin, 0, diff);
+
+        fd->xl_count = new_xl_count;
+    }
+
+    fd->_ctx[set_idx].xl_key = xlator;
+    fd->_ctx[set_idx].value1 = value;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
+fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value)
 {
-        int ret = 0;
+    int ret = 0;
 
-	if (!fd || !xlator)
-		return -1;
+    if (!fd || !xlator) {
+        gf_msg_callingfn("fd", GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "%p %p", fd, xlator);
+        return -1;
+    }
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_set (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_set(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 int
-__fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value)
+__fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-	int index = 0;
-        int ret = 0;
+    int index = 0;
+    int ret = 0;
 
-	if (!fd || !xlator)
-		return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        for (index = 0; index < fd->xl_count; index++) {
-                if (fd->_ctx[index].xl_key == xlator)
-                        break;
-        }
+    for (index = 0; index < fd->xl_count; index++) {
+        if (fd->_ctx[index].xl_key == xlator)
+            break;
+    }
 
-        if (index == fd->xl_count) {
-                ret = -1;
-                goto out;
-        }
+    if (index == fd->xl_count) {
+        ret = -1;
+        goto out;
+    }
 
-        if (value)
-                *value = fd->_ctx[index].value1;
+    if (value)
+        *value = fd->_ctx[index].value1;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value)
+fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int ret = 0;
+    int ret = 0;
 
-	if (!fd || !xlator)
-		return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 int
-__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value)
+__fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-	int index = 0;
-        int ret = 0;
+    int index = 0;
+    int ret = 0;
 
-	if (!fd || !xlator)
-		return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        for (index = 0; index < fd->xl_count; index++) {
-                if (fd->_ctx[index].xl_key == xlator)
-                        break;
-        }
+    for (index = 0; index < fd->xl_count; index++) {
+        if (fd->_ctx[index].xl_key == xlator)
+            break;
+    }
 
-        if (index == fd->xl_count) {
-                ret = -1;
-                goto out;
-        }
+    if (index == fd->xl_count) {
+        ret = -1;
+        goto out;
+    }
 
-        if (value)
-                *value = fd->_ctx[index].value1;
+    if (value)
+        *value = fd->_ctx[index].value1;
 
-        fd->_ctx[index].key   = 0;
-        fd->_ctx[index].value1 = 0;
+    fd->_ctx[index].key = 0;
+    fd->_ctx[index].value1 = 0;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 int
-fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value)
+fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value)
 {
-        int ret = 0;
+    int ret = 0;
 
-	if (!fd || !xlator)
-		return -1;
+    if (!fd || !xlator)
+        return -1;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_del (fd, xlator, value);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_del(fd, xlator, value);
+    }
+    UNLOCK(&fd->lock);
 
-        return ret;
+    return ret;
 }
 
-
 void
-fd_dump (fd_t *fd, char *prefix)
-{
-        char        key[GF_DUMP_MAX_BUF_LEN];
-
-        if (!fd)
-                return;
-
-        memset(key, 0, sizeof(key));
-        gf_proc_dump_build_key(key, prefix, "pid");
-        gf_proc_dump_write(key, "%d", fd->pid);
-        gf_proc_dump_build_key(key, prefix, "refcount");
-        gf_proc_dump_write(key, "%d", fd->refcount);
-        gf_proc_dump_build_key(key, prefix, "flags");
-        gf_proc_dump_write(key, "%d", fd->flags);
-        if (fd->inode) {
-                gf_proc_dump_build_key(key, prefix, "inode");
-                gf_proc_dump_write(key, "%ld", fd->inode->ino);
-        }
-}
+fd_dump(fd_t *fd, char *prefix)
+{
+    char key[GF_DUMP_MAX_BUF_LEN];
+
+    if (!fd)
+        return;
+
+    gf_proc_dump_write("pid", "%" PRIu64, fd->pid);
+    gf_proc_dump_write("refcount", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(fd->refcount));
+    gf_proc_dump_write("flags", "%d", fd->flags);
 
+    if (fd->inode) {
+        gf_proc_dump_build_key(key, "inode", NULL);
+        gf_proc_dump_add_section("%s", key);
+        inode_dump(fd->inode, key);
+    }
+}
 
 void
-fdentry_dump (fdentry_t *fdentry, char *prefix)
+fdentry_dump(fdentry_t *fdentry, char *prefix)
 {
-        if (!fdentry)
-                return;
+    if (!fdentry)
+        return;
 
-        if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
-                return;
+    if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
+        return;
 
-        if (fdentry->fd)
-                fd_dump(fdentry->fd, prefix);
+    if (fdentry->fd)
+        fd_dump(fdentry->fd, prefix);
 }
 
-
 void
-fdtable_dump (fdtable_t *fdtable, char *prefix)
+fdtable_dump(fdtable_t *fdtable, char *prefix)
 {
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-        int     ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    int ret = -1;
 
-        if (!fdtable)
-                return;
+    if (!fdtable)
+        return;
+
+    ret = pthread_rwlock_tryrdlock(&fdtable->lock);
+    if (ret)
+        goto out;
+
+    gf_proc_dump_build_key(key, prefix, "refcount");
+    gf_proc_dump_write(key, "%d", fdtable->refcount);
+    gf_proc_dump_build_key(key, prefix, "maxfds");
+    gf_proc_dump_write(key, "%d", fdtable->max_fds);
+    gf_proc_dump_build_key(key, prefix, "first_free");
+    gf_proc_dump_write(key, "%d", fdtable->first_free);
+
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (GF_FDENTRY_ALLOCATED == fdtable->fdentries[i].next_free) {
+            gf_proc_dump_build_key(key, prefix, "fdentry[%d]", i);
+            gf_proc_dump_add_section("%s", key);
+            fdentry_dump(&fdtable->fdentries[i], key);
+        }
+    }
 
-	ret = pthread_mutex_trylock (&fdtable->lock);
+    pthread_rwlock_unlock(&fdtable->lock);
 
-        if (ret) {
-		gf_log ("fd", GF_LOG_WARNING, "Unable to acquire lock");
-                return;
+out:
+    if (ret != 0)
+        gf_proc_dump_write("Unable to dump the fdtable",
+                           "(Lock acquistion failed) %p", fdtable);
+    return;
+}
+
+void
+fd_ctx_dump(fd_t *fd, char *prefix)
+{
+    struct _fd_ctx *fd_ctx = NULL;
+    xlator_t *xl = NULL;
+    int i = 0;
+
+    if ((fd == NULL) || (fd->_ctx == NULL)) {
+        goto out;
+    }
+
+    LOCK(&fd->lock);
+    {
+        if (fd->_ctx != NULL) {
+            fd_ctx = GF_CALLOC(fd->xl_count, sizeof(*fd_ctx),
+                               gf_common_mt_fd_ctx);
+            if (fd_ctx == NULL) {
+                goto unlock;
+            }
+
+            for (i = 0; i < fd->xl_count; i++) {
+                fd_ctx[i] = fd->_ctx[i];
+            }
         }
+    }
+unlock:
+    UNLOCK(&fd->lock);
 
-        memset(key, 0, sizeof(key));
-        gf_proc_dump_build_key(key, prefix, "refcount");
-        gf_proc_dump_write(key, "%d", fdtable->refcount);
-        gf_proc_dump_build_key(key, prefix, "maxfds");
-        gf_proc_dump_write(key, "%d", fdtable->max_fds);
-        gf_proc_dump_build_key(key, prefix, "first_free");
-        gf_proc_dump_write(key, "%d", fdtable->first_free);
-
-        for ( i = 0 ; i < fdtable->max_fds; i++) {
-                if (GF_FDENTRY_ALLOCATED ==
-                                fdtable->fdentries[i].next_free) {
-                        gf_proc_dump_build_key(key, prefix, "fdentry[%d]", i);
-                        gf_proc_dump_add_section(key);
-                        fdentry_dump(&fdtable->fdentries[i], key);
-                }
+    if (fd_ctx == NULL) {
+        goto out;
+    }
+
+    for (i = 0; i < fd->xl_count; i++) {
+        if (fd_ctx[i].xl_key) {
+            xl = (xlator_t *)(long)fd_ctx[i].xl_key;
+            if (xl->dumpops && xl->dumpops->fdctx)
+                xl->dumpops->fdctx(xl, fd);
         }
+    }
 
-        pthread_mutex_unlock(&fdtable->lock);
+out:
+    GF_FREE(fd_ctx);
+
+    return;
 }
 
+void
+fdentry_dump_to_dict(fdentry_t *fdentry, char *prefix, dict_t *dict,
+                     int *openfds)
+{
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int ret = -1;
+
+    if (!fdentry)
+        return;
+    if (!dict)
+        return;
+
+    if (GF_FDENTRY_ALLOCATED != fdentry->next_free)
+        return;
+
+    if (fdentry->fd) {
+        snprintf(key, sizeof(key), "%s.pid", prefix);
+        ret = dict_set_uint64(dict, key, fdentry->fd->pid);
+        if (ret)
+            return;
+
+        snprintf(key, sizeof(key), "%s.refcount", prefix);
+        ret = dict_set_int32(dict, key, GF_ATOMIC_GET(fdentry->fd->refcount));
+        if (ret)
+            return;
+
+        snprintf(key, sizeof(key), "%s.flags", prefix);
+        ret = dict_set_int32(dict, key, fdentry->fd->flags);
+        if (ret)
+            return;
+
+        (*openfds)++;
+    }
+    return;
+}
 
 void
-fd_ctx_dump (fd_t *fd, char *prefix)
+fdtable_dump_to_dict(fdtable_t *fdtable, char *prefix, dict_t *dict)
 {
-        struct _fd_ctx *fd_ctx = NULL;
-        xlator_t       *xl     = NULL;
-        int    i               = 0;
-        
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int i = 0;
+    int openfds = 0;
+    int ret = -1;
+
+    if (!fdtable)
+        return;
+    if (!dict)
+        return;
 
-        if ((fd == NULL) || (fd->_ctx == NULL)) {
-                goto out;
-        }
+    ret = pthread_rwlock_tryrdlock(&fdtable->lock);
+    if (ret)
+        return;
 
-        LOCK (&fd->lock);
-        {
-                if (fd->_ctx != NULL) {
-                        fd_ctx = GF_CALLOC (fd->inode->table->xl->graph->xl_count,
-                                            sizeof (*fd_ctx),
-                                            gf_common_mt_fd_ctx);
-                        if (fd_ctx == NULL) {
-                                gf_log ("fd", GF_LOG_ERROR, "out of memory");
-                                goto unlock;
-                        }
-
-                        for (i = 0; i < fd->inode->table->xl->graph->xl_count;
-                             i++) {
-                                fd_ctx[i] = fd->_ctx[i];
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&fd->lock);
-        
-        if (fd_ctx == NULL) {
-                goto out;
+    snprintf(key, sizeof(key), "%s.fdtable.refcount", prefix);
+    ret = dict_set_int32(dict, key, fdtable->refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.fdtable.maxfds", prefix);
+    ret = dict_set_uint32(dict, key, fdtable->max_fds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.fdtable.firstfree", prefix);
+    ret = dict_set_int32(dict, key, fdtable->first_free);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < fdtable->max_fds; i++) {
+        if (GF_FDENTRY_ALLOCATED == fdtable->fdentries[i].next_free) {
+            snprintf(key, sizeof(key), "%s.fdtable.fdentry%d", prefix, i);
+            fdentry_dump_to_dict(&fdtable->fdentries[i], key, dict, &openfds);
         }
+    }
 
-        for (i = 0; i < fd->inode->table->xl->graph->xl_count; i++) {
-                if (fd_ctx[i].xl_key) {
-                        xl = (xlator_t *)(long)fd_ctx[i].xl_key;
-                        if (xl->dumpops && xl->dumpops->fdctx)
-                                xl->dumpops->fdctx (xl, fd);
-                }
-        }
+    snprintf(key, sizeof(key), "%s.fdtable.openfds", prefix);
+    ret = dict_set_int32(dict, key, openfds);
+    if (ret)
+        goto out;
 
 out:
-        if (fd_ctx != NULL) {
-                GF_FREE (fd_ctx);
-        }
-
-        return;
+    pthread_rwlock_unlock(&fdtable->lock);
+    return;
 }
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
deleted file mode 100644
index d19b0724f61..00000000000
--- a/libglusterfs/src/fd.h
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _FD_H
-#define _FD_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "list.h"
-#include <sys/types.h>
-#include <unistd.h>
-#include "glusterfs.h"
-#include "locking.h"
-
-struct _inode;
-struct _dict;
-
-struct _fd_ctx {
-        union {
-                uint64_t  key;
-                void     *xl_key;
-        };
-        union {
-                uint64_t  value1;
-                void     *ptr1;
-        };
-};
-
-/* If this structure changes, please have mercy on the booster maintainer
- * and update the fd_t struct in booster/src/booster-fd.h.
- * See the comment there to know why.
- */
-struct _fd {
-        pid_t             pid;
-	int32_t           flags;
-        int32_t           refcount;
-        struct list_head  inode_list;
-        struct _inode    *inode;
-        gf_lock_t         lock; /* used ONLY for manipulating
-                                   'struct _fd_ctx' array (_ctx).*/
-	struct _fd_ctx   *_ctx;
-        int               xl_count; /* Number of xl referred in this fd */
-};
-typedef struct _fd fd_t;
-
-
-struct fd_table_entry {
-        fd_t    *fd;
-        int     next_free;
-};
-typedef struct fd_table_entry fdentry_t;
-
-
-struct _fdtable {
-        int             refcount;
-        uint32_t        max_fds;
-        pthread_mutex_t lock;
-        fdentry_t       *fdentries;
-        int             first_free;
-};
-typedef struct _fdtable fdtable_t;
-
-
-/* Signifies no more entries in the fd table. */
-#define GF_FDTABLE_END  -1
-
-/* This is used to invalidated
- * the next_free value in an fdentry that has been allocated
- */
-#define GF_FDENTRY_ALLOCATED    -2
-
-#include "logging.h"
-#include "xlator.h"
-
-
-inline void
-gf_fd_put (fdtable_t *fdtable, int32_t fd);
-
-
-fd_t *
-gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd);
-
-
-fdtable_t *
-gf_fd_fdtable_alloc (void);
-
-
-int
-gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr);
-
-
-fdentry_t *
-gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count);
-
-
-void
-gf_fd_fdtable_destroy (fdtable_t *fdtable);
-
-
-fd_t *
-fd_ref (fd_t *fd);
-
-
-void
-fd_unref (fd_t *fd);
-
-
-fd_t *
-fd_create (struct _inode *inode, pid_t pid);
-
-
-fd_t *
-fd_lookup (struct _inode *inode, pid_t pid);
-
-
-uint8_t
-fd_list_empty (struct _inode *inode);
-
-
-fd_t *
-fd_bind (fd_t *fd);
-
-
-int
-fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value);
-
-
-int
-fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-int
-fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-int
-__fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value);
-
-
-int
-__fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-
-int
-__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
-
-fd_t *
-_fd_ref (fd_t *fd);
-
-void
-fd_ctx_dump (fd_t *fd, char *prefix);
-
-#endif /* _FD_H */
diff --git a/libglusterfs/src/gen-defaults.py b/libglusterfs/src/gen-defaults.py
new file mode 100755
index 00000000000..e31d3a9fe8a
--- /dev/null
+++ b/libglusterfs/src/gen-defaults.py
@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import sys
+from generator import ops, fop_subs, cbk_subs, generate
+
+FAILURE_CBK_TEMPLATE = """
+int32_t
+default_@NAME@_failure_cbk (call_frame_t *frame, int32_t op_errno)
+{
+	STACK_UNWIND_STRICT (@NAME@, frame, -1, op_errno, @ERROR_ARGS@);
+	return 0;
+}
+"""
+
+CBK_RESUME_TEMPLATE = """
+int32_t
+default_@NAME@_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int32_t op_ret, int32_t op_errno, @LONG_ARGS@)
+{
+	STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
+			     @SHORT_ARGS@);
+	return 0;
+}
+"""
+
+CBK_TEMPLATE = """
+int32_t
+default_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, @LONG_ARGS@)
+{
+	STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
+			     @SHORT_ARGS@);
+	return 0;
+}
+"""
+
+RESUME_TEMPLATE = """
+int32_t
+default_@NAME@_resume (call_frame_t *frame, xlator_t *this, @LONG_ARGS@)
+{
+	STACK_WIND (frame, default_@NAME@_cbk,
+		    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+		    @SHORT_ARGS@);
+	return 0;
+}
+"""
+
+FOP_TEMPLATE = """
+int32_t
+default_@NAME@ (
+	call_frame_t *frame,
+	xlator_t *this,
+	@LONG_ARGS@)
+{
+	STACK_WIND_TAIL (frame,
+			 FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+			 @SHORT_ARGS@);
+	return 0;
+}
+"""
+
+def gen_defaults ():
+	for name in list(ops.keys()):
+		print(generate(FAILURE_CBK_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(CBK_RESUME_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(CBK_TEMPLATE, name, cbk_subs))
+	for name in list(ops.keys()):
+		print(generate(RESUME_TEMPLATE, name, fop_subs))
+	for name in list(ops.keys()):
+		print(generate(FOP_TEMPLATE, name, fop_subs))
+
+for l in open(sys.argv[1], 'r').readlines():
+	if l.find('#pragma generate') != -1:
+		print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+		gen_defaults()
+		print("/* END GENERATED CODE */")
+	else:
+		print(l[:-1])
diff --git a/libglusterfs/src/generator.py b/libglusterfs/src/generator.py
new file mode 100755
index 00000000000..5b7aa4764a0
--- /dev/null
+++ b/libglusterfs/src/generator.py
@@ -0,0 +1,777 @@
+#!/usr/bin/python3
+
+import string
+
+# ops format: 'fop-arg' name type stub-field [nosync]
+#             'cbk-arg' name type
+#             'extra'   name type arg-str
+#             'journal' fop-type
+#             'link'    inode iatt
+#
+# 'role' indicates the significance of this line to the code generator (sort of
+# our own type).
+#
+# For fop-arg, we first need to know the name and the type of the arg so that
+# we can generate SHORT_ARGS (for function calls) and LONG_ARGS (for
+# declarations).  For code that uses stubs, we also need to know the name of
+# the stub field, which might be different than the argument itself.  Lastly,
+# for code that uses syncops, we need to know whether whoever wrote the syncop
+# for this fop "forgot" to include this argument.  (Editorial: this kind of
+# creeping inconsistency is why we should have used code generation for stubs
+# and syncops as well as defaults all along.)  To address this need, we use the
+# optional 'nosync' field for arguments (e.g. mkdir.umask) that we should skip
+# in generated syncop code.
+#
+# 'cbk-arg' is like fop-arg but simpler and used for generating callbacks
+# instead of fop functions.
+#
+# 'extra' is also like fop-arg, but it's another hack for syncops.  This time
+# the problem is that some of what would normally be *callback* arguments are
+# instead created in the caller and passed to the syncop.  We handle that by
+# adding an entry at the appropriate place in the fop-arg list, with the name
+# and type to generate a declaration and an argument string to generate the
+# actual syncop call.
+#
+# The mere presence of a 'journal' item is sufficient for most of the journal
+# code to recognize that it should do something.  However, reconciliation also
+# needs to decide how reconciliation builds the arguments it needs to call down
+# to the syncop layer, based on what's in the journal.  To do that, we divide
+# ops into three types and store those types in the ops table.  In general,
+# these three types work as follows.
+#
+#    For an fd-op, the GFID in the journal is used (in loc.gfid) field to
+#    look up an inode, then an anonymous fd is found/created for that inode.
+#
+#    For an inode-op, the GFID in the journal is used the same way, but no fd
+#    is needed.
+#
+#    For an entry-op, the *parent* GFID and name from the journal are used to
+#    look up an inode (via loc.pargfid and par.name respectively).
+#
+# The only places this seems to fall down is for link and create.  In link,
+# which is generally an entry-op, the source is looked up as though it's an
+# inode-op.  In create, we have an fd argument but it's really a return
+# argument so we get a fresh inode instead of looking one up.  Those two cases
+# need to be handled as special cases in the reconciliation code.
+#
+# 'link' is (hopefully) the last of the journal/syncop hacks.  Much like
+# 'extra', some values that are returned as callback arguments in the normal
+# case are handled differently for syncops.  For syncops that create objects
+# (e.g. mkdir) we need to link those objects into our inode table.  The 'inode'
+# and 'iatt' fields here give us the information we need to construct the
+# proper inode_link call(s).
+
+ops = {}
+xlator_cbks = {}
+xlator_dumpops = {}
+
+ops['fgetxattr'] = (
+	('fop-arg', 'fd',			'fd_t *'),
+	('fop-arg',	'name',			'const char *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'dict',			'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['fsetxattr'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'dict',			'dict_t *',			'xattr'),
+	('fop-arg',	'flags',		'int32_t',			'flags'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['setxattr'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'dict',			'dict_t *',			'xattr'),
+	('fop-arg',	'flags',		'int32_t',			'flags'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'inode-op'),
+)
+
+ops['statfs'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'buf',			'struct statvfs *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['fsyncdir'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'flags',		'int32_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['opendir'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'fd',			'fd_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['fstat'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['fsync'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'flags',		'int32_t'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'prebuf',		'struct iatt *'),
+	('cbk-arg',	'postbuf',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['flush'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['writev'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'vector',		'struct iovec *',	'vector'),
+	('fop-arg',	'count',		'int32_t'),
+	('fop-arg',	'off',			'off_t',			'offset'),
+	('fop-arg',	'flags',		'uint32_t',			'flags'),
+	('fop-arg',	'iobref',		'struct iobref *'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'prebuf',		'struct iatt *'),
+	('cbk-arg',	'postbuf',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['readv'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'size',			'size_t'),
+	('fop-arg',	'offset',		'off_t'),
+	('fop-arg',	'flags',		'uint32_t'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'vector',		'struct iovec *'),
+	('cbk-arg',	'count',		'int32_t'),
+	('cbk-arg',	'stbuf',		'struct iatt *'),
+	('cbk-arg',	'iobref',		'struct iobref *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['open'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'flags',		'int32_t'),
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'fd',			'fd_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['create'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'flags',		'int32_t',			'flags'),
+	('fop-arg',	'mode',			'mode_t',			'mode'),
+	('fop-arg',	'umask',		'mode_t',			'umask',	'nosync'),
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'fd',			'fd_t *'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+	('link',	'loc.inode',	'&iatt'),
+)
+
+ops['link'] = (
+	('fop-arg',	'oldloc',		'loc_t *',			'loc'),
+	('fop-arg',	'newloc',		'loc_t *',			'loc2'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['rename'] = (
+	('fop-arg',	'oldloc',		'loc_t *',			'loc'),
+	('fop-arg',	'newloc',		'loc_t *',			'loc2'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preoldparent',	'struct iatt *'),
+	('cbk-arg',	'postoldparent', 'struct iatt *'),
+	('cbk-arg',	'prenewparent',	'struct iatt *'),
+	('cbk-arg',	'postnewparent', 'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['symlink'] = (
+	('fop-arg',	'linkpath',		'const char *',		'linkname'),
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'umask',		'mode_t',			'mode',		'nosync'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['rmdir'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'flags',		'int32_t',			'flags'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['unlink'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'flags',		'int32_t',			'flags',	'nosync'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['mkdir'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'mode',			'mode_t',			'mode'),
+	('fop-arg',	'umask',		'mode_t',			'umask',	'nosync'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+	('link',	'loc.inode',	'&iatt'),
+)
+
+ops['mknod'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'mode',			'mode_t',			'mode'),
+	('fop-arg',	'rdev',			'dev_t',			'rdev'),
+	('fop-arg',	'umask',		'mode_t',			'umask',	'nosync'),
+	('extra',	'iatt',			'struct iatt',		'&iatt'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'preparent',	'struct iatt *'),
+	('cbk-arg',	'postparent',	'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'entry-op'),
+)
+
+ops['readlink'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'size',			'size_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'path',			'const char *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['access'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'mask',			'int32_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['ftruncate'] = (
+	('fop-arg',	'fd',			'fd_t *',				'fd'),
+	('fop-arg',	'offset',		'off_t',				'offset'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
+	('fop-arg',	'xdata',		'dict_t *',				'xdata'),
+	('cbk-arg',	'prebuf',		'struct iatt *'),
+	('cbk-arg',	'postbuf',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['getxattr'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'name',			'const char *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'dict',			'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['xattrop'] = (
+	('fop-arg',	'loc',			'loc_t *',				'loc'),
+	('fop-arg',	'flags',		'gf_xattrop_flags_t',	'optype'),
+	('fop-arg',	'dict',			'dict_t *',				'xattr'),
+	('fop-arg',	'xdata',		'dict_t *',				'xdata'),
+	('cbk-arg',	'dict',			'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'inode-op'),
+)
+
+ops['fxattrop'] = (
+	('fop-arg',	'fd',			'fd_t *',				'fd'),
+	('fop-arg',	'flags',		'gf_xattrop_flags_t',	'optype'),
+	('fop-arg',	'dict',			'dict_t *',				'xattr'),
+	('fop-arg',	'xdata',		'dict_t *',				'xdata'),
+	('cbk-arg',	'dict',			'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['removexattr'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'name',			'const char *',		'name'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'inode-op'),
+)
+
+ops['fremovexattr'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'name',			'const char *',		'name'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['lk'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'cmd',			'int32_t'),
+	('fop-arg',	'lock',			'struct gf_flock *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'lock',			'struct gf_flock *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['inodelk'] = (
+	('fop-arg',	'volume',		'const char *'),
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'cmd',			'int32_t'),
+	('fop-arg',	'lock',			'struct gf_flock *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['finodelk'] = (
+	('fop-arg',	'volume',		'const char *'),
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'cmd',			'int32_t'),
+	('fop-arg',	'lock',			'struct gf_flock *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['entrylk'] = (
+	('fop-arg',	'volume',		'const char *'),
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'basename',		'const char *'),
+	('fop-arg',	'cmd',			'entrylk_cmd'),
+	('fop-arg',	'type',			'entrylk_type'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['fentrylk'] = (
+	('fop-arg',	'volume',		'const char *'),
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'basename',		'const char *'),
+	('fop-arg',	'cmd',			'entrylk_cmd'),
+	('fop-arg',	'type',			'entrylk_type'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['rchecksum'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'offset',		'off_t'),
+	('fop-arg',	'len',			'int32_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'weak_cksum',	'uint32_t'),
+	('cbk-arg',	'strong_cksum',	'uint8_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['readdir'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'size',			'size_t'),
+	('fop-arg',	'off',			'off_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'entries',		'gf_dirent_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['readdirp'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'size',			'size_t'),
+	('fop-arg',	'off',			'off_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'entries',		'gf_dirent_t *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['setattr'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'stbuf',		'struct iatt *',	'stat'),
+	('fop-arg',	'valid',		'int32_t',			'valid'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'statpre',		'struct iatt *'),
+	('cbk-arg',	'statpost',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'inode-op'),
+)
+
+ops['truncate'] = (
+	('fop-arg',	'loc',			'loc_t *',			'loc'),
+	('fop-arg',	'offset',		'off_t',			'offset'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'prebuf',		'struct iatt *'),
+	('cbk-arg',	'postbuf',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'inode-op'),
+)
+
+ops['stat'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['lookup'] = (
+	('fop-arg',	'loc',			'loc_t *'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'inode',		'inode_t *'),
+	('cbk-arg',	'buf',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	# We could add xdata everywhere automatically if somebody hadn't put
+	# something after it here.
+	('cbk-arg',	'postparent',	'struct iatt *'),
+)
+
+ops['fsetattr'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'stbuf',		'struct iatt *',	'stat'),
+	('fop-arg',	'valid',		'int32_t',			'valid'),
+	('extra',	'preop',		'struct iatt',		'&preop'),
+	('extra',	'postop',		'struct iatt',		'&postop'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'statpre',		'struct iatt *'),
+	('cbk-arg',	'statpost',		'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['fallocate'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'keep_size',	'int32_t',			'mode'),
+	('fop-arg',	'offset',		'off_t',			'offset'),
+	('fop-arg',	'len',			'size_t',			'size'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'pre',			'struct iatt *'),
+	('cbk-arg',	'post',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['discard'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'offset',		'off_t',			'offset'),
+	('fop-arg',	'len',			'size_t',			'size'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'pre',			'struct iatt *'),
+	('cbk-arg',	'post',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['zerofill'] = (
+	('fop-arg',	'fd',			'fd_t *',			'fd'),
+	('fop-arg',	'offset',		'off_t',			'offset'),
+	# As e.g. fallocate/discard (above) "len" should really be a size_t.
+	('fop-arg',	'len',			'off_t',			'size'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'pre',			'struct iatt *'),
+	('cbk-arg',	'post',			'struct iatt *'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['ipc'] = (
+	('fop-arg',	'op',			'int32_t'),
+	('fop-arg',	'xdata',		'dict_t *',			'xdata'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+	('journal',	'fd-op'),
+)
+
+ops['seek'] = (
+	('fop-arg',	'fd',			'fd_t *'),
+	('fop-arg',	'offset',		'off_t'),
+	('fop-arg',	'what',			'gf_seek_what_t'),
+	('fop-arg',	'xdata',		'dict_t *'),
+	('cbk-arg',	'offset',		'off_t'),
+	('cbk-arg',	'xdata',		'dict_t *'),
+)
+
+ops['getspec'] = (
+	('fop-arg',	'key',			'const char *'),
+	('fop-arg',	'flags',		'int32_t'),
+	('cbk-arg',	'spec_data',	'char *'),
+)
+
+ops['lease'] = (
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'lease',                'struct gf_lease *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'lease',                'struct gf_lease *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['getactivelk'] = (
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'locklist',             'lock_migration_info_t *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['setactivelk'] = (
+	('fop-arg',     'loc',                  'loc_t *'),
+	('fop-arg',     'locklist',             'lock_migration_info_t *'),
+	('fop-arg',     'xdata',                'dict_t *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['put'] = (
+	('fop-arg',     'loc',                  'loc_t *',                      'loc'),
+	('fop-arg',     'mode',                 'mode_t',                       'mode'),
+	('fop-arg',     'umask',                'mode_t',                       'umask'),
+	('fop-arg',     'flags',                'uint32_t',                     'flags'),
+	('fop-arg',     'vector',               'struct iovec *',               'vector'),
+	('fop-arg',     'count',                'int32_t'),
+	('fop-arg',     'off',                  'off_t',                        'offset'),
+	('fop-arg',     'iobref',               'struct iobref *'),
+	('fop-arg',     'dict',                 'dict_t *',                     'xattr'),
+	('fop-arg',     'xdata',                'dict_t *',                     'xdata'),
+	('cbk-arg',     'inode',                'inode_t *'),
+	('cbk-arg',     'buf',                  'struct iatt *'),
+	('cbk-arg',     'preparent',            'struct iatt *'),
+	('cbk-arg',     'postparent',           'struct iatt *'),
+	('cbk-arg',     'xdata',                'dict_t *'),
+)
+
+ops['icreate'] = (
+	('fop-arg',     'loc',                   'loc_t *'),
+	('fop-arg',     'mode',                  'mode_t'),
+	('fop-arg',     'xdata',                 'dict_t *'),
+	('cbk-arg',     'inode',                 'inode_t *'),
+	('cbk-arg',     'buf',                   'struct iatt *'),
+	('cbk-arg',     'xdata',                 'dict_t *'),
+)
+
+ops['namelink'] = (
+	('fop-arg',     'loc',                   'loc_t *'),
+	('fop-arg',     'xdata',                 'dict_t *'),
+	('cbk-arg',     'prebuf',                'struct iatt *'),
+	('cbk-arg',     'postbuf',               'struct iatt *'),
+	('cbk-arg',     'xdata',                 'dict_t *'),
+)
+
+ops['copy_file_range'] = (
+        ('fop-arg',     'fd_in',                 'fd_t *'),
+        ('fop-arg',     'off_in',                'off64_t '),
+        ('fop-arg',     'fd_out',                'fd_t *'),
+        ('fop-arg',     'off_out',               'off64_t '),
+        ('fop-arg',     'len',                   'size_t'),
+        ('fop-arg',     'flags',                 'uint32_t'),
+        ('fop-arg',     'xdata',                 'dict_t *'),
+        ('cbk-arg',     'stbuf',                 'struct iatt *'),
+        ('cbk-arg',     'prebuf_dst',            'struct iatt *'),
+        ('cbk-arg',     'postbuf_dst',           'struct iatt *'),
+        ('cbk-arg',     'xdata',                 'dict_t *'),
+)
+#####################################################################
+xlator_cbks['forget'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['release'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['releasedir'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['invalidate'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['client_destroy'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'client',      'client_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['client_disconnect'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'client',      'client_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['ictxmerge'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('fn-arg',      'inode',       'inode_t *'),
+	('fn-arg',      'linked_inode', 'inode_t *'),
+	('ret-val',     'void',        ''),
+)
+
+#####################################################################
+xlator_dumpops['priv'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inode'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fd'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inodectx'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'ino',         'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fdctx'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['priv_to_dict'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inode_to_dict'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fd_to_dict'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inodectx_to_dict'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'ino',         'inode_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fdctx_to_dict'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('fn-arg',      'fd',          'fd_t *'),
+	('fn-arg',      'dict',        'dict_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['history'] = (
+	('fn-arg',      'this',        'xlator_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+def get_error_arg (type_str):
+	if type_str.find(" *") != -1:
+		return "NULL"
+	return "-1"
+
+def get_subs (names, types, cbktypes=None):
+	sdict = {}
+	sdict["@SHORT_ARGS@"] = ', '.join(names)
+	# Convert two separate tuples to one of (name, type) sub-tuples.
+	as_tuples = list(zip(types, names))
+	# Convert each sub-tuple into a "type name" string.
+	as_strings = [' '.join(item) for item in as_tuples]
+	# Join all of those into one big string.
+	sdict["@LONG_ARGS@"] = ',\n\t'.join(as_strings)
+	# So much more readable than string.join(map(string.join,zip(...))))
+	sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, types)))
+	if cbktypes is not None:
+		sdict["@CBK_ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, cbktypes)))
+	return sdict
+
+def generate (tmpl, name, subs):
+	text = tmpl.replace("@NAME@", name)
+	if name == "writev":
+		# More spurious inconsistency.
+		text = text.replace("@UPNAME@", "WRITE")
+	elif name == "readv":
+		text = text.replace("@UPNAME@", "READ")
+	else:
+		text = text.replace("@UPNAME@", name.upper())
+	for old, new in subs[name].items():
+		text = text.replace(old, new)
+	# TBD: reindent/reformat the result for maximum readability.
+	return text
+
+fop_subs = {}
+cbk_subs = {}
+
+for name, args in ops.items():
+
+	# Create the necessary substitution strings for fops.
+	arg_names = [ a[1] for a in args if a[0] == 'fop-arg']
+	arg_types = [ a[2] for a in args if a[0] == 'fop-arg']
+	cbk_types = [ a[2] for a in args if a[0] == 'cbk-arg']
+	fop_subs[name] = get_subs(arg_names, arg_types, cbk_types)
+
+	# Same thing for callbacks.
+	arg_names = [ a[1] for a in args if a[0] == 'cbk-arg']
+	arg_types = [ a[2] for a in args if a[0] == 'cbk-arg']
+	cbk_subs[name] = get_subs(arg_names, arg_types)
+
+	# Callers can add other subs to these tables, or even create their
+	# own tables, using these same techniques, and then pass the result
+	# to generate() which would Do The Right Thing with them.
diff --git a/libglusterfs/src/gf-dirent.c b/libglusterfs/src/gf-dirent.c
index af09cb06c0f..a809efc97ef 100644
--- a/libglusterfs/src/gf-dirent.c
+++ b/libglusterfs/src/gf-dirent.c
@@ -1,94 +1,301 @@
 /*
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include "glusterfs/compat.h"
+#include "glusterfs/syncop.h"
 
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+#define ONE 1ULL
+#define PRESENT_D_OFF_BITS 63
+#define BACKEND_D_OFF_BITS 63
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define MASK (~0ULL)
+#define SHIFT_BITS (max(0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
 
+static uint64_t
+bits_for(uint64_t num)
+{
+    uint64_t bits = 0, ctrl = 1;
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+    while (ctrl < num) {
+        ctrl *= 2;
+        bits++;
+    }
 
+    return bits;
+}
 
-#include <stdio.h>
-#include <string.h>
-#include <stdint.h>
-#include "compat.h"
-#include "xlator.h"
+int
+gf_deitransform(xlator_t *this, uint64_t offset)
+{
+    int cnt = 0;
+    int max = 0;
+    int max_bits = 0;
+    uint64_t off_mask = 0;
+    uint64_t host_mask = 0;
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        cnt = 0;
+        goto out;
+    }
+
+    if (offset & TOP_BIT) {
+        /* HUGE d_off */
+        max_bits = bits_for(max);
+        off_mask = (MASK << max_bits);
+        host_mask = ~(off_mask);
+
+        cnt = offset & host_mask;
+    } else {
+        /* small d_off */
+        cnt = offset % max;
+    }
+out:
+    return cnt;
+}
+
+uint64_t
+gf_dirent_orig_offset(xlator_t *this, uint64_t offset)
+{
+    int max = 0;
+    int max_bits = 0;
+    uint64_t off_mask = 0;
+    uint64_t orig_offset;
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        orig_offset = offset;
+        goto out;
+    }
+
+    if (offset & TOP_BIT) {
+        /* HUGE d_off */
+        max_bits = bits_for(max);
+        off_mask = (MASK << max_bits);
+        orig_offset = ((offset & ~TOP_BIT) & off_mask) << SHIFT_BITS;
+    } else {
+        /* small d_off */
+        orig_offset = offset / max;
+    }
+out:
+    return orig_offset;
+}
+
+int
+gf_itransform(xlator_t *this, uint64_t x, uint64_t *y_p, int client_id)
+{
+    int max = 0;
+    uint64_t y = 0;
+    uint64_t hi_mask = 0;
+    uint64_t off_mask = 0;
+    int max_bits = 0;
+
+    if (x == ((uint64_t)-1)) {
+        y = (uint64_t)-1;
+        goto out;
+    }
+
+    if (!x) {
+        y = 0;
+        goto out;
+    }
+
+    max = glusterfs_get_leaf_count(this->graph);
+
+    if (max == 1) {
+        y = x;
+        goto out;
+    }
+
+    max_bits = bits_for(max);
+
+    hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
+
+    if (x & hi_mask) {
+        /* HUGE d_off */
+        off_mask = MASK << max_bits;
+        y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | client_id;
+    } else {
+        /* small d_off */
+        y = ((x * max) + client_id);
+    }
+
+out:
+    if (y_p)
+        *y_p = y;
+
+    return 0;
+}
 
 gf_dirent_t *
-gf_dirent_for_namelen (int len)
+gf_dirent_for_name(const char *name)
 {
-	gf_dirent_t *gf_dirent = NULL;
+    gf_dirent_t *gf_dirent = NULL;
 
-	/* TODO: use mem-pool */
-	gf_dirent = CALLOC (len, sizeof(char));
-	if (!gf_dirent)
-		return NULL;
+    /* TODO: use mem-pool */
+    gf_dirent = GF_CALLOC(gf_dirent_size(name), 1, gf_common_mt_gf_dirent_t);
+    if (!gf_dirent)
+        return NULL;
 
-	INIT_LIST_HEAD (&gf_dirent->list);
+    INIT_LIST_HEAD(&gf_dirent->list);
+    strcpy(gf_dirent->d_name, name);
 
-	gf_dirent->d_off = 0;
-	gf_dirent->d_ino = -1;
-	gf_dirent->d_type = 0;
+    gf_dirent->d_off = 0;
+    gf_dirent->d_ino = -1;
+    gf_dirent->d_type = 0;
+    gf_dirent->d_len = strlen(name);
 
-	return gf_dirent;
+    return gf_dirent;
 }
 
+void
+gf_dirent_entry_free(gf_dirent_t *entry)
+{
+    if (!entry)
+        return;
+
+    if (entry->dict)
+        dict_unref(entry->dict);
+    if (entry->inode)
+        inode_unref(entry->inode);
+
+    list_del_init(&entry->list);
+    GF_FREE(entry);
+}
+
+void
+gf_dirent_free(gf_dirent_t *entries)
+{
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+
+    if (!entries)
+        return;
+
+    if (list_empty(&entries->list))
+        return;
+
+    list_for_each_entry_safe(entry, tmp, &entries->list, list)
+    {
+        gf_dirent_entry_free(entry);
+    }
+}
 
 gf_dirent_t *
-gf_dirent_for_name (const char *name)
+entry_copy(gf_dirent_t *source)
 {
-	gf_dirent_t *gf_dirent = NULL;
+    gf_dirent_t *sink = NULL;
 
-	/* TODO: use mem-pool */
-	gf_dirent = GF_CALLOC (gf_dirent_size (name), 1,
-                                gf_common_mt_gf_dirent_t);
-	if (!gf_dirent)
-		return NULL;
+    sink = gf_dirent_for_name(source->d_name);
+    if (!sink)
+        return NULL;
 
-	INIT_LIST_HEAD (&gf_dirent->list);
-	strcpy (gf_dirent->d_name, name);
+    sink->d_off = source->d_off;
+    sink->d_ino = source->d_ino;
+    sink->d_type = source->d_type;
+    sink->d_stat = source->d_stat;
+    sink->d_len = source->d_len;
 
-	gf_dirent->d_off = 0;
-	gf_dirent->d_ino = -1;
-	gf_dirent->d_type = 0;
-	gf_dirent->d_len = strlen (name);
+    if (source->inode)
+        sink->inode = inode_ref(source->inode);
 
-	return gf_dirent;
+    if (source->dict)
+        sink->dict = dict_ref(source->dict);
+    return sink;
 }
 
-
 void
-gf_dirent_free (gf_dirent_t *entries)
+gf_link_inode_from_dirent(xlator_t *this, inode_t *parent, gf_dirent_t *entry)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
+    inode_t *link_inode = NULL;
+    inode_t *tmp = NULL;
+
+    if (!entry->inode)
+        return;
+    link_inode = inode_link(entry->inode, parent, entry->d_name,
+                            &entry->d_stat);
+    if (!link_inode)
+        return;
 
-        if (!entries)
-                return;
+    inode_lookup(link_inode);
+    tmp = entry->inode;
+    entry->inode = link_inode;
+    inode_unref(tmp);
+}
+
+/* TODO: Currently, with this function, we will be breaking the
+   policy of 1-1 mapping of kernel nlookup refs with our inode_t's
+   nlookup count.
+   Need more thoughts before finalizing this function
+*/
+int
+gf_link_inodes_from_dirent(xlator_t *this, inode_t *parent,
+                           gf_dirent_t *entries)
+{
+    gf_dirent_t *entry = NULL;
 
-        if (list_empty (&entries->list))
-                return;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        gf_link_inode_from_dirent(this, parent, entry);
+    }
 
-	list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-		list_del (&entry->list);
-		GF_FREE (entry);
-	}
+    return 0;
 }
 
+int
+gf_fill_iatt_for_dirent(gf_dirent_t *entry, inode_t *parent, xlator_t *subvol)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    loc.inode = inode_grep(parent->table, parent, entry->d_name);
+    if (!loc.inode) {
+        loc.inode = inode_new(parent->table);
+        gf_uuid_copy(loc.inode->gfid, entry->d_stat.ia_gfid);
+    }
+
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = entry->d_name;
+    loc.parent = inode_ref(parent);
+    ret = inode_path(loc.parent, entry->d_name, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
 
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+
+    entry->d_stat = iatt;
+    entry->inode = inode_ref(loc.inode);
+    /* We don't need to link inode here, because as part of readdirp_cbk
+     * we will link all dirents.
+     *
+     * Since we did a proper lookup, we don't need to set need_lookup
+     * flag.
+     */
+
+    ret = 0;
+out:
+    loc_wipe(&loc);
+    return ret;
+}
diff --git a/libglusterfs/src/gf-dirent.h b/libglusterfs/src/gf-dirent.h
deleted file mode 100644
index 8a6533fc346..00000000000
--- a/libglusterfs/src/gf-dirent.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _GF_DIRENT_H
-#define _GF_DIRENT_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "iatt.h"
-
-#define gf_dirent_size(name) (sizeof (gf_dirent_t) + strlen (name) + 1)
-
-struct _dir_entry_t {
-        struct _dir_entry_t *next;
-	char                *name;
-	char                *link;
-	struct iatt          buf;
-};
-
-
-struct _gf_dirent_t {
-	union {
-		struct list_head             list;
-		struct {
-			struct _gf_dirent_t *next;
-			struct _gf_dirent_t *prev;
-		};
-	};
-	uint64_t                             d_ino;
-	uint64_t                             d_off;
-	uint32_t                             d_len;
-	uint32_t                             d_type;
-        struct iatt                          d_stat;
-	char                                 d_name[0];
-};
-
-
-gf_dirent_t *gf_dirent_for_name (const char *name);
-void gf_dirent_free (gf_dirent_t *entries);
-gf_dirent_t * gf_dirent_for_namelen (int len);
-
-#endif /* _GF_DIRENT_H */
diff --git a/libglusterfs/src/gidcache.c b/libglusterfs/src/gidcache.c
new file mode 100644
index 00000000000..64a93802f76
--- /dev/null
+++ b/libglusterfs/src/gidcache.c
@@ -0,0 +1,211 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/gidcache.h"
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/common-utils.h"
+
+/*
+ * We treat this as a very simple set-associative LRU cache, with entries aged
+ * out after a configurable interval.  Hardly rocket science, but lots of
+ * details to worry about.
+ */
+#define BUCKET_START(p, n) ((p) + ((n)*AUX_GID_CACHE_ASSOC))
+
+/*
+ * Initialize the cache.
+ */
+int
+gid_cache_init(gid_cache_t *cache, uint32_t timeout)
+{
+    if (!cache)
+        return -1;
+
+    LOCK_INIT(&cache->gc_lock);
+    cache->gc_max_age = timeout;
+    cache->gc_nbuckets = AUX_GID_CACHE_BUCKETS;
+    memset(cache->gc_cache, 0, sizeof(gid_list_t) * AUX_GID_CACHE_SIZE);
+
+    return 0;
+}
+
+/*
+ * Reconfigure the cache timeout.
+ */
+int
+gid_cache_reconf(gid_cache_t *cache, uint32_t timeout)
+{
+    if (!cache)
+        return -1;
+
+    LOCK(&cache->gc_lock);
+    cache->gc_max_age = timeout;
+    UNLOCK(&cache->gc_lock);
+
+    return 0;
+}
+
+/*
+ * Look up an ID in the cache. If found, return the actual cache entry to avoid
+ * an additional allocation and memory copy. The caller should copy the data and
+ * release (unlock) the cache as soon as possible.
+ */
+const gid_list_t *
+gid_cache_lookup(gid_cache_t *cache, uint64_t id, uint64_t uid, uint64_t gid)
+{
+    int bucket;
+    int i;
+    time_t now;
+    const gid_list_t *agl;
+
+    now = gf_time();
+    LOCK(&cache->gc_lock);
+    bucket = id % cache->gc_nbuckets;
+    agl = BUCKET_START(cache->gc_cache, bucket);
+    for (i = 0; i < AUX_GID_CACHE_ASSOC; i++, agl++) {
+        if (!agl->gl_list)
+            continue;
+        if (agl->gl_id != id)
+            continue;
+
+        /*
+          @uid and @gid reflect the latest UID/GID of the
+           process performing the syscall (taken from frame->root).
+
+           If the UID and GID has changed for the PID since the
+           time we cached it, we should treat the cache as having
+           stale values and query them freshly.
+        */
+        if (agl->gl_uid != uid || agl->gl_gid != gid)
+            break;
+
+        /*
+         * We don't put new entries in the cache when expiration=0, but
+         * there might be entries still in there if expiration was
+         * changed very recently.  Writing the check this way ensures
+         * that they're not used.
+         */
+        if (now < agl->gl_deadline) {
+            return agl;
+        }
+
+        /*
+         * We're not going to find any more UID matches, and reaping
+         * is handled further down to maintain LRU order.
+         */
+        break;
+    }
+    UNLOCK(&cache->gc_lock);
+    return NULL;
+}
+
+/*
+ * Release an entry found via lookup.
+ */
+void
+gid_cache_release(gid_cache_t *cache, const gid_list_t *agl)
+{
+    UNLOCK(&cache->gc_lock);
+}
+
+/*
+ * Add a new list entry to the cache. If an entry for this ID already exists,
+ * update it.
+ */
+int
+gid_cache_add(gid_cache_t *cache, gid_list_t *gl)
+{
+    gid_list_t *agl;
+    int bucket;
+    int i;
+    time_t now;
+
+    if (!gl || !gl->gl_list)
+        return -1;
+
+    if (!cache->gc_max_age)
+        return 0;
+
+    now = gf_time();
+    LOCK(&cache->gc_lock);
+
+    /*
+     * Scan for the first free entry or one that matches this id. The id
+     * check is added to address a bug where the cache might contain an
+     * expired entry for this id. Since lookup occurs in LRU order and
+     * does not reclaim entries, it will always return failure on discovery
+     * of an expired entry. This leads to duplicate entries being added,
+     * which still do not satisfy lookups until the expired entry (and
+     * everything before it) is reclaimed.
+     *
+     * We address this through reuse of an entry already allocated to this
+     * id, whether expired or not, since we have obviously already received
+     * more recent data. The entry is repopulated with the new data and a new
+     * deadline and is pushed forward to reside as the last populated entry in
+     * the bucket.
+     */
+    bucket = gl->gl_id % cache->gc_nbuckets;
+    agl = BUCKET_START(cache->gc_cache, bucket);
+    for (i = 0; i < AUX_GID_CACHE_ASSOC; ++i, ++agl) {
+        if (agl->gl_id == gl->gl_id)
+            break;
+        if (!agl->gl_list)
+            break;
+    }
+
+    /*
+     * The way we allocate free entries naturally places the newest
+     * ones at the highest indices, so evicting the lowest makes
+     * sense, but that also means we can't just replace it with the
+     * one that caused the eviction.  That would cause us to thrash
+     * the first entry while others remain idle.  Therefore, we
+     * need to slide the other entries down and add the new one at
+     * the end just as if the *last* slot had been free.
+     *
+     * Deadline expiration is also handled here, since the oldest
+     * expired entry will be in the first position.  This does mean
+     * the bucket can stay full of expired entries if we're idle
+     * but, if the small amount of extra memory or scan time before
+     * we decide to evict someone ever become issues, we could
+     * easily add a reaper thread.
+     */
+
+    if (i >= AUX_GID_CACHE_ASSOC) {
+        /* cache full, evict the first (LRU) entry */
+        i = 0;
+        agl = BUCKET_START(cache->gc_cache, bucket);
+        GF_FREE(agl->gl_list);
+    } else if (agl->gl_list) {
+        /* evict the old entry we plan to reuse */
+        GF_FREE(agl->gl_list);
+    }
+
+    /*
+     * If we have evicted an entry, slide the subsequent populated entries
+     * back and populate the last entry.
+     */
+    for (; i < AUX_GID_CACHE_ASSOC - 1; i++) {
+        if (!agl[1].gl_list)
+            break;
+        agl[0] = agl[1];
+        agl++;
+    }
+
+    agl->gl_id = gl->gl_id;
+    agl->gl_uid = gl->gl_uid;
+    agl->gl_gid = gl->gl_gid;
+    agl->gl_count = gl->gl_count;
+    agl->gl_list = gl->gl_list;
+    agl->gl_deadline = now + cache->gc_max_age;
+
+    UNLOCK(&cache->gc_lock);
+
+    return 1;
+}
diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c
index 979f84e3296..ae06f8be386 100644
--- a/libglusterfs/src/globals.c
+++ b/libglusterfs/src/globals.c
@@ -1,359 +1,366 @@
 /*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* !_CONFIG_H */
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "globals.h"
-#include "xlator.h"
-#include "mem-pool.h"
+#include "glusterfs/syncop.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+const char *gf_fop_list[GF_FOP_MAXVALUE] = {
+    [GF_FOP_NULL] = "NULL",
+    [GF_FOP_STAT] = "STAT",
+    [GF_FOP_READLINK] = "READLINK",
+    [GF_FOP_MKNOD] = "MKNOD",
+    [GF_FOP_MKDIR] = "MKDIR",
+    [GF_FOP_UNLINK] = "UNLINK",
+    [GF_FOP_RMDIR] = "RMDIR",
+    [GF_FOP_SYMLINK] = "SYMLINK",
+    [GF_FOP_RENAME] = "RENAME",
+    [GF_FOP_LINK] = "LINK",
+    [GF_FOP_TRUNCATE] = "TRUNCATE",
+    [GF_FOP_OPEN] = "OPEN",
+    [GF_FOP_READ] = "READ",
+    [GF_FOP_WRITE] = "WRITE",
+    [GF_FOP_STATFS] = "STATFS",
+    [GF_FOP_FLUSH] = "FLUSH",
+    [GF_FOP_FSYNC] = "FSYNC",
+    [GF_FOP_SETXATTR] = "SETXATTR",
+    [GF_FOP_GETXATTR] = "GETXATTR",
+    [GF_FOP_REMOVEXATTR] = "REMOVEXATTR",
+    [GF_FOP_OPENDIR] = "OPENDIR",
+    [GF_FOP_FSYNCDIR] = "FSYNCDIR",
+    [GF_FOP_ACCESS] = "ACCESS",
+    [GF_FOP_CREATE] = "CREATE",
+    [GF_FOP_FTRUNCATE] = "FTRUNCATE",
+    [GF_FOP_FSTAT] = "FSTAT",
+    [GF_FOP_LK] = "LK",
+    [GF_FOP_LOOKUP] = "LOOKUP",
+    [GF_FOP_READDIR] = "READDIR",
+    [GF_FOP_INODELK] = "INODELK",
+    [GF_FOP_FINODELK] = "FINODELK",
+    [GF_FOP_ENTRYLK] = "ENTRYLK",
+    [GF_FOP_FENTRYLK] = "FENTRYLK",
+    [GF_FOP_XATTROP] = "XATTROP",
+    [GF_FOP_FXATTROP] = "FXATTROP",
+    [GF_FOP_FSETXATTR] = "FSETXATTR",
+    [GF_FOP_FGETXATTR] = "FGETXATTR",
+    [GF_FOP_RCHECKSUM] = "RCHECKSUM",
+    [GF_FOP_SETATTR] = "SETATTR",
+    [GF_FOP_FSETATTR] = "FSETATTR",
+    [GF_FOP_READDIRP] = "READDIRP",
+    [GF_FOP_GETSPEC] = "GETSPEC",
+    [GF_FOP_FORGET] = "FORGET",
+    [GF_FOP_RELEASE] = "RELEASE",
+    [GF_FOP_RELEASEDIR] = "RELEASEDIR",
+    [GF_FOP_FREMOVEXATTR] = "FREMOVEXATTR",
+    [GF_FOP_FALLOCATE] = "FALLOCATE",
+    [GF_FOP_DISCARD] = "DISCARD",
+    [GF_FOP_ZEROFILL] = "ZEROFILL",
+    [GF_FOP_IPC] = "IPC",
+    [GF_FOP_SEEK] = "SEEK",
+    [GF_FOP_LEASE] = "LEASE",
+    [GF_FOP_COMPOUND] = "COMPOUND",
+    [GF_FOP_GETACTIVELK] = "GETACTIVELK",
+    [GF_FOP_SETACTIVELK] = "SETACTIVELK",
+    [GF_FOP_PUT] = "PUT",
+    [GF_FOP_ICREATE] = "ICREATE",
+    [GF_FOP_NAMELINK] = "NAMELINK",
+    [GF_FOP_COPY_FILE_RANGE] = "COPY_FILE_RANGE",
+};
 
+const char *gf_upcall_list[GF_UPCALL_FLAGS_MAXVALUE] = {
+    [GF_UPCALL_NULL] = "NULL",
+    [GF_UPCALL] = "UPCALL",
+    [GF_UPCALL_CI_STAT] = "CI_IATT",
+    [GF_UPCALL_CI_XATTR] = "CI_XATTR",
+    [GF_UPCALL_CI_RENAME] = "CI_RENAME",
+    [GF_UPCALL_CI_NLINK] = "CI_UNLINK",
+    [GF_UPCALL_CI_FORGET] = "CI_FORGET",
+    [GF_UPCALL_LEASE_RECALL] = "LEASE_RECALL",
+};
 
-/* gf_*_list[] */
+/* THIS */
 
-char *gf_fop_list[GF_FOP_MAXVALUE];
-char *gf_mgmt_list[GF_MGMT_MAXVALUE];
+/* This global ctx is a bad hack to prevent some of the libgfapi crashes.
+ * This should be removed once the patch on resource pool is accepted
+ */
+glusterfs_ctx_t *global_ctx = NULL;
+pthread_mutex_t global_ctx_mutex = PTHREAD_MUTEX_INITIALIZER;
+xlator_t global_xlator;
+static int gf_global_mem_acct_enable = 1;
+static pthread_once_t globals_inited = PTHREAD_ONCE_INIT;
 
+static pthread_key_t free_key;
 
-void
-gf_op_list_init()
+static __thread xlator_t *thread_xlator = NULL;
+static __thread void *thread_synctask = NULL;
+static __thread void *thread_leaseid = NULL;
+static __thread struct syncopctx thread_syncopctx = {};
+static __thread char thread_uuid_buf[GF_UUID_BUF_SIZE] = {};
+static __thread char thread_lkowner_buf[GF_LKOWNER_BUF_SIZE] = {};
+static __thread char thread_leaseid_buf[GF_LEASE_ID_BUF_SIZE] = {};
+
+int
+gf_global_mem_acct_enable_get(void)
 {
-	gf_fop_list[GF_FOP_NULL]        = "NULL";
-	gf_fop_list[GF_FOP_STAT]        = "STAT";
-	gf_fop_list[GF_FOP_READLINK]    = "READLINK";
-	gf_fop_list[GF_FOP_MKNOD]       = "MKNOD";
-	gf_fop_list[GF_FOP_MKDIR]       = "MKDIR";
-	gf_fop_list[GF_FOP_UNLINK]      = "UNLINK";
-	gf_fop_list[GF_FOP_RMDIR]       = "RMDIR";
-	gf_fop_list[GF_FOP_SYMLINK]     = "SYMLINK";
-	gf_fop_list[GF_FOP_RENAME]      = "RENAME";
-	gf_fop_list[GF_FOP_LINK]        = "LINK";
-	gf_fop_list[GF_FOP_TRUNCATE]    = "TRUNCATE";
-	gf_fop_list[GF_FOP_OPEN]        = "OPEN";
-	gf_fop_list[GF_FOP_READ]        = "READ";
-	gf_fop_list[GF_FOP_WRITE]       = "WRITE";
-	gf_fop_list[GF_FOP_STATFS]      = "STATFS";
-	gf_fop_list[GF_FOP_FLUSH]       = "FLUSH";
-	gf_fop_list[GF_FOP_FSYNC]       = "FSYNC";
-	gf_fop_list[GF_FOP_SETXATTR]    = "SETXATTR";
-	gf_fop_list[GF_FOP_GETXATTR]    = "GETXATTR";
-	gf_fop_list[GF_FOP_REMOVEXATTR] = "REMOVEXATTR";
-	gf_fop_list[GF_FOP_OPENDIR]     = "OPENDIR";
-	gf_fop_list[GF_FOP_FSYNCDIR]    = "FSYNCDIR";
-	gf_fop_list[GF_FOP_ACCESS]      = "ACCESS";
-	gf_fop_list[GF_FOP_CREATE]      = "CREATE";
-	gf_fop_list[GF_FOP_FTRUNCATE]   = "FTRUNCATE";
-	gf_fop_list[GF_FOP_FSTAT]       = "FSTAT";
-	gf_fop_list[GF_FOP_LK]          = "LK";
-	gf_fop_list[GF_FOP_LOOKUP]      = "LOOKUP";
-	gf_fop_list[GF_FOP_READDIR]     = "READDIR";
-	gf_fop_list[GF_FOP_INODELK]     = "INODELK";
-	gf_fop_list[GF_FOP_FINODELK]    = "FINODELK";
-	gf_fop_list[GF_FOP_ENTRYLK]     = "ENTRYLK";
-	gf_fop_list[GF_FOP_FENTRYLK]    = "FENTRYLK";
-	gf_fop_list[GF_FOP_XATTROP]     = "XATTROP";
-	gf_fop_list[GF_FOP_FXATTROP]    = "FXATTROP";
-	gf_fop_list[GF_FOP_FSETXATTR]   = "FSETXATTR";
-	gf_fop_list[GF_FOP_FGETXATTR]   = "FGETXATTR";
-        gf_fop_list[GF_FOP_RCHECKSUM]   = "RCHECKSUM";
-        gf_fop_list[GF_FOP_SETATTR]     = "SETATTR";
-        gf_fop_list[GF_FOP_FSETATTR]    = "FSETATTR";
-	gf_fop_list[GF_FOP_READDIRP]    = "READDIRP";
-	gf_fop_list[GF_FOP_GETSPEC]     = "GETSPEC";
-	gf_fop_list[GF_FOP_FORGET]      = "FORGET";
-	gf_fop_list[GF_FOP_RELEASE]     = "RELEASE";
-	gf_fop_list[GF_FOP_RELEASEDIR]  = "RELEASEDIR";
-
-	gf_fop_list[GF_MGMT_NULL]  = "NULL";
-	return;
+    return gf_global_mem_acct_enable;
 }
 
+int
+gf_global_mem_acct_enable_set(int val)
+{
+    gf_global_mem_acct_enable = val;
+    return 0;
+}
 
-/* CTX */
-static glusterfs_ctx_t *glusterfs_ctx;
+static struct xlator_cbks global_cbks = {
+    .forget = NULL,
+    .release = NULL,
+    .releasedir = NULL,
+    .invalidate = NULL,
+    .client_destroy = NULL,
+    .client_disconnect = NULL,
+    .ictxmerge = NULL,
+    .ictxsize = NULL,
+    .fdctxsize = NULL,
+};
 
+/* This is required to get through the check in graph.c */
+static struct xlator_fops global_fops = {};
 
-int
-glusterfs_ctx_init ()
+static int
+global_xl_reconfigure(xlator_t *this, dict_t *options)
 {
-        int  ret = 0;
+    int ret = -1;
+    gf_boolean_t bool_opt = _gf_false;
 
-        if (glusterfs_ctx)
-                goto out;
+    /* This is not added in volume dump, hence adding the options in log
+       would be helpful for debugging later */
+    dict_dump_to_log(options);
 
-        glusterfs_ctx = CALLOC (1, sizeof (*glusterfs_ctx));
-        if (!glusterfs_ctx) {
-                ret = -1;
-                goto out;
-        }
+    GF_OPTION_RECONF("measure-latency", bool_opt, options, bool, out);
+    this->ctx->measure_latency = bool_opt;
 
-        INIT_LIST_HEAD (&glusterfs_ctx->graphs);
-        ret = pthread_mutex_init (&glusterfs_ctx->lock, NULL);
+    GF_OPTION_RECONF("metrics-dump-path", this->ctx->config.metrics_dumppath,
+                     options, str, out);
 
+    /* TODO: add more things here */
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
-glusterfs_ctx_t *
-glusterfs_ctx_get ()
+static int
+global_xl_init(xlator_t *this)
 {
-        return glusterfs_ctx;
+    int ret = -1;
+    gf_boolean_t bool_opt = false;
 
-}
+    GF_OPTION_INIT("measure-latency", bool_opt, bool, out);
+    this->ctx->measure_latency = bool_opt;
 
+    GF_OPTION_INIT("metrics-dump-path", this->ctx->config.metrics_dumppath, str,
+                   out);
 
-/* THIS */
+    ret = 0;
 
-xlator_t global_xlator;
-static pthread_key_t this_xlator_key;
-
-void
-glusterfs_this_destroy (void *ptr)
-{
-        if (ptr)
-                FREE (ptr);
+out:
+    return ret;
 }
 
-
-int
-glusterfs_this_init ()
+static void
+global_xl_fini(xlator_t *this)
 {
-        int  ret = 0;
-
-        ret = pthread_key_create (&this_xlator_key, glusterfs_this_destroy);
-        if (ret != 0) {
-                return ret;
-        }
-
-        global_xlator.name = "glusterfs";
-        global_xlator.type = "global";
-        global_xlator.ctx  = glusterfs_ctx;
-
-        return ret;
+    return;
 }
 
+struct volume_options global_xl_options[] = {
+    {.key = {"measure-latency"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"global", "context"},
+     .description = "Use this option to toggle measuring latency"},
+    {.key = {"metrics-dump-path"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "{{gluster_workdir}}/metrics",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"global", "context"},
+     .description = "Use this option to set the metrics dump path"},
+
+    {
+        .key = {NULL},
+    },
+};
 
-xlator_t **
-__glusterfs_this_location ()
+static volume_opt_list_t global_xl_opt_list;
+
+void
+glusterfs_this_init()
 {
-        xlator_t **this_location = NULL;
-        int        ret = 0;
-
-        this_location = pthread_getspecific (this_xlator_key);
-
-        if (!this_location) {
-                this_location = CALLOC (1, sizeof (*this_location));
-                if (!this_location)
-                        goto out;
-
-                ret = pthread_setspecific (this_xlator_key, this_location);
-                if (ret != 0) {
-                        FREE (this_location);
-                        this_location = NULL;
-                        goto out;
-                }
-        }
-out:
-        if (this_location) {
-                if (!*this_location)
-                        *this_location = &global_xlator;
-        }
-        return this_location;
+    global_xlator.name = "glusterfs";
+    global_xlator.type = GF_GLOBAL_XLATOR_NAME;
+    global_xlator.cbks = &global_cbks;
+    global_xlator.fops = &global_fops;
+    global_xlator.reconfigure = global_xl_reconfigure;
+    global_xlator.init = global_xl_init;
+    global_xlator.fini = global_xl_fini;
+
+    INIT_LIST_HEAD(&global_xlator.volume_options);
+    INIT_LIST_HEAD(&global_xl_opt_list.list);
+    global_xl_opt_list.given_opt = global_xl_options;
+
+    list_add_tail(&global_xl_opt_list.list, &global_xlator.volume_options);
 }
 
-
-xlator_t *
-glusterfs_this_get ()
+xlator_t **
+__glusterfs_this_location()
 {
-        xlator_t **this_location = NULL;
+    xlator_t **this_location;
 
-        this_location = __glusterfs_this_location ();
-        if (!this_location)
-                return &global_xlator;
+    this_location = &thread_xlator;
+    if (*this_location == NULL) {
+        thread_xlator = &global_xlator;
+    }
 
-        return *this_location;
+    return this_location;
 }
 
-
-int
-glusterfs_this_set (xlator_t *this)
+xlator_t *
+glusterfs_this_get()
 {
-        xlator_t **this_location = NULL;
-
-        this_location = __glusterfs_this_location ();
-        if (!this_location)
-                return -ENOMEM;
-
-        *this_location = this;
-
-        return 0;
+    return *__glusterfs_this_location();
 }
 
-
-/* IS_CENTRAL_LOG */
-
-static pthread_key_t central_log_flag_key;
-
 void
-glusterfs_central_log_flag_destroy (void *ptr)
+glusterfs_this_set(xlator_t *this)
 {
-        if (ptr)
-                FREE (ptr);
+    thread_xlator = this;
 }
 
+/* SYNCOPCTX */
 
-int
-glusterfs_central_log_flag_init ()
+void *
+syncopctx_getctx()
 {
-        int ret = 0;
-
-        ret = pthread_key_create (&central_log_flag_key,
-                                  glusterfs_central_log_flag_destroy);
-
-        if (ret != 0) {
-                return ret;
-        }
+    return &thread_syncopctx;
+}
 
-        pthread_setspecific (central_log_flag_key, (void *) 0);
+/* SYNCTASK */
 
-        return ret;
+void *
+synctask_get()
+{
+    return thread_synctask;
 }
 
-
 void
-glusterfs_central_log_flag_set ()
+synctask_set(void *synctask)
 {
-        pthread_setspecific (central_log_flag_key, (void *) 1);
+    thread_synctask = synctask;
 }
 
+// UUID_BUFFER
 
-long
-glusterfs_central_log_flag_get ()
+char *
+glusterfs_uuid_buf_get()
 {
-        long flag = 0;
-
-        flag = (long) pthread_getspecific (central_log_flag_key);
-
-        return flag;
+    return thread_uuid_buf;
 }
 
+/* LKOWNER_BUFFER */
 
-void
-glusterfs_central_log_flag_unset ()
+char *
+glusterfs_lkowner_buf_get()
 {
-        pthread_setspecific (central_log_flag_key, (void *) 0);
+    return thread_lkowner_buf;
 }
 
+/* Leaseid buffer */
 
-
-/* SYNCTASK */
-
-static pthread_key_t synctask_key;
-
-
-int
-synctask_init ()
+char *
+glusterfs_leaseid_buf_get()
 {
-        int  ret = 0;
+    char *buf = NULL;
 
-        ret = pthread_key_create (&synctask_key, NULL);
+    buf = thread_leaseid;
+    if (buf == NULL) {
+        buf = thread_leaseid_buf;
+        thread_leaseid = buf;
+    }
 
-        return ret;
+    return buf;
 }
 
-
-void *
-synctask_get ()
+char *
+glusterfs_leaseid_exist()
 {
-        void   *synctask = NULL;
-
-        synctask = pthread_getspecific (synctask_key);
-
-        return synctask;
+    return thread_leaseid;
 }
 
-
-int
-synctask_set (void *synctask)
+static void
+glusterfs_cleanup(void *ptr)
 {
-        int     ret = 0;
+    if (thread_syncopctx.groups != NULL) {
+        GF_FREE(thread_syncopctx.groups);
+    }
 
-        pthread_setspecific (synctask_key, synctask);
-
-        return ret;
+    mem_pool_thread_destructor(NULL);
 }
 
-
-int
-glusterfs_globals_init ()
+void
+gf_thread_needs_cleanup(void)
 {
-        int ret = 0;
+    /* The value stored in free_key TLS is not really used for anything, but
+     * pthread implementation doesn't call the TLS destruction function unless
+     * it's != NULL. This function must be called whenever something is
+     * allocated for this thread so that glusterfs_cleanup() will be called
+     * and resources can be released. */
+    (void)pthread_setspecific(free_key, (void *)1);
+}
 
-        gf_op_list_init ();
+static void
+gf_globals_init_once()
+{
+    int ret = 0;
 
-        ret = glusterfs_ctx_init ();
-        if (ret)
-                goto out;
+    glusterfs_this_init();
 
-        ret = glusterfs_this_init ();
-        if (ret)
-                goto out;
+    /* This is needed only to cleanup the potential allocation of
+     * thread_syncopctx.groups. */
+    ret = pthread_key_create(&free_key, glusterfs_cleanup);
+    if (ret != 0) {
+        gf_msg("", GF_LOG_ERROR, ret, LG_MSG_PTHREAD_KEY_CREATE_FAILED,
+               "failed to create the pthread key");
 
-        ret = glusterfs_central_log_flag_init ();
-        if (ret)
-                goto out;
+        gf_msg("", GF_LOG_CRITICAL, 0, LG_MSG_GLOBAL_INIT_FAILED,
+               "Exiting as global initialization failed");
 
-        gf_mem_acct_enable_set ();
+        exit(ret);
+    }
+}
 
-        ret = synctask_init ();
-        if (ret)
-                goto out;
+int
+glusterfs_globals_init(glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
 
-out:
-        return ret;
-}
+    gf_log_globals_init(ctx, GF_LOG_INFO);
 
+    ret = pthread_once(&globals_inited, gf_globals_init_once);
 
-char eventstring[GF_EVENT_MAXVAL][64] = {
-        "Invalid event",
-        "Parent Up",
-        "Poll In",
-        "Poll Out",
-        "Poll Err",
-        "Child Up",
-        "Child Down",
-        "Child Connecting",
-        "Transport Cleanup",
-        "Transport Connected",
-        "Volfile Modified",
-};
+    if (ret)
+        gf_msg("", GF_LOG_CRITICAL, ret, LG_MSG_PTHREAD_FAILED,
+               "pthread_once failed");
 
-/* Copy the string ptr contents if needed for yourself */
-char *
-glusterfs_strevent (glusterfs_event_t ev)
-{
-        return eventstring[ev];
+    return ret;
 }
-
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
deleted file mode 100644
index 96a11362372..00000000000
--- a/libglusterfs/src/globals.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _GLOBALS_H
-#define _GLOBALS_H
-
-#define GF_DEFAULT_BASE_PORT 24007
-
-/* This corresponds to the max 16 number of group IDs that are sent through an
- * RPC request. Since NFS is the only one going to set this, we can be safe
- * in keeping this size hardcoded.
- */
-#define GF_REQUEST_MAXGROUPS    16
-
-#include "glusterfs.h"
-
-/* CTX */
-#define CTX (glusterfs_ctx_get())
-
-glusterfs_ctx_t *glusterfs_ctx_get ();
-
-#include "xlator.h"
-
-/* THIS */
-#define THIS (*__glusterfs_this_location())
-
-xlator_t **__glusterfs_this_location ();
-xlator_t *glusterfs_this_get ();
-int glusterfs_this_set (xlator_t *);
-
-/* central log */
-
-void glusterfs_central_log_flag_set ();
-long glusterfs_central_log_flag_get ();
-void glusterfs_central_log_flag_unset ();
-
-/* task */
-void *synctask_get ();
-int synctask_set (void *);
-
-/* init */
-int glusterfs_globals_init (void);
-
-#endif /* !_GLOBALS_H */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
deleted file mode 100644
index 2acd3d9d589..00000000000
--- a/libglusterfs/src/glusterfs.h
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _GLUSTERFS_H
-#define _GLUSTERFS_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <netdb.h>
-#include <errno.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <arpa/inet.h>
-#include <sys/poll.h>
-#include <pthread.h>
-
-#include "list.h"
-#include "logging.h"
-
-
-#define GF_YES 1
-#define GF_NO  0
-
-#ifndef O_LARGEFILE
-/* savannah bug #20053, patch for compiling on darwin */
-#define O_LARGEFILE 0
-#endif
-
-#ifndef O_DIRECT
-/* savannah bug #20050, #20052 */
-#define O_DIRECT 0 /* From asm/fcntl.h */
-#endif
-
-#ifndef O_DIRECTORY
-/* FreeBSD does not need O_DIRECTORY */
-#define O_DIRECTORY 0
-#endif
-
-#ifndef FNM_EXTMATCH
-#define FNM_EXTMATCH 0
-#endif
-
-#define GF_XATTR_PATHINFO_KEY   "trusted.glusterfs.pathinfo"
-#define GF_XATTR_LINKINFO_KEY   "trusted.distribute.linkinfo"
-
-#define ZR_FILE_CONTENT_STR     "glusterfs.file."
-#define ZR_FILE_CONTENT_STRLEN 15
-
-#define GLUSTERFS_OPEN_FD_COUNT "glusterfs.open-fd-count"
-#define GLUSTERFS_INODELK_COUNT "glusterfs.inodelk-count"
-#define GLUSTERFS_ENTRYLK_COUNT "glusterfs.entrylk-count"
-#define GLUSTERFS_POSIXLK_COUNT "glusterfs.posixlk-count"
-#define GLUSTERFS_RDMA_INLINE_THRESHOLD       (2048)
-#define GLUSTERFS_RDMA_MAX_HEADER_SIZE        (228) /* (sizeof (rdma_header_t)                 \
-                                                       + RDMA_MAX_SEGMENTS \
-                                                       * sizeof (rdma_read_chunk_t))
-                                                       */
-
-#define GLUSTERFS_RPC_REPLY_SIZE               24
-
-#define ZR_FILE_CONTENT_REQUEST(key) (!strncmp(key, ZR_FILE_CONTENT_STR, \
-					       ZR_FILE_CONTENT_STRLEN))
-
-/* TODO: Should we use PATH-MAX? On some systems it may save space */
-#define ZR_PATH_MAX 4096
-
-/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
-typedef enum {
-        GF_FOP_NULL = 0,
-        GF_FOP_STAT,
-        GF_FOP_READLINK,
-        GF_FOP_MKNOD,
-        GF_FOP_MKDIR,
-        GF_FOP_UNLINK,
-        GF_FOP_RMDIR,
-        GF_FOP_SYMLINK,
-        GF_FOP_RENAME,
-        GF_FOP_LINK,
-        GF_FOP_TRUNCATE,
-        GF_FOP_OPEN,
-        GF_FOP_READ,
-        GF_FOP_WRITE,
-        GF_FOP_STATFS,
-        GF_FOP_FLUSH,
-        GF_FOP_FSYNC,      /* 15 */
-        GF_FOP_SETXATTR,
-        GF_FOP_GETXATTR,
-        GF_FOP_REMOVEXATTR,
-        GF_FOP_OPENDIR,
-        GF_FOP_FSYNCDIR,
-        GF_FOP_ACCESS,
-        GF_FOP_CREATE,
-        GF_FOP_FTRUNCATE,
-        GF_FOP_FSTAT,      /* 25 */
-        GF_FOP_LK,
-        GF_FOP_LOOKUP,
-        GF_FOP_READDIR,
-        GF_FOP_INODELK,
-        GF_FOP_FINODELK,
-	GF_FOP_ENTRYLK,
-	GF_FOP_FENTRYLK,
-        GF_FOP_XATTROP,
-        GF_FOP_FXATTROP,
-        GF_FOP_FGETXATTR,
-        GF_FOP_FSETXATTR,
-        GF_FOP_RCHECKSUM,
-        GF_FOP_SETATTR,
-        GF_FOP_FSETATTR,
-        GF_FOP_READDIRP,
-        GF_FOP_FORGET,
-        GF_FOP_RELEASE,
-        GF_FOP_RELEASEDIR,
-        GF_FOP_GETSPEC,
-        GF_FOP_MAXVALUE,
-} glusterfs_fop_t;
-
-
-typedef enum {
-        GF_MGMT_NULL = 0,
-        GF_MGMT_MAXVALUE,
-} glusterfs_mgmt_t;
-
-typedef enum {
-        GF_OP_TYPE_NULL = 0,
-        GF_OP_TYPE_FOP,
-        GF_OP_TYPE_MGMT,
-        GF_OP_TYPE_MAX,
-} gf_op_type_t;
-
-struct gf_flock {
-        short    l_type;
-        short    l_whence;
-        off_t    l_start;
-        off_t    l_len;
-        pid_t    l_pid;
-        uint64_t l_owner;
-};
-
-/* NOTE: all the miscellaneous flags used by GlusterFS should be listed here */
-typedef enum {
-        GF_LK_GETLK = 0,
-        GF_LK_SETLK,
-        GF_LK_SETLKW,
-        GF_LK_RESLK_LCK,
-        GF_LK_RESLK_LCKW,
-        GF_LK_RESLK_UNLCK,
-        GF_LK_GETLK_FD,
-} glusterfs_lk_cmds_t;
-
-
-typedef enum {
-        GF_LK_F_RDLCK = 0,
-        GF_LK_F_WRLCK,
-        GF_LK_F_UNLCK,
-        GF_LK_EOL,
-} glusterfs_lk_types_t;
-
-typedef enum {
-        F_RESLK_LCK = 200,
-        F_RESLK_LCKW,
-        F_RESLK_UNLCK,
-        F_GETLK_FD,
-} glusterfs_lk_recovery_cmds_t;
-
-typedef enum {
-        GF_LOCK_POSIX,
-        GF_LOCK_INTERNAL
-} gf_lk_domain_t;
-
-
-typedef enum {
-	ENTRYLK_LOCK,
-	ENTRYLK_UNLOCK,
-	ENTRYLK_LOCK_NB
-} entrylk_cmd;
-
-
-typedef enum {
-	ENTRYLK_RDLCK,
-	ENTRYLK_WRLCK
-} entrylk_type;
-
-
-typedef enum {
-	GF_XATTROP_ADD_ARRAY,
-} gf_xattrop_flags_t;
-
-
-#define GF_SET_IF_NOT_PRESENT 0x1 /* default behaviour */
-#define GF_SET_OVERWRITE      0x2 /* Overwrite with the buf given */
-#define GF_SET_DIR_ONLY       0x4
-#define GF_SET_EPOCH_TIME     0x8 /* used by afr dir lookup selfheal */
-
-/* Directory into which replicate self-heal will move deleted files and
-   directories into. The storage/posix janitor thread will periodically
-   clean up this directory */
-
-#define GF_REPLICATE_TRASH_DIR          ".landfill"
-
-/* key value which quick read uses to get small files in lookup cbk */
-#define GF_CONTENT_KEY "glusterfs.content"
-
-struct _xlator_cmdline_option {
-	struct list_head    cmd_args;
-	char               *volume;
-	char               *key;
-	char               *value;
-};
-typedef struct _xlator_cmdline_option xlator_cmdline_option_t;
-
-
-#define GF_OPTION_ENABLE   _gf_true
-#define GF_OPTION_DISABLE  _gf_false
-#define GF_OPTION_DEFERRED 2
-
-struct _cmd_args {
-	/* basic options */
-	char            *volfile_server;
-	char            *volfile;
-        char            *log_server;
-	gf_loglevel_t    log_level;
-	char            *log_file;
-        int32_t          max_connect_attempts;
-	/* advanced options */
-	uint32_t         volfile_server_port;
-	char            *volfile_server_transport;
-        uint32_t         log_server_port;
-	char            *pid_file;
-	int              no_daemon_mode;
-	char            *run_id;
-	int              debug_mode;
-        int              read_only;
-        int              mac_compat;
-	struct list_head xlator_options;  /* list of xlator_option_t */
-
-	/* fuse options */
-	int              fuse_direct_io_mode;
-        int              volfile_check;
-	double           fuse_entry_timeout;
-	double           fuse_attribute_timeout;
-	char            *volume_name;
-	int              fuse_nodev;
-	int              fuse_nosuid;
-	char            *dump_fuse;
-
-	/* key args */
-	char            *mount_point;
-	char            *volfile_id;
-
-        /* required for portmap */
-        int             brick_port;
-        char           *brick_name;
-};
-typedef struct _cmd_args cmd_args_t;
-
-
-struct _glusterfs_graph {
-        struct list_head          list;
-        char                      graph_uuid[128];
-        struct timeval            dob;
-        void                     *first;
-        void                     *top;   /* selected by -n */
-        int                       xl_count;
-        uint32_t                  volfile_checksum;
-};
-typedef struct _glusterfs_graph glusterfs_graph_t;
-
-
-struct _glusterfs_ctx {
-	cmd_args_t          cmd_args;
-	char               *process_uuid;
-	FILE               *pidfp;
-	char                fin;
-	void               *timer;
-	void               *ib;
-	void               *pool;
-	void               *event_pool;
-        void               *iobuf_pool;
-	pthread_mutex_t     lock;
-        size_t              page_size;
-        struct list_head    graphs; /* double linked list of graphs - one per volfile parse */
-        glusterfs_graph_t  *active; /* the latest graph in use */
-        void               *master; /* fuse, or libglusterfsclient (however, not protocol/server) */
-        void               *mgmt;   /* xlator implementing MOPs for centralized logging, volfile server */
-        unsigned char       measure_latency; /* toggle switch for latency measurement */
-        pthread_t           sigwaiter;
-        struct mem_pool    *stub_mem_pool;
-        unsigned char       cleanup_started;
-
-};
-typedef struct _glusterfs_ctx glusterfs_ctx_t;
-
-
-/* If you edit this structure then, make a corresponding change in
- * globals.c in the eventstring.
- */
-typedef enum {
-        GF_EVENT_PARENT_UP = 1,
-        GF_EVENT_POLLIN,
-        GF_EVENT_POLLOUT,
-        GF_EVENT_POLLERR,
-        GF_EVENT_CHILD_UP,
-        GF_EVENT_CHILD_DOWN,
-        GF_EVENT_CHILD_CONNECTING,
-        GF_EVENT_TRANSPORT_CLEANUP,
-        GF_EVENT_TRANSPORT_CONNECTED,
-        GF_EVENT_VOLFILE_MODIFIED,
-        GF_EVENT_GRAPH_NEW,
-        GF_EVENT_MAXVAL,
-} glusterfs_event_t;
-
-extern char *
-glusterfs_strevent (glusterfs_event_t ev);
-
-#define GF_MUST_CHECK __attribute__((warn_unused_result))
-
-int glusterfs_graph_prepare (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx);
-int glusterfs_graph_destroy (glusterfs_graph_t *graph);
-int glusterfs_graph_activate (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx);
-glusterfs_graph_t *glusterfs_graph_construct (FILE *fp);
-glusterfs_graph_t *glusterfs_graph_new ();
-int glusterfs_graph_reconfigure (glusterfs_graph_t *oldgraph,
-                                  glusterfs_graph_t *newgraph);
-
-#endif /* _GLUSTERFS_H */
diff --git a/libglusterfs/src/glusterfs/async.h b/libglusterfs/src/glusterfs/async.h
new file mode 100644
index 00000000000..d1d70ae0bc7
--- /dev/null
+++ b/libglusterfs/src/glusterfs/async.h
@@ -0,0 +1,209 @@
+/*
+  Copyright (c) 2019 Red Hat, Inc <https://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __GLUSTERFS_ASYNC_H__
+#define __GLUSTERFS_ASYNC_H__
+
+#define _LGPL_SOURCE
+
+#include <sys/types.h>
+#include <signal.h>
+#include <errno.h>
+
+#ifdef URCU_OLD
+
+/* TODO: Fix the include paths. Since this is a .h included from many places
+ *       it makes no sense to append a '-I$(CONTRIBDIR)/userspace-rcu/' to each
+ *       Makefile.am. I've also seen some problems with CI builders (they
+ *       failed to find the include files, but the same source on another setup
+ *       is working fine). */
+#include "wfcqueue.h"
+#include "wfstack.h"
+
+#else /* !URCU_OLD */
+
+#include <urcu/wfcqueue.h>
+#include <urcu/wfstack.h>
+
+#endif /* URCU_OLD */
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/list.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+/* This is the name prefix that all worker threads will have. A number will
+ * be added to differentiate them. */
+#define GF_ASYNC_THREAD_NAME "tpw"
+
+/* This value determines the maximum number of threads that are allowed. */
+#define GF_ASYNC_MAX_THREADS 128
+
+/* This value determines how many additional threads will be started but will
+ * remain inactive until they are explicitly activated by the leader. This is
+ * useful to react faster to bursts of load, but at the same time we minimize
+ * contention if they are not really needed to handle current load.
+ *
+ * TODO: Instead of a fixed number, it would probably be better to use a
+ *       prcentage of the available cores. */
+#define GF_ASYNC_SPARE_THREADS 2
+
+/* This value determines the signal used to wake the leader when new work has
+ * been added to the queue. To do so we reuse SIGALRM, since the most logical
+ * candidates (SIGUSR1/SIGUSR2) are already used. This signal must not be used
+ * by anything else in the process. */
+#define GF_ASYNC_SIGQUEUE SIGALRM
+
+/* This value determines the signal that will be used to transfer leader role
+ * to other workers. */
+#define GF_ASYNC_SIGCTRL SIGVTALRM
+
+#define gf_async_warning(_err, _msg, _args...)                                 \
+    gf_msg("async", GF_LOG_WARNING, -(_err), LG_MSG_ASYNC_WARNING, _msg,       \
+           ##_args)
+
+#define gf_async_error(_err, _msg, _args...)                                   \
+    gf_msg("async", GF_LOG_ERROR, -(_err), LG_MSG_ASYNC_FAILURE, _msg, ##_args)
+
+#define gf_async_fatal(_err, _msg, _args...)                                   \
+    do {                                                                       \
+        GF_ABORT("Critical error in async module. Unable to continue. (" _msg  \
+                 "). Error %d.",                                               \
+                 ##_args, -(_err));                                            \
+    } while (0)
+
+struct _gf_async;
+typedef struct _gf_async gf_async_t;
+
+struct _gf_async_worker;
+typedef struct _gf_async_worker gf_async_worker_t;
+
+struct _gf_async_queue;
+typedef struct _gf_async_queue gf_async_queue_t;
+
+struct _gf_async_control;
+typedef struct _gf_async_control gf_async_control_t;
+
+typedef void (*gf_async_callback_f)(xlator_t *xl, gf_async_t *async);
+
+struct _gf_async {
+    /* TODO: remove dependency on xl/THIS. */
+    xlator_t *xl;
+    gf_async_callback_f cbk;
+    struct cds_wfcq_node queue;
+};
+
+struct _gf_async_worker {
+    /* Used to send asynchronous jobs related to the worker. */
+    gf_async_t async;
+
+    /* Member of the available workers stack. */
+    struct cds_wfs_node stack;
+
+    /* Thread object of the current worker. */
+    pthread_t thread;
+
+    /* Unique identifier of this worker. */
+    int32_t id;
+
+    /* Indicates if this worker is enabled. */
+    bool running;
+};
+
+struct _gf_async_queue {
+    /* Structures needed to manage a wait-free queue. For better performance
+     * they are placed in two different cache lines, as recommended by URCU
+     * documentation, even though in our case some threads will be producers
+     * and consumers at the same time. */
+    struct cds_wfcq_head head __attribute__((aligned(64)));
+    struct cds_wfcq_tail tail __attribute__((aligned(64)));
+};
+
+#define GF_ASYNC_COUNTS(_run, _stop) (((uint32_t)(_run) << 16) + (_stop))
+#define GF_ASYNC_COUNT_RUNNING(_count) ((_count) >> 16)
+#define GF_ASYNC_COUNT_STOPPING(_count) ((_count)&65535)
+
+struct _gf_async_control {
+    gf_async_queue_t queue;
+
+    /* Stack of unused workers. */
+    struct __cds_wfs_stack available;
+
+    /* Array of preallocated worker structures. */
+    gf_async_worker_t *table;
+
+    /* Used to synchronize main thread with workers on termination. */
+    pthread_barrier_t sync;
+
+    /* The id of the last thread that will be used for synchronization. */
+    pthread_t sync_thread;
+
+    /* Signal mask to wait for control signals from leader. */
+    sigset_t sigmask_ctrl;
+
+    /* Signal mask to wait for queued items. */
+    sigset_t sigmask_queue;
+
+    /* Saved signal handlers. */
+    struct sigaction handler_ctrl;
+    struct sigaction handler_queue;
+
+    /* PID of the current process. */
+    pid_t pid;
+
+    /* Maximum number of allowed threads. */
+    uint32_t max_threads;
+
+    /* Current number of running and stopping workers. This value is split
+     * into 2 16-bits fields to track both counters atomically at the same
+     * time. */
+    uint32_t counts;
+
+    /* It's used to control whether the asynchronous infrastructure is used
+     * or not. */
+    bool enabled;
+};
+
+extern gf_async_control_t gf_async_ctrl;
+
+int32_t
+gf_async_init(glusterfs_ctx_t *ctx);
+
+void
+gf_async_fini(void);
+
+void
+gf_async_adjust_threads(int32_t threads);
+
+static inline void
+gf_async(gf_async_t *async, xlator_t *xl, gf_async_callback_f cbk)
+{
+    if (!gf_async_ctrl.enabled) {
+        cbk(xl, async);
+        return;
+    }
+
+    async->xl = xl;
+    async->cbk = cbk;
+    cds_wfcq_node_init(&async->queue);
+    if (caa_unlikely(!cds_wfcq_enqueue(&gf_async_ctrl.queue.head,
+                                       &gf_async_ctrl.queue.tail,
+                                       &async->queue))) {
+        /* The queue was empty, so the leader could be sleeping. We need to
+         * wake it so that the new item can be processed. If the queue was not
+         * empty, we don't need to do anything special since the leader will
+         * take care of it. */
+        if (caa_unlikely(kill(gf_async_ctrl.pid, GF_ASYNC_SIGQUEUE) < 0)) {
+            gf_async_fatal(errno, "Unable to wake leader worker.");
+        };
+    }
+}
+
+#endif /* !__GLUSTERFS_ASYNC_H__ */
diff --git a/libglusterfs/src/glusterfs/atomic.h b/libglusterfs/src/glusterfs/atomic.h
new file mode 100644
index 00000000000..ced81748218
--- /dev/null
+++ b/libglusterfs/src/glusterfs/atomic.h
@@ -0,0 +1,459 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _ATOMIC_H
+#define _ATOMIC_H
+
+#include <inttypes.h>
+#include <stdbool.h>
+
+#include "glusterfs/locking.h"
+
+/* Macros used to join two arguments and generate a new macro name. */
+#define GF_ATOMIC_MACRO_1(_macro) _macro
+#define GF_ATOMIC_MACRO(_base, _name) GF_ATOMIC_MACRO_1(_base##_name)
+
+/* There's a problem on 32-bit architectures when we try to use atomic
+ * builtins with 64-bit types. Only way to solve the problem is to use
+ * a mutex to protect the access to the atomic, but we don't want to
+ * use mutexes for other smaller types that could work with the atomic
+ * builtins.
+ *
+ * So on each atomic type we add a field for the mutex if atomic operation
+ * is not supported and a dummy zero size field if it's supported. This way
+ * we can have different atomic types, some with a mutex and some without.
+ *
+ * To define these types, we use two macros:
+ *
+ *     GF_ATOMIC_MUTEX_FIELD_0 = char lk[0]
+ *     GF_ATOMIC_MUTEX_FILED_1 = gf_lock_t lk
+ *
+ * Both macros define the 'lk' field that will be used in the atomic
+ * structure. One when the atomic is supported by the architecture and
+ * another when not. We need to define the field even if it won't be
+ * used. Otherwise the compiler will return an error.
+ *
+ * Now we need to take the mutex or not depending on the existence of
+ * the mutex field in the structure. To do so we check the size of the
+ * structure, and if it's bigger than uint64_t (all structures with a
+ * mutex will be bigger), we use the mutex-based version. Otherwise we
+ * use the atomic builtin. This check is easily optimized out by the
+ * compiler, leaving a clean and efficient compiled code. */
+
+#define GF_ATOMIC_MUTEX_FIELD_0 char lk[0]
+#define GF_ATOMIC_MUTEX_FIELD_1 gf_lock_t lk
+
+/* We'll use SIZEOF_LONG to determine the architecture. 32-bit machines
+ * will have 4 here, while 64-bit machines will have 8. If additional
+ * needs or restrictions appear on other platforms, these tests can be
+ * extended to handle them. */
+
+/* GF_ATOMIC_SIZE_X macros map each type size to one of the
+ * GF_ATOMIC_MUTEX_FIELD_X macros, depending on detected conditions. */
+
+#if defined(HAVE_ATOMIC_BUILTINS) || defined(HAVE_SYNC_BUILTINS)
+
+#define GF_ATOMIC_SIZE_1 GF_ATOMIC_MUTEX_FIELD_0
+#define GF_ATOMIC_SIZE_2 GF_ATOMIC_MUTEX_FIELD_0
+#define GF_ATOMIC_SIZE_4 GF_ATOMIC_MUTEX_FIELD_0
+
+#if SIZEOF_LONG >= 8
+#define GF_ATOMIC_SIZE_8 GF_ATOMIC_MUTEX_FIELD_0
+#endif
+
+#endif /* HAVE_(ATOMIC|SYNC)_BUILTINS */
+
+/* Any GF_ATOMIC_SIZE_X macro not yet defined will use the mutex version */
+#ifndef GF_ATOMIC_SIZE_1
+#define GF_ATOMIC_SIZE_1 GF_ATOMIC_MUTEX_FIELD_1
+#endif
+
+#ifndef GF_ATOMIC_SIZE_2
+#define GF_ATOMIC_SIZE_2 GF_ATOMIC_MUTEX_FIELD_1
+#endif
+
+#ifndef GF_ATOMIC_SIZE_4
+#define GF_ATOMIC_SIZE_4 GF_ATOMIC_MUTEX_FIELD_1
+#endif
+
+#ifndef GF_ATOMIC_SIZE_8
+#define GF_ATOMIC_SIZE_8 GF_ATOMIC_MUTEX_FIELD_1
+#endif
+
+/* This macro is used to define all atomic types supported. First field
+ * represents the size of the type in bytes, and the second one the name. */
+#define GF_ATOMIC_TYPE(_size, _name)                                           \
+    typedef struct _gf_atomic_##_name##_t {                                    \
+        GF_ATOMIC_MACRO(GF_ATOMIC_SIZE_, _size);                               \
+        _name##_t value;                                                       \
+    } gf_atomic_##_name##_t
+
+/* The atomic types we support */
+GF_ATOMIC_TYPE(1, int8);              /* gf_atomic_int8_t */
+GF_ATOMIC_TYPE(2, int16);             /* gf_atomic_int16_t */
+GF_ATOMIC_TYPE(4, int32);             /* gf_atomic_int32_t */
+GF_ATOMIC_TYPE(8, int64);             /* gf_atomic_int64_t */
+GF_ATOMIC_TYPE(SIZEOF_LONG, intptr);  /* gf_atomic_intptr_t */
+GF_ATOMIC_TYPE(1, uint8);             /* gf_atomic_uint8_t */
+GF_ATOMIC_TYPE(2, uint16);            /* gf_atomic_uint16_t */
+GF_ATOMIC_TYPE(4, uint32);            /* gf_atomic_uint32_t */
+GF_ATOMIC_TYPE(8, uint64);            /* gf_atomic_uint64_t */
+GF_ATOMIC_TYPE(SIZEOF_LONG, uintptr); /* gf_atomic_uintptr_t */
+
+/* Define the default atomic type as int64_t */
+#define gf_atomic_t gf_atomic_int64_t
+
+/* This macro will choose between the mutex based version and the atomic
+ * builtin version depending on the size of the atomic structure. */
+#define GF_ATOMIC_CHOOSE(_atomic, _op, _args...)                               \
+    ((sizeof(_atomic) > sizeof(uint64_t))                                      \
+         ? ({                                                                  \
+               GF_ATOMIC_MACRO(GF_ATOMIC_LOCK_, _op)                           \
+               (_atomic, ##_args);                                             \
+           })                                                                  \
+         : ({                                                                  \
+               GF_ATOMIC_MACRO(GF_ATOMIC_BASE_, _op)                           \
+               (_atomic, ##_args);                                             \
+           }))
+
+/* Macros to implement the mutex-based atomics. */
+#define GF_ATOMIC_OP_PREPARE(_atomic, _name)                                   \
+    typeof(_atomic) *__atomic = &(_atomic);                                    \
+    gf_lock_t *__lock = (gf_lock_t *)&__atomic->lk;                            \
+    LOCK(__lock);                                                              \
+    typeof(__atomic->value) _name = __atomic->value
+
+#define GF_ATOMIC_OP_STORE(_value) (__atomic->value = (_value))
+
+#define GF_ATOMIC_OP_RETURN(_value)                                            \
+    ({                                                                         \
+        UNLOCK(__lock);                                                        \
+        _value;                                                                \
+    })
+
+#define GF_ATOMIC_LOCK_INIT(_atomic, _value)                                   \
+    do {                                                                       \
+        typeof(_atomic) *__atomic = &(_atomic);                                \
+        LOCK_INIT((gf_lock_t *)&__atomic->lk);                                 \
+        __atomic->value = (_value);                                            \
+    } while (0)
+
+#define GF_ATOMIC_LOCK_GET(_atomic)                                            \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_ADD(_atomic, _value)                                    \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value += (_value));                               \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_SUB(_atomic, _value)                                    \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value -= (_value));                               \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_AND(_atomic, _value)                                    \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value &= (_value));                               \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_OR(_atomic, _value)                                     \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value |= (_value));                               \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_XOR(_atomic, _value)                                    \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value ^= (_value));                               \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_NAND(_atomic, _value)                                   \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value = ~(__value & (_value)));                   \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_ADD(_atomic, _value)                              \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value + (_value));                                \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_SUB(_atomic, _value)                              \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value - (_value));                                \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_AND(_atomic, _value)                              \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value &(_value));                                 \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_OR(_atomic, _value)                               \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value | (_value));                                \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_XOR(_atomic, _value)                              \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(__value ^ (_value));                                \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_FETCH_NAND(_atomic, _value)                             \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(~(__value & (_value)));                             \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_SWAP(_atomic, _value)                                   \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        GF_ATOMIC_OP_STORE(_value);                                            \
+        GF_ATOMIC_OP_RETURN(__value);                                          \
+    })
+
+#define GF_ATOMIC_LOCK_CMP_SWAP(_atomic, _expected, _value)                    \
+    ({                                                                         \
+        GF_ATOMIC_OP_PREPARE(_atomic, __value);                                \
+        bool __ret = (__value == (_expected));                                 \
+        if (__ret) {                                                           \
+            GF_ATOMIC_OP_STORE(_value);                                        \
+        }                                                                      \
+        GF_ATOMIC_OP_RETURN(__ret);                                            \
+    })
+
+#if defined(HAVE_ATOMIC_BUILTINS)
+
+/* If compiler supports __atomic builtins, we use them. */
+
+#define GF_ATOMIC_BASE_INIT(_atomic, _value)                                   \
+    __atomic_store_n(&(_atomic).value, (_value), __ATOMIC_RELEASE)
+
+#define GF_ATOMIC_BASE_GET(_atomic)                                            \
+    __atomic_load_n(&(_atomic).value, __ATOMIC_ACQUIRE)
+
+#define GF_ATOMIC_BASE_ADD(_atomic, _value)                                    \
+    __atomic_add_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_SUB(_atomic, _value)                                    \
+    __atomic_sub_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_AND(_atomic, _value)                                    \
+    __atomic_and_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_OR(_atomic, _value)                                     \
+    __atomic_or_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_XOR(_atomic, _value)                                    \
+    __atomic_xor_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_NAND(_atomic, _value)                                   \
+    __atomic_nand_fetch(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_ADD(_atomic, _value)                              \
+    __atomic_fetch_add(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_SUB(_atomic, _value)                              \
+    __atomic_fetch_sub(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_AND(_atomic, _value)                              \
+    __atomic_fetch_and(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_OR(_atomic, _value)                               \
+    __atomic_fetch_or(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_XOR(_atomic, _value)                              \
+    __atomic_fetch_xor(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_FETCH_NAND(_atomic, _value)                             \
+    __atomic_fetch_nand(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_SWAP(_atomic, _value)                                   \
+    __atomic_exchange_n(&(_atomic).value, (_value), __ATOMIC_ACQ_REL)
+
+#define GF_ATOMIC_BASE_CMP_SWAP(_atomic, _expected, _value)                    \
+    ({                                                                         \
+        typeof((_atomic).value) __expected = (_expected);                      \
+        __atomic_compare_exchange_n(&(_atomic).value, &__expected, (_value),   \
+                                    0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE);    \
+    })
+
+#elif defined(HAVE_SYNC_BUILTINS)
+
+/* If compiler doesn't support __atomic builtins but supports __sync builtins,
+ * we use them. */
+
+#define GF_ATOMIC_BASE_INIT(_atomic, _value)                                   \
+    do {                                                                       \
+        (_atomic).value = (_value);                                            \
+        __sync_synchronize();                                                  \
+    } while (0)
+
+#define GF_ATOMIC_BASE_ADD(_atomic, _value)                                    \
+    __sync_add_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_SUB(_atomic, _value)                                    \
+    __sync_sub_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_AND(_atomic, _value)                                    \
+    __sync_and_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_OR(_atomic, _value)                                     \
+    __sync_or_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_XOR(_atomic, _value)                                    \
+    __sync_xor_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_NAND(_atomic, _value)                                   \
+    __sync_nand_and_fetch(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_ADD(_atomic, _value)                              \
+    __sync_fetch_and_add(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_SUB(_atomic, _value)                              \
+    __sync_fetch_and_sub(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_AND(_atomic, _value)                              \
+    __sync_fetch_and_and(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_OR(_atomic, _value)                               \
+    __sync_fetch_and_or(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_XOR(_atomic, _value)                              \
+    __sync_fetch_and_xor(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_FETCH_NAND(_atomic, _value)                             \
+    __sync_fetch_and_nand(&(_atomic).value, (_value))
+
+#define GF_ATOMIC_BASE_SWAP(_atomic, _value)                                   \
+    ({                                                                         \
+        __sync_synchronize();                                                  \
+        __sync_lock_test_and_set(&(_atomic).value, (_value));                  \
+    })
+
+#define GF_ATOMIC_BASE_CMP_SWAP(_atomic, _expected, _value)                    \
+    __sync_bool_compare_and_swap(&(_atomic).value, (_expected), (_value))
+
+#define GF_ATOMIC_BASE_GET(_atomic) GF_ATOMIC_BASE_ADD(_atomic, 0)
+
+#else /* !HAVE_ATOMIC_BUILTINS && !HAVE_SYNC_BUILTINS */
+
+/* The compiler doesn't support any atomic builtin. We fallback to the
+ * mutex-based implementation. */
+
+#define GF_ATOMIC_BASE_INIT(_atomic, _value)                                   \
+    GF_ATOMIC_LOCK_INIT(_atomic, _value)
+
+#define GF_ATOMIC_BASE_GET(_atomic) GF_ATOMIC_LOCK_GET(_atomic)
+
+#define GF_ATOMIC_BASE_ADD(_atomic, _value) GF_ATOMIC_LOCK_ADD(_atomic, _value)
+
+#define GF_ATOMIC_BASE_SUB(_atomic, _value) GF_ATOMIC_LOCK_SUB(_atomic, _value)
+
+#define GF_ATOMIC_BASE_AND(_atomic, _value) GF_ATOMIC_LOCK_AND(_atomic, _value)
+
+#define GF_ATOMIC_BASE_OR(_atomic, _value) GF_ATOMIC_LOCK_OR(_atomic, _value)
+
+#define GF_ATOMIC_BASE_XOR(_atomic, _value) GF_ATOMIC_LOCK_XOR(_atomic, _value)
+
+#define GF_ATOMIC_BASE_NAND(_atomic, _value)                                   \
+    GF_ATOMIC_LOCK_NAND(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_ADD(_atomic, _value)                              \
+    GF_ATOMIC_LOCK_FETCH_ADD(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_SUB(_atomic, _value)                              \
+    GF_ATOMIC_LOCK_FETCH_SUB(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_AND(_atomic, _value)                              \
+    GF_ATOMIC_LOCK_FETCH_AND(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_OR(_atomic, _value)                               \
+    GF_ATOMIC_LOCK_FETCH_OR(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_XOR(_atomic, _value)                              \
+    GF_ATOMIC_LOCK_FETCH_XOR(_atomic, _value)
+
+#define GF_ATOMIC_BASE_FETCH_NAND(_atomic, _value)                             \
+    GF_ATOMIC_LOCK_FETCH_NAND(_atomic, _value)
+
+#define GF_ATOMIC_BASE_SWAP(_atomic, _value)                                   \
+    GF_ATOMIC_LOCK_SWAP(_atomic, _value)
+
+#define GF_ATOMIC_BASE_CMP_SWAP(_atomic, _expected, _value)                    \
+    GF_ATOMIC_LOCK_CMP_SWAP(_atomic, _expected, _value)
+
+#endif /* HAVE_(ATOMIC|SYNC)_BUILTINS */
+
+/* Here we declare the real atomic macros available to the user. */
+
+/* All macros have a 'gf_atomic_xxx' as 1st argument */
+
+#define GF_ATOMIC_INIT(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, INIT, _value)
+#define GF_ATOMIC_GET(_atomic) GF_ATOMIC_CHOOSE(_atomic, GET)
+#define GF_ATOMIC_ADD(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, ADD, _value)
+#define GF_ATOMIC_SUB(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, SUB, _value)
+#define GF_ATOMIC_AND(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, AND, _value)
+#define GF_ATOMIC_OR(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, OR, _value)
+#define GF_ATOMIC_XOR(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, XOR, _value)
+#define GF_ATOMIC_NAND(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, NAND, _value)
+
+#define GF_ATOMIC_FETCH_ADD(_atomic, _value)                                   \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_ADD, _value)
+
+#define GF_ATOMIC_FETCH_SUB(_atomic, _value)                                   \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_SUB, _value)
+
+#define GF_ATOMIC_FETCH_AND(_atomic, _value)                                   \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_AND, _value)
+
+#define GF_ATOMIC_FETCH_OR(_atomic, _value)                                    \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_OR, _value)
+
+#define GF_ATOMIC_FETCH_XOR(_atomic, _value)                                   \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_XOR, _value)
+
+#define GF_ATOMIC_FETCH_NAND(_atomic, _value)                                  \
+    GF_ATOMIC_CHOOSE(_atomic, FETCH_NAND, _value)
+
+#define GF_ATOMIC_SWAP(_atomic, _value) GF_ATOMIC_CHOOSE(_atomic, SWAP, _value)
+
+#define GF_ATOMIC_CMP_SWAP(_atomic, _expected, _value)                         \
+    GF_ATOMIC_CHOOSE(_atomic, CMP_SWAP, _expected, _value)
+
+#define GF_ATOMIC_INC(_atomic) GF_ATOMIC_ADD(_atomic, 1)
+#define GF_ATOMIC_DEC(_atomic) GF_ATOMIC_SUB(_atomic, 1)
+#define GF_ATOMIC_FETCH_INC(_atomic) GF_ATOMIC_FETCH_ADD(_atomic, 1)
+#define GF_ATOMIC_FETCH_DEC(_atomic) GF_ATOMIC_FETCH_SUB(_atomic, 1)
+
+#endif /* _ATOMIC_H */
diff --git a/libglusterfs/src/glusterfs/byte-order.h b/libglusterfs/src/glusterfs/byte-order.h
new file mode 100644
index 00000000000..fd8cef9e58d
--- /dev/null
+++ b/libglusterfs/src/glusterfs/byte-order.h
@@ -0,0 +1,279 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _BYTE_ORDER_H
+#define _BYTE_ORDER_H
+
+#include <inttypes.h>
+
+#define LS1 0x00ffU
+#define MS1 0xff00U
+#define LS2 0x0000ffffU
+#define MS2 0xffff0000U
+#define LS4 0x00000000ffffffffULL
+#define MS4 0xffffffff00000000ULL
+
+static uint16_t (*hton16)(uint16_t);
+static uint32_t (*hton32)(uint32_t);
+static uint64_t (*hton64)(uint64_t);
+
+#define ntoh16 hton16
+#define ntoh32 hton32
+#define ntoh64 hton64
+
+static uint16_t (*htole16)(uint16_t);
+static uint32_t (*htole32)(uint32_t);
+static uint64_t (*htole64)(uint64_t);
+
+#define letoh16 htole16
+#define letoh32 htole32
+#define letoh64 htole64
+
+static uint16_t (*htobe16)(uint16_t);
+static uint32_t (*htobe32)(uint32_t);
+static uint64_t (*htobe64)(uint64_t);
+
+#define betoh16 htobe16
+#define betoh32 htobe32
+#define betoh64 htobe64
+
+#define do_swap2(x) (((x & LS1) << 8) | (((x & MS1) >> 8)))
+#define do_swap4(x) ((do_swap2(x & LS2) << 16) | (do_swap2((x & MS2) >> 16)))
+#define do_swap8(x) ((do_swap4(x & LS4) << 32) | (do_swap4((x & MS4) >> 32)))
+
+static inline uint16_t
+__swap16(uint16_t x)
+{
+    return do_swap2(x);
+}
+
+static inline uint32_t
+__swap32(uint32_t x)
+{
+    return do_swap4(x);
+}
+
+static inline uint64_t
+__swap64(uint64_t x)
+{
+    return do_swap8(x);
+}
+
+static inline uint16_t
+__noswap16(uint16_t x)
+{
+    return x;
+}
+
+static inline uint32_t
+__noswap32(uint32_t x)
+{
+    return x;
+}
+
+static inline uint64_t
+__noswap64(uint64_t x)
+{
+    return x;
+}
+
+static inline uint16_t
+__byte_order_n16(uint16_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        hton16 = __swap16;
+        hton32 = __swap32;
+        hton64 = __swap64;
+    } else {
+        /* cpu is be */
+        hton16 = __noswap16;
+        hton32 = __noswap32;
+        hton64 = __noswap64;
+    }
+
+    return hton16(i);
+}
+
+static inline uint32_t
+__byte_order_n32(uint32_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        hton16 = __swap16;
+        hton32 = __swap32;
+        hton64 = __swap64;
+    } else {
+        /* cpu is be */
+        hton16 = __noswap16;
+        hton32 = __noswap32;
+        hton64 = __noswap64;
+    }
+
+    return hton32(i);
+}
+
+static inline uint64_t
+__byte_order_n64(uint64_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        hton16 = __swap16;
+        hton32 = __swap32;
+        hton64 = __swap64;
+    } else {
+        /* cpu is be */
+        hton16 = __noswap16;
+        hton32 = __noswap32;
+        hton64 = __noswap64;
+    }
+
+    return hton64(i);
+}
+
+static uint16_t (*hton16)(uint16_t) = __byte_order_n16;
+static uint32_t (*hton32)(uint32_t) = __byte_order_n32;
+static uint64_t (*hton64)(uint64_t) = __byte_order_n64;
+
+static inline uint16_t
+__byte_order_le16(uint16_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htole16 = __noswap16;
+        htole32 = __noswap32;
+        htole64 = __noswap64;
+    } else {
+        /* cpu is be */
+        htole16 = __swap16;
+        htole32 = __swap32;
+        htole64 = __swap64;
+    }
+
+    return htole16(i);
+}
+
+static inline uint32_t
+__byte_order_le32(uint32_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htole16 = __noswap16;
+        htole32 = __noswap32;
+        htole64 = __noswap64;
+    } else {
+        /* cpu is be */
+        htole16 = __swap16;
+        htole32 = __swap32;
+        htole64 = __swap64;
+    }
+
+    return htole32(i);
+}
+
+static inline uint64_t
+__byte_order_le64(uint64_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htole16 = __noswap16;
+        htole32 = __noswap32;
+        htole64 = __noswap64;
+    } else {
+        /* cpu is be */
+        htole16 = __swap16;
+        htole32 = __swap32;
+        htole64 = __swap64;
+    }
+
+    return htole64(i);
+}
+
+static uint16_t (*htole16)(uint16_t) = __byte_order_le16;
+static uint32_t (*htole32)(uint32_t) = __byte_order_le32;
+static uint64_t (*htole64)(uint64_t) = __byte_order_le64;
+
+static inline uint16_t
+__byte_order_be16(uint16_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htobe16 = __swap16;
+        htobe32 = __swap32;
+        htobe64 = __swap64;
+    } else {
+        /* cpu is be */
+        htobe16 = __noswap16;
+        htobe32 = __noswap32;
+        htobe64 = __noswap64;
+    }
+
+    return htobe16(i);
+}
+
+static inline uint32_t
+__byte_order_be32(uint32_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htobe16 = __swap16;
+        htobe32 = __swap32;
+        htobe64 = __swap64;
+    } else {
+        /* cpu is be */
+        htobe16 = __noswap16;
+        htobe32 = __noswap32;
+        htobe64 = __noswap64;
+    }
+
+    return htobe32(i);
+}
+
+static inline uint64_t
+__byte_order_be64(uint64_t i)
+{
+    uint32_t num = 1;
+
+    if (((char *)(&num))[0] == 1) {
+        /* cpu is le */
+        htobe16 = __swap16;
+        htobe32 = __swap32;
+        htobe64 = __swap64;
+    } else {
+        /* cpu is be */
+        htobe16 = __noswap16;
+        htobe32 = __noswap32;
+        htobe64 = __noswap64;
+    }
+
+    return htobe64(i);
+}
+
+static uint16_t (*htobe16)(uint16_t) = __byte_order_be16;
+static uint32_t (*htobe32)(uint32_t) = __byte_order_be32;
+static uint64_t (*htobe64)(uint64_t) = __byte_order_be64;
+
+#endif /* _BYTE_ORDER_H */
diff --git a/libglusterfs/src/glusterfs/call-stub.h b/libglusterfs/src/glusterfs/call-stub.h
new file mode 100644
index 00000000000..8237ea459bf
--- /dev/null
+++ b/libglusterfs/src/glusterfs/call-stub.h
@@ -0,0 +1,622 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CALL_STUB_H_
+#define _CALL_STUB_H_
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/defaults.h"
+#include "glusterfs/default-args.h"
+#include "glusterfs/stack.h"
+#include "glusterfs/list.h"
+
+typedef struct _call_stub {
+    struct list_head list;
+    call_frame_t *frame;
+    struct mem_pool *stub_mem_pool; /* pointer to stub mempool in ctx_t */
+    uint32_t jnl_meta_len;
+    uint32_t jnl_data_len;
+    void (*serialize)(struct _call_stub *, char *, char *);
+    union {
+        fop_lookup_t lookup;
+        fop_stat_t stat;
+        fop_fstat_t fstat;
+        fop_truncate_t truncate;
+        fop_ftruncate_t ftruncate;
+        fop_access_t access;
+        fop_readlink_t readlink;
+        fop_mknod_t mknod;
+        fop_mkdir_t mkdir;
+        fop_unlink_t unlink;
+        fop_rmdir_t rmdir;
+        fop_symlink_t symlink;
+        fop_rename_t rename;
+        fop_link_t link;
+        fop_create_t create;
+        fop_open_t open;
+        fop_readv_t readv;
+        fop_writev_t writev;
+        fop_flush_t flush;
+        fop_fsync_t fsync;
+        fop_opendir_t opendir;
+        fop_fsyncdir_t fsyncdir;
+        fop_statfs_t statfs;
+        fop_setxattr_t setxattr;
+        fop_getxattr_t getxattr;
+        fop_fgetxattr_t fgetxattr;
+        fop_fsetxattr_t fsetxattr;
+        fop_removexattr_t removexattr;
+        fop_fremovexattr_t fremovexattr;
+        fop_lk_t lk;
+        fop_inodelk_t inodelk;
+        fop_finodelk_t finodelk;
+        fop_entrylk_t entrylk;
+        fop_fentrylk_t fentrylk;
+        fop_readdir_t readdir;
+        fop_readdirp_t readdirp;
+        fop_rchecksum_t rchecksum;
+        fop_xattrop_t xattrop;
+        fop_fxattrop_t fxattrop;
+        fop_setattr_t setattr;
+        fop_fsetattr_t fsetattr;
+        fop_fallocate_t fallocate;
+        fop_discard_t discard;
+        fop_zerofill_t zerofill;
+        fop_ipc_t ipc;
+        fop_seek_t seek;
+        fop_lease_t lease;
+        fop_getactivelk_t getactivelk;
+        fop_setactivelk_t setactivelk;
+        fop_put_t put;
+        fop_icreate_t icreate;
+        fop_namelink_t namelink;
+        fop_copy_file_range_t copy_file_range;
+    } fn;
+
+    union {
+        fop_lookup_cbk_t lookup;
+        fop_stat_cbk_t stat;
+        fop_fstat_cbk_t fstat;
+        fop_truncate_cbk_t truncate;
+        fop_ftruncate_cbk_t ftruncate;
+        fop_access_cbk_t access;
+        fop_readlink_cbk_t readlink;
+        fop_mknod_cbk_t mknod;
+        fop_mkdir_cbk_t mkdir;
+        fop_unlink_cbk_t unlink;
+        fop_rmdir_cbk_t rmdir;
+        fop_symlink_cbk_t symlink;
+        fop_rename_cbk_t rename;
+        fop_link_cbk_t link;
+        fop_create_cbk_t create;
+        fop_open_cbk_t open;
+        fop_readv_cbk_t readv;
+        fop_writev_cbk_t writev;
+        fop_flush_cbk_t flush;
+        fop_fsync_cbk_t fsync;
+        fop_opendir_cbk_t opendir;
+        fop_fsyncdir_cbk_t fsyncdir;
+        fop_statfs_cbk_t statfs;
+        fop_setxattr_cbk_t setxattr;
+        fop_getxattr_cbk_t getxattr;
+        fop_fgetxattr_cbk_t fgetxattr;
+        fop_fsetxattr_cbk_t fsetxattr;
+        fop_removexattr_cbk_t removexattr;
+        fop_fremovexattr_cbk_t fremovexattr;
+        fop_lk_cbk_t lk;
+        fop_inodelk_cbk_t inodelk;
+        fop_finodelk_cbk_t finodelk;
+        fop_entrylk_cbk_t entrylk;
+        fop_fentrylk_cbk_t fentrylk;
+        fop_readdir_cbk_t readdir;
+        fop_readdirp_cbk_t readdirp;
+        fop_rchecksum_cbk_t rchecksum;
+        fop_xattrop_cbk_t xattrop;
+        fop_fxattrop_cbk_t fxattrop;
+        fop_setattr_cbk_t setattr;
+        fop_fsetattr_cbk_t fsetattr;
+        fop_fallocate_cbk_t fallocate;
+        fop_discard_cbk_t discard;
+        fop_zerofill_cbk_t zerofill;
+        fop_ipc_cbk_t ipc;
+        fop_seek_cbk_t seek;
+        fop_lease_cbk_t lease;
+        fop_getactivelk_cbk_t getactivelk;
+        fop_setactivelk_cbk_t setactivelk;
+        fop_put_cbk_t put;
+        fop_icreate_cbk_t icreate;
+        fop_namelink_cbk_t namelink;
+        fop_copy_file_range_cbk_t copy_file_range;
+    } fn_cbk;
+    glusterfs_fop_t fop;
+    gf_boolean_t poison;
+    char wind;
+    default_args_t args;
+    default_args_cbk_t args_cbk;
+} call_stub_t;
+
+call_stub_t *
+fop_lookup_stub(call_frame_t *frame, fop_lookup_t fn, loc_t *loc,
+                dict_t *xdata);
+
+call_stub_t *
+fop_lookup_cbk_stub(call_frame_t *frame, fop_lookup_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode, struct iatt *buf,
+                    dict_t *xdata, struct iatt *postparent);
+call_stub_t *
+fop_stat_stub(call_frame_t *frame, fop_stat_t fn, loc_t *loc, dict_t *xdata);
+call_stub_t *
+fop_stat_cbk_stub(call_frame_t *frame, fop_stat_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, struct iatt *buf, dict_t *xdata);
+call_stub_t *
+fop_fstat_stub(call_frame_t *frame, fop_fstat_t fn, fd_t *fd, dict_t *xdata);
+call_stub_t *
+fop_fstat_cbk_stub(call_frame_t *frame, fop_fstat_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *buf, dict_t *xdata);
+
+call_stub_t *
+fop_truncate_stub(call_frame_t *frame, fop_truncate_t fn, loc_t *loc, off_t off,
+                  dict_t *xdata);
+
+call_stub_t *
+fop_truncate_cbk_stub(call_frame_t *frame, fop_truncate_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata);
+
+call_stub_t *
+fop_ftruncate_stub(call_frame_t *frame, fop_ftruncate_t fn, fd_t *fd, off_t off,
+                   dict_t *xdata);
+
+call_stub_t *
+fop_ftruncate_cbk_stub(call_frame_t *frame, fop_ftruncate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata);
+
+call_stub_t *
+fop_access_stub(call_frame_t *frame, fop_access_t fn, loc_t *loc, int32_t mask,
+                dict_t *xdata);
+
+call_stub_t *
+fop_access_cbk_stub(call_frame_t *frame, fop_access_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_readlink_stub(call_frame_t *frame, fop_readlink_t fn, loc_t *loc,
+                  size_t size, dict_t *xdata);
+
+call_stub_t *
+fop_readlink_cbk_stub(call_frame_t *frame, fop_readlink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, const char *path,
+                      struct iatt *buf, dict_t *xdata);
+
+call_stub_t *
+fop_mknod_stub(call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
+               dev_t rdev, mode_t umask, dict_t *xdata);
+
+call_stub_t *
+fop_mknod_cbk_stub(call_frame_t *frame, fop_mknod_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata);
+
+call_stub_t *
+fop_mkdir_stub(call_frame_t *frame, fop_mkdir_t fn, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata);
+
+call_stub_t *
+fop_mkdir_cbk_stub(call_frame_t *frame, fop_mkdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata);
+
+call_stub_t *
+fop_unlink_stub(call_frame_t *frame, fop_unlink_t fn, loc_t *loc, int xflag,
+                dict_t *xdata);
+
+call_stub_t *
+fop_unlink_cbk_stub(call_frame_t *frame, fop_unlink_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+
+call_stub_t *
+fop_rmdir_stub(call_frame_t *frame, fop_rmdir_t fn, loc_t *loc, int flags,
+               dict_t *xdata);
+
+call_stub_t *
+fop_rmdir_cbk_stub(call_frame_t *frame, fop_rmdir_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata);
+
+call_stub_t *
+fop_symlink_stub(call_frame_t *frame, fop_symlink_t fn, const char *linkname,
+                 loc_t *loc, mode_t umask, dict_t *xdata);
+
+call_stub_t *
+fop_symlink_cbk_stub(call_frame_t *frame, fop_symlink_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata);
+
+call_stub_t *
+fop_rename_stub(call_frame_t *frame, fop_rename_t fn, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata);
+
+call_stub_t *
+fop_rename_cbk_stub(call_frame_t *frame, fop_rename_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata);
+
+call_stub_t *
+fop_link_stub(call_frame_t *frame, fop_link_t fn, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
+
+call_stub_t *
+fop_link_cbk_stub(call_frame_t *frame, fop_link_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, inode_t *inode, struct iatt *buf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata);
+
+call_stub_t *
+fop_create_stub(call_frame_t *frame, fop_create_t fn, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+call_stub_t *
+fop_create_cbk_stub(call_frame_t *frame, fop_create_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+
+call_stub_t *
+fop_open_stub(call_frame_t *frame, fop_open_t fn, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata);
+
+call_stub_t *
+fop_open_cbk_stub(call_frame_t *frame, fop_open_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+call_stub_t *
+fop_readv_stub(call_frame_t *frame, fop_readv_t fn, fd_t *fd, size_t size,
+               off_t off, uint32_t flags, dict_t *xdata);
+
+call_stub_t *
+fop_readv_cbk_stub(call_frame_t *frame, fop_readv_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iovec *vector, int32_t count,
+                   struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
+
+call_stub_t *
+fop_writev_stub(call_frame_t *frame, fop_writev_t fn, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                struct iobref *iobref, dict_t *xdata);
+
+call_stub_t *
+fop_writev_cbk_stub(call_frame_t *frame, fop_writev_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                    dict_t *xdata);
+
+call_stub_t *
+fop_flush_stub(call_frame_t *frame, fop_flush_t fn, fd_t *fd, dict_t *xdata);
+
+call_stub_t *
+fop_flush_cbk_stub(call_frame_t *frame, fop_flush_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_fsync_stub(call_frame_t *frame, fop_fsync_t fn, fd_t *fd, int32_t datasync,
+               dict_t *xdata);
+
+call_stub_t *
+fop_fsync_cbk_stub(call_frame_t *frame, fop_fsync_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                   dict_t *xdata);
+
+call_stub_t *
+fop_opendir_stub(call_frame_t *frame, fop_opendir_t fn, loc_t *loc, fd_t *fd,
+                 dict_t *xdata);
+
+call_stub_t *
+fop_opendir_cbk_stub(call_frame_t *frame, fop_opendir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+call_stub_t *
+fop_fsyncdir_stub(call_frame_t *frame, fop_fsyncdir_t fn, fd_t *fd,
+                  int32_t datasync, dict_t *xdata);
+
+call_stub_t *
+fop_fsyncdir_cbk_stub(call_frame_t *frame, fop_fsyncdir_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_statfs_stub(call_frame_t *frame, fop_statfs_t fn, loc_t *loc,
+                dict_t *xdata);
+
+call_stub_t *
+fop_statfs_cbk_stub(call_frame_t *frame, fop_statfs_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, struct statvfs *buf, dict_t *xdata);
+
+call_stub_t *
+fop_setxattr_stub(call_frame_t *frame, fop_setxattr_t fn, loc_t *loc,
+                  dict_t *dict, int32_t flags, dict_t *xdata);
+
+call_stub_t *
+fop_setxattr_cbk_stub(call_frame_t *frame, fop_setxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_getxattr_stub(call_frame_t *frame, fop_getxattr_t fn, loc_t *loc,
+                  const char *name, dict_t *xdata);
+
+call_stub_t *
+fop_getxattr_cbk_stub(call_frame_t *frame, fop_getxattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *value,
+                      dict_t *xdata);
+
+call_stub_t *
+fop_fsetxattr_stub(call_frame_t *frame, fop_fsetxattr_t fn, fd_t *fd,
+                   dict_t *dict, int32_t flags, dict_t *xdata);
+
+call_stub_t *
+fop_fsetxattr_cbk_stub(call_frame_t *frame, fop_fsetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_fgetxattr_stub(call_frame_t *frame, fop_fgetxattr_t fn, fd_t *fd,
+                   const char *name, dict_t *xdata);
+
+call_stub_t *
+fop_fgetxattr_cbk_stub(call_frame_t *frame, fop_fgetxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *value,
+                       dict_t *xdata);
+
+call_stub_t *
+fop_removexattr_stub(call_frame_t *frame, fop_removexattr_t fn, loc_t *loc,
+                     const char *name, dict_t *xdata);
+
+call_stub_t *
+fop_removexattr_cbk_stub(call_frame_t *frame, fop_removexattr_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_fremovexattr_stub(call_frame_t *frame, fop_fremovexattr_t fn, fd_t *fd,
+                      const char *name, dict_t *xdata);
+
+call_stub_t *
+fop_fremovexattr_cbk_stub(call_frame_t *frame, fop_fremovexattr_cbk_t fn,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_lk_stub(call_frame_t *frame, fop_lk_t fn, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata);
+
+call_stub_t *
+fop_lk_cbk_stub(call_frame_t *frame, fop_lk_cbk_t fn, int32_t op_ret,
+                int32_t op_errno, struct gf_flock *lock, dict_t *xdata);
+
+call_stub_t *
+fop_inodelk_stub(call_frame_t *frame, fop_inodelk_t fn, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+call_stub_t *
+fop_finodelk_stub(call_frame_t *frame, fop_finodelk_t fn, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+call_stub_t *
+fop_entrylk_stub(call_frame_t *frame, fop_entrylk_t fn, const char *volume,
+                 loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata);
+
+call_stub_t *
+fop_fentrylk_stub(call_frame_t *frame, fop_fentrylk_t fn, const char *volume,
+                  fd_t *fd, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata);
+
+call_stub_t *
+fop_inodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_finodelk_cbk_stub(call_frame_t *frame, fop_inodelk_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_entrylk_cbk_stub(call_frame_t *frame, fop_entrylk_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_fentrylk_cbk_stub(call_frame_t *frame, fop_entrylk_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_readdir_stub(call_frame_t *frame, fop_readdir_t fn, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata);
+
+call_stub_t *
+fop_readdirp_stub(call_frame_t *frame, fop_readdirp_t fn, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata);
+
+call_stub_t *
+fop_readdirp_cbk_stub(call_frame_t *frame, fop_readdir_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
+
+call_stub_t *
+fop_readdir_cbk_stub(call_frame_t *frame, fop_readdir_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
+
+call_stub_t *
+fop_rchecksum_stub(call_frame_t *frame, fop_rchecksum_t fn, fd_t *fd,
+                   off_t offset, int32_t len, dict_t *xdata);
+
+call_stub_t *
+fop_rchecksum_cbk_stub(call_frame_t *frame, fop_rchecksum_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata);
+
+call_stub_t *
+fop_xattrop_stub(call_frame_t *frame, fop_xattrop_t fn, loc_t *loc,
+                 gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+call_stub_t *
+fop_xattrop_stub_cbk_stub(call_frame_t *frame, fop_xattrop_cbk_t fn,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_fxattrop_stub(call_frame_t *frame, fop_fxattrop_t fn, fd_t *fd,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+call_stub_t *
+fop_fxattrop_stub_cbk_stub(call_frame_t *frame, fop_xattrop_cbk_t fn,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_setattr_stub(call_frame_t *frame, fop_setattr_t fn, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+call_stub_t *
+fop_setattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_fsetattr_stub(call_frame_t *frame, fop_fsetattr_t fn, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+call_stub_t *
+fop_fsetattr_cbk_stub(call_frame_t *frame, fop_setattr_cbk_t fn, int32_t op_ret,
+                      int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_fallocate_stub(call_frame_t *frame, fop_fallocate_t fn, fd_t *fd,
+                   int32_t mode, off_t offset, size_t len, dict_t *xdata);
+
+call_stub_t *
+fop_fallocate_cbk_stub(call_frame_t *frame, fop_fallocate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_discard_stub(call_frame_t *frame, fop_discard_t fn, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata);
+
+call_stub_t *
+fop_discard_cbk_stub(call_frame_t *frame, fop_discard_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_zerofill_stub(call_frame_t *frame, fop_zerofill_t fn, fd_t *fd,
+                  off_t offset, off_t len, dict_t *xdata);
+
+call_stub_t *
+fop_zerofill_cbk_stub(call_frame_t *frame, fop_zerofill_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_ipc_stub(call_frame_t *frame, fop_ipc_t fn, int32_t op, dict_t *xdata);
+
+call_stub_t *
+fop_ipc_cbk_stub(call_frame_t *frame, fop_ipc_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_seek_stub(call_frame_t *frame, fop_seek_t fn, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata);
+
+call_stub_t *
+fop_seek_cbk_stub(call_frame_t *frame, fop_seek_cbk_t fn, int32_t op_ret,
+                  int32_t op_errno, off_t offset, dict_t *xdata);
+
+call_stub_t *
+fop_lease_stub(call_frame_t *frame, fop_lease_t fn, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata);
+
+call_stub_t *
+fop_lease_cbk_stub(call_frame_t *frame, fop_lease_cbk_t fn, int32_t op_ret,
+                   int32_t op_errno, struct gf_lease *lease, dict_t *xdata);
+
+call_stub_t *
+fop_getactivelk_stub(call_frame_t *frame, fop_getactivelk_t fn, loc_t *loc,
+                     dict_t *xdata);
+
+call_stub_t *
+fop_getactivelk_cbk_stub(call_frame_t *frame, fop_getactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *lmi, dict_t *xdata);
+
+call_stub_t *
+fop_setactivelk_stub(call_frame_t *frame, fop_setactivelk_t fn, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata);
+
+call_stub_t *
+fop_setactivelk_cbk_stub(call_frame_t *frame, fop_setactivelk_cbk_t fn,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+call_stub_t *
+fop_put_stub(call_frame_t *frame, fop_put_t fn, loc_t *loc, mode_t mode,
+             mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+             off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata);
+
+call_stub_t *
+fop_put_cbk_stub(call_frame_t *frame, fop_put_cbk_t fn, int32_t op_ret,
+                 int32_t op_errno, inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata);
+
+call_stub_t *
+fop_icreate_stub(call_frame_t *frame, fop_icreate_t fn, loc_t *loc, mode_t mode,
+                 dict_t *xdata);
+
+call_stub_t *
+fop_namelink_stub(call_frame_t *frame, fop_namelink_t fn, loc_t *loc,
+                  dict_t *xdata);
+
+call_stub_t *
+fop_icreate_cbk_stub(call_frame_t *frame, fop_icreate_cbk_t fn, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode, struct iatt *buf,
+                     dict_t *xdata);
+
+call_stub_t *
+fop_namelink_cbk_stub(call_frame_t *frame, fop_namelink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata);
+
+call_stub_t *
+fop_copy_file_range_stub(call_frame_t *frame, fop_copy_file_range_t fn,
+                         fd_t *fd_in, off64_t off_in, fd_t *fd_out,
+                         off64_t off_out, size_t len, uint32_t flags,
+                         dict_t *xdata);
+
+call_stub_t *
+fop_copy_file_range_cbk_stub(call_frame_t *frame, fop_copy_file_range_cbk_t fn,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata);
+
+void
+call_resume(call_stub_t *stub);
+void
+call_resume_keep_stub(call_stub_t *stub);
+void
+call_stub_destroy(call_stub_t *stub);
+void
+call_unwind_error(call_stub_t *stub, int op_ret, int op_errno);
+void
+call_unwind_error_keep_stub(call_stub_t *stub, int op_ret, int op_errno);
+
+/*
+ * Sometimes we might want to call just this, perhaps repeatedly, without
+ * having (or being able) to destroy and recreate it.
+ */
+void
+call_resume_wind(call_stub_t *stub);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/checksum.h b/libglusterfs/src/glusterfs/checksum.h
new file mode 100644
index 00000000000..019bb14df71
--- /dev/null
+++ b/libglusterfs/src/glusterfs/checksum.h
@@ -0,0 +1,22 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CHECKSUM_H__
+#define __CHECKSUM_H__
+
+uint32_t
+gf_rsync_weak_checksum(unsigned char *buf, size_t len);
+
+void
+gf_rsync_strong_checksum(unsigned char *buf, size_t len, unsigned char *sum);
+
+void
+gf_rsync_md5_checksum(unsigned char *data, size_t len, unsigned char *md5);
+#endif /* __CHECKSUM_H__ */
diff --git a/libglusterfs/src/glusterfs/circ-buff.h b/libglusterfs/src/glusterfs/circ-buff.h
new file mode 100644
index 00000000000..822345b641b
--- /dev/null
+++ b/libglusterfs/src/glusterfs/circ-buff.h
@@ -0,0 +1,61 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CB_H
+#define _CB_H
+
+#include "glusterfs/common-utils.h"
+
+#define BUFFER_SIZE 10
+#define TOTAL_SIZE BUFFER_SIZE + 1
+
+struct _circular_buffer {
+    struct timeval tv;
+    void *data;
+};
+
+typedef struct _circular_buffer circular_buffer_t;
+
+struct _buffer {
+    unsigned int w_index;
+    size_t size_buffer;
+    gf_boolean_t use_once;
+    /* This variable is assigned the proper value at the time of initing */
+    /* the buffer. It indicates, whether the buffer should be used once */
+    /*  it becomes full. */
+
+    int used_len;
+    /* indicates the amount of circular buffer used. */
+
+    circular_buffer_t **cb;
+    void (*destroy_buffer_data)(void *data);
+    pthread_mutex_t lock;
+};
+
+typedef struct _buffer buffer_t;
+
+int
+cb_add_entry_buffer(buffer_t *buffer, void *item);
+
+void
+cb_buffer_show(buffer_t *buffer);
+
+buffer_t *
+cb_buffer_new(size_t buffer_size, gf_boolean_t use_buffer_once,
+              void (*destroy_data)(void *data));
+
+void
+cb_buffer_destroy(buffer_t *buffer);
+
+void
+cb_buffer_dump(buffer_t *buffer, void *data,
+               int(fn)(circular_buffer_t *buffer, void *data));
+
+#endif /* _CB_H */
diff --git a/libglusterfs/src/glusterfs/client_t.h b/libglusterfs/src/glusterfs/client_t.h
new file mode 100644
index 00000000000..a2c508e1d5c
--- /dev/null
+++ b/libglusterfs/src/glusterfs/client_t.h
@@ -0,0 +1,147 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CLIENT_T_H
+#define _CLIENT_T_H
+
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/locking.h" /* for gf_lock_t, not included by glusterfs.h */
+#include "glusterfs/atomic.h"  /* for gf_atomic_t */
+
+/* auth_data structure is required by RPC layer. But as it is also used in
+ * client_t structure validation, comparision, it is critical that it is defined
+ * in the larger scope of libglusterfs, instead of libgfrpc. With this change,
+ * even RPC will use this structure */
+#define GF_CLIENTT_AUTH_BYTES 400
+typedef struct client_auth_data {
+    int flavour;
+    int datalen;
+    char authdata[GF_CLIENTT_AUTH_BYTES];
+} client_auth_data_t;
+
+struct client_ctx {
+    void *ctx_key;
+    void *ctx_value;
+};
+
+typedef struct _client {
+    struct {
+        /* e.g. protocol/server stashes its ctx here */
+        gf_lock_t lock;
+        unsigned short count;
+        struct client_ctx *ctx;
+    } scratch_ctx;
+    gf_atomic_t bind;
+    gf_atomic_t count;
+    xlator_t *bound_xl;
+    xlator_t *this;
+    int tbl_index;
+    char *client_uid;
+    char *client_name;
+    struct {
+        int flavour;
+        size_t len;
+        char *data;
+        char *username;
+        char *passwd;
+    } auth;
+
+    /* subdir_mount */
+    char *subdir_mount;
+    inode_t *subdir_inode;
+    uuid_t subdir_gfid;
+    int32_t opversion;
+    /* Variable to save fd_count for detach brick */
+    gf_atomic_t fd_cnt;
+} client_t;
+
+#define GF_CLIENTCTX_INITIAL_SIZE 8
+
+struct client_table_entry {
+    client_t *client;
+    int next_free;
+};
+typedef struct client_table_entry cliententry_t;
+
+struct clienttable {
+    unsigned int max_clients;
+    gf_lock_t lock;
+    cliententry_t *cliententries;
+    int first_free;
+    client_t *local;
+};
+typedef struct clienttable clienttable_t;
+
+#define GF_CLIENTTABLE_INITIAL_SIZE 128
+
+/* Signifies no more entries in the client table. */
+#define GF_CLIENTTABLE_END -1
+
+/* This is used to invalidate
+ * the next_free value in an cliententry that has been allocated
+ */
+#define GF_CLIENTENTRY_ALLOCATED -2
+
+void
+gf_client_put(client_t *client, gf_boolean_t *detached);
+
+clienttable_t *
+gf_clienttable_alloc(void);
+
+client_t *
+gf_client_ref(client_t *client);
+
+void
+gf_client_unref(client_t *client);
+
+int
+gf_client_dump_fdtable_to_dict(xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_fdtable(xlator_t *this);
+
+int
+gf_client_dump_inodes_to_dict(xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_inodes(xlator_t *this);
+
+void *
+client_ctx_set(client_t *client, void *key, void *value);
+
+int
+client_ctx_get(client_t *client, void *key, void **value);
+
+int
+client_ctx_del(client_t *client, void *key, void **value);
+
+void
+client_ctx_dump(client_t *client, char *prefix);
+
+int
+gf_client_dump_fdtables_to_dict(xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_fdtables(xlator_t *this);
+
+int
+gf_client_dump_inodes_to_dict(xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_inodes(xlator_t *this);
+
+int
+gf_client_disconnect(client_t *client);
+
+client_t *
+gf_client_get(xlator_t *this, client_auth_data_t *cred, char *client_uid,
+              char *subdir_mount);
+
+#endif /* _CLIENT_T_H */
diff --git a/libglusterfs/src/glusterfs/cluster-syncop.h b/libglusterfs/src/glusterfs/cluster-syncop.h
new file mode 100644
index 00000000000..d0ad5ed548c
--- /dev/null
+++ b/libglusterfs/src/glusterfs/cluster-syncop.h
@@ -0,0 +1,227 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CLUSTER_SYNCOP_H
+#define _CLUSTER_SYNCOP_H
+
+#include <sys/time.h>
+#include <pthread.h>
+#include <ucontext.h>
+
+#include "glusterfs/defaults.h"
+#include "glusterfs/default-args.h"
+#include "glusterfs/syncop.h"
+
+/*********************************************************************
+ *
+ * PARALLEL_FOP_ONLIST:
+ *          Performs file operations in parallel on bricks.
+ * This macro expects a helper function(func) to implement the
+ * functionality.
+ *
+ ********************************************************************/
+#define PARALLEL_FOP_ONLIST(subvols, on, numsubvols, replies, frame, func,     \
+                            args...)                                           \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        cluster_local_t __local = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        void *__old_local = frame->local;                                      \
+                                                                               \
+        __local.replies = replies;                                             \
+        cluster_replies_wipe(replies, numsubvols);                             \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+        if (syncbarrier_init(&__local.barrier))                                \
+            break;                                                             \
+        frame->local = &__local;                                               \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (on[__i]) {                                                     \
+                __count++;                                                     \
+            }                                                                  \
+        }                                                                      \
+        __local.barrier.waitfor = __count;                                     \
+        for (__i = 0; __i < numsubvols; __i++) {                               \
+            if (on[__i]) {                                                     \
+                func(frame, subvols[__i], __i, ##args);                        \
+            }                                                                  \
+        }                                                                      \
+        syncbarrier_wait(&__local.barrier, __count);                           \
+        syncbarrier_destroy(&__local.barrier);                                 \
+        frame->local = __old_local;                                            \
+        STACK_RESET(frame->root);                                              \
+    } while (0)
+
+typedef struct cluster_local_ {
+    default_args_cbk_t *replies;
+    syncbarrier_t barrier;
+} cluster_local_t;
+
+int32_t
+cluster_lookup(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+int32_t
+cluster_setattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata);
+int32_t
+cluster_getxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata);
+int32_t
+cluster_setxattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata);
+
+int
+cluster_inodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                off_t off, size_t size);
+
+int
+cluster_uninodelk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, off_t off, size_t size);
+
+int
+cluster_entrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *locked_on,
+                call_frame_t *frame, xlator_t *this, char *dom, inode_t *inode,
+                const char *name);
+
+int32_t
+cluster_rmdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata);
+
+int32_t
+cluster_unlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+               default_args_cbk_t *replies, unsigned char *output,
+               call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata);
+
+int
+cluster_mkdir(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata);
+
+int32_t
+cluster_readlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata);
+
+int
+cluster_symlink(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+cluster_link(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata);
+
+int
+cluster_mknod(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata);
+
+int
+cluster_unentrylk(xlator_t **subvols, unsigned char *locked_on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, char *dom,
+                  inode_t *inode, const char *name);
+
+int
+cluster_tryentrylk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
+                   call_frame_t *frame, xlator_t *this, char *dom,
+                   inode_t *inode, const char *name);
+
+int32_t
+cluster_fxattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+cluster_xattrop(xlator_t **subvols, unsigned char *on, int numsubvols,
+                default_args_cbk_t *replies, unsigned char *output,
+                call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+cluster_fstat(xlator_t **subvols, unsigned char *on, int numsubvols,
+              default_args_cbk_t *replies, unsigned char *output,
+              call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+cluster_ftruncate(xlator_t **subvols, unsigned char *on, int numsubvols,
+                  default_args_cbk_t *replies, unsigned char *output,
+                  call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata);
+
+int32_t
+cluster_open(xlator_t **subvols, unsigned char *on, int numsubvols,
+             default_args_cbk_t *replies, unsigned char *output,
+             call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata);
+
+int
+cluster_tryinodelk(xlator_t **subvols, unsigned char *on, int numsubvols,
+                   default_args_cbk_t *replies, unsigned char *locked_on,
+                   call_frame_t *frame, xlator_t *this, char *dom,
+                   inode_t *inode, off_t off, size_t size);
+
+int32_t
+cluster_fsetattr(xlator_t **subvols, unsigned char *on, int numsubvols,
+                 default_args_cbk_t *replies, unsigned char *output,
+                 call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+cluster_put(xlator_t **subvols, unsigned char *on, int numsubvols,
+            default_args_cbk_t *replies, unsigned char *output,
+            call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+            off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata);
+
+void
+cluster_replies_wipe(default_args_cbk_t *replies, int num_subvols);
+
+int32_t
+cluster_fop_success_fill(default_args_cbk_t *replies, int numsubvols,
+                         unsigned char *success);
+
+int32_t
+cluster_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata);
+
+int
+cluster_tiebreaker_inodelk(xlator_t **subvols, unsigned char *on,
+                           int numsubvols, default_args_cbk_t *replies,
+                           unsigned char *locked_on, call_frame_t *frame,
+                           xlator_t *this, char *dom, inode_t *inode, off_t off,
+                           size_t size);
+#endif /* !_CLUSTER_SYNCOP_H */
diff --git a/libglusterfs/src/glusterfs/common-utils.h b/libglusterfs/src/glusterfs/common-utils.h
new file mode 100644
index 00000000000..f297fdab5c9
--- /dev/null
+++ b/libglusterfs/src/glusterfs/common-utils.h
@@ -0,0 +1,1256 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _COMMON_UTILS_H
+#define _COMMON_UTILS_H
+
+#include <stdint.h>
+#include <sys/uio.h>
+#include <netdb.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <openssl/md5.h>
+#ifndef GF_BSD_HOST_OS
+#include <alloca.h>
+#endif
+#include <limits.h>
+#include <fnmatch.h>
+#include <uuid/uuid.h>
+
+/* FreeBSD, etc. */
+#ifndef __BITS_PER_LONG
+#define __BITS_PER_LONG (CHAR_BIT * (sizeof(long)))
+#endif
+
+#ifndef ffsll
+#define ffsll(x) __builtin_ffsll(x)
+#endif
+
+void
+trap(void);
+
+#define GF_UNIVERSAL_ANSWER 42 /* :O */
+
+/* To solve type punned error */
+#define VOID(ptr) ((void **)((void *)ptr))
+
+#include "glusterfs/mem-pool.h"
+#include "glusterfs/compat-uuid.h"
+#include "glusterfs/iatt.h"
+#include "glusterfs/libglusterfs-messages.h"
+
+#define STRINGIFY(val) #val
+#define TOSTRING(val) STRINGIFY(val)
+
+#define alloca0(size)                                                          \
+    ({                                                                         \
+        void *__ptr;                                                           \
+        __ptr = alloca(size);                                                  \
+        memset(__ptr, 0, size);                                                \
+        __ptr;                                                                 \
+    })
+
+#define min(a, b) ((a) < (b) ? (a) : (b))
+#define max(a, b) ((a) > (b) ? (a) : (b))
+#define gf_roof(a, b) ((((a) + (b)-1) / ((b != 0) ? (b) : 1)) * (b))
+#define gf_floor(a, b) (((a) / ((b != 0) ? (b) : 1)) * (b))
+
+#define IPv4_ADDR_SIZE 32
+
+#define GF_UNIT_KB 1024ULL
+#define GF_UNIT_MB 1048576ULL
+#define GF_UNIT_GB 1073741824ULL
+#define GF_UNIT_TB 1099511627776ULL
+#define GF_UNIT_PB 1125899906842624ULL
+
+#define GF_UNIT_B_STRING "B"
+#define GF_UNIT_KB_STRING "KB"
+#define GF_UNIT_MB_STRING "MB"
+#define GF_UNIT_GB_STRING "GB"
+#define GF_UNIT_TB_STRING "TB"
+#define GF_UNIT_PB_STRING "PB"
+
+#define GF_UNIT_PERCENT_STRING "%"
+
+#define GEOREP "geo-replication"
+#define GLUSTERD_NAME "glusterd"
+
+#define GF_SELINUX_XATTR_KEY "security.selinux"
+
+#define WIPE(statp)                                                            \
+    do {                                                                       \
+        typeof(*statp) z = {                                                   \
+            0,                                                                 \
+        };                                                                     \
+        if (statp)                                                             \
+            *statp = z;                                                        \
+    } while (0)
+
+#define IS_EXT_FS(fs_name)                                                     \
+    (!strcmp(fs_name, "ext2") || !strcmp(fs_name, "ext3") ||                   \
+     !strcmp(fs_name, "ext4"))
+
+/* process mode definitions */
+#define GF_SERVER_PROCESS 0
+#define GF_CLIENT_PROCESS 1
+#define GF_GLUSTERD_PROCESS 2
+
+/* Defining this here as it is needed by glusterd for setting
+ * nfs port in volume status.
+ */
+#define GF_NFS3_PORT 2049
+
+#define GF_CLIENT_PORT_CEILING 1024
+#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
+#define GF_CLNT_INSECURE_PORT_CEILING (GF_IANA_PRIV_PORTS_START - 1)
+#define GF_PORT_MAX 65535
+#define GF_PORT_ARRAY_SIZE ((GF_PORT_MAX + 7) / 8)
+#define GF_LOCK_TIMER 180
+#define GF_MINUTE_IN_SECONDS 60
+#define GF_HOUR_IN_SECONDS (60 * 60)
+#define GF_DAY_IN_SECONDS (24 * 60 * 60)
+#define GF_WEEK_IN_SECONDS (7 * 24 * 60 * 60)
+#define GF_SEC_IN_NS 1000000000
+#define GF_MS_IN_NS 1000000
+#define GF_US_IN_NS 1000
+
+/* Default timeout for both barrier and changelog translator */
+#define BARRIER_TIMEOUT "120"
+
+/* Default value of signing waiting time to sign a file for bitrot */
+#define SIGNING_TIMEOUT "120"
+#define BR_WORKERS "4"
+
+/* xxhash */
+#define GF_XXH64_DIGEST_LENGTH 8
+#define GF_XXHSUM64_DEFAULT_SEED 0
+
+/* Shard */
+#define GF_XATTR_SHARD_FILE_SIZE "trusted.glusterfs.shard.file-size"
+#define SHARD_ROOT_GFID "be318638-e8a0-4c6d-977d-7a937aa84806"
+#define DOT_SHARD_REMOVE_ME_GFID "77dd5a45-dbf5-4592-b31b-b440382302e9"
+
+/* Lease: buffer length for stringified lease id
+ * Format: 4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum-4hexnum
+ * Eg:6c69-6431-2d63-6c6e-7431-0000-0000-0000
+ */
+#define GF_LEASE_ID_BUF_SIZE ((LEASE_ID_SIZE * 2) + (LEASE_ID_SIZE / 2))
+
+#define GF_PERCENTAGE(val, total) (((val)*100) / (total))
+
+/* pthread related */
+/* as per the man page, thread-name should be at max 16 bytes */
+/* with prefix of 'glfs_' (5), we are left with 11 more bytes */
+#define GF_THREAD_NAME_LIMIT 16
+#define GF_THREAD_NAME_PREFIX "glfs_"
+
+/* Advisory buffer size for formatted timestamps (see gf_time_fmt) */
+#define GF_TIMESTR_SIZE 256
+
+/*
+ * we could have initialized these as +ve values and treated
+ * them as negative while comparing etc.. (which would have
+ * saved us with the pain of assigning values), but since we
+ * only have a few clients that use this feature, it's okay.
+ */
+enum _gf_special_pid {
+    GF_CLIENT_PID_MAX = 0,
+    GF_CLIENT_PID_GSYNCD = -1,
+    GF_CLIENT_PID_HADOOP = -2,
+    GF_CLIENT_PID_DEFRAG = -3,
+    GF_CLIENT_PID_NO_ROOT_SQUASH = -4,
+    GF_CLIENT_PID_QUOTA_MOUNT = -5,
+    GF_CLIENT_PID_SELF_HEALD = -6,
+    GF_CLIENT_PID_GLFS_HEAL = -7,
+    GF_CLIENT_PID_BITD = -8,
+    GF_CLIENT_PID_SCRUB = -9,
+    GF_CLIENT_PID_TIER_DEFRAG = -10,
+    GF_SERVER_PID_TRASH = -11,
+    GF_CLIENT_PID_ADD_REPLICA_MOUNT = -12,
+    GF_CLIENT_PID_SET_UTIME = -13,
+};
+
+enum _gf_xlator_ipc_targets {
+    GF_IPC_TARGET_CHANGELOG = 0,
+    GF_IPC_TARGET_CTR = 1,
+    GF_IPC_TARGET_UPCALL = 2
+};
+
+typedef enum _gf_special_pid gf_special_pid_t;
+typedef enum _gf_xlator_ipc_targets _gf_xlator_ipc_targets_t;
+
+/* Array to hold custom xattr keys */
+extern char *xattrs_to_heal[];
+
+char **
+get_xattrs_to_heal();
+
+/* The DHT file rename operation is not a straightforward rename.
+ * It involves creating linkto and linkfiles, and can unlink or rename the
+ * source file depending on the hashed and cached subvols for the source
+ * and target files. this makes it difficult for geo-rep to figure out that
+ * a rename operation has taken place.
+ *
+ * We now send a special key and the values of the source and target pargfids
+ * and basenames to indicate to changelog that the operation in question
+ * should be treated as a rename. We are explicitly filling and sending this
+ * as a binary value in the dictionary as the unlink op will not have the
+ * source file information. The lengths of the src and target basenames
+ * are used to calculate where to start reading the names in the structure.
+ * XFS allows a max of 255 chars for filenames but other file systems might
+ * not have such restrictions
+ */
+typedef struct dht_changelog_rename_info {
+    uuid_t old_pargfid;
+    uuid_t new_pargfid;
+    int32_t oldname_len;
+    int32_t newname_len;
+    char buffer[1];
+} dht_changelog_rename_info_t;
+
+typedef int (*gf_cmp)(void *, void *);
+
+struct _dict;
+
+struct dnscache {
+    struct _dict *cache_dict;
+    time_t ttl;
+};
+
+struct dnscache_entry {
+    char *ip;
+    char *fqdn;
+    time_t timestamp;
+};
+
+struct dnscache6 {
+    struct addrinfo *first;
+    struct addrinfo *next;
+};
+
+struct list_node {
+    void *ptr;
+    struct list_head list;
+};
+
+extern char *vol_type_str[];
+
+struct list_node *
+list_node_add(void *ptr, struct list_head *list);
+struct list_node *
+list_node_add_order(void *ptr, struct list_head *list,
+                    int (*compare)(struct list_head *, struct list_head *));
+void
+list_node_del(struct list_node *node);
+
+struct dnscache *
+gf_dnscache_init(time_t ttl);
+void
+gf_dnscache_deinit(struct dnscache *cache);
+struct dnscache_entry *
+gf_dnscache_entry_init(void);
+void
+gf_dnscache_entry_deinit(struct dnscache_entry *entry);
+char *
+gf_rev_dns_lookup_cached(const char *ip, struct dnscache *dnscache);
+
+char *
+gf_resolve_path_parent(const char *path);
+
+void
+gf_global_variable_init(void);
+
+int32_t
+gf_resolve_ip6(const char *hostname, uint16_t port, int family, void **dnscache,
+               struct addrinfo **addr_info);
+
+void
+gf_log_dump_graph(FILE *specfp, glusterfs_graph_t *graph);
+void
+gf_print_trace(int32_t signal, glusterfs_ctx_t *ctx);
+int
+gf_set_log_file_path(cmd_args_t *cmd_args, glusterfs_ctx_t *ctx);
+int
+gf_set_log_ident(cmd_args_t *cmd_args);
+
+int
+gf_process_getspec_servers_list(cmd_args_t *cmd_args, const char *servers_list);
+int
+gf_set_volfile_server_common(cmd_args_t *cmd_args, const char *host,
+                             const char *transport, int port);
+
+static inline void
+BIT_SET(unsigned char *array, unsigned int index)
+{
+    unsigned int offset = index / 8;
+    unsigned int shift = index % 8;
+
+    array[offset] |= (1 << shift);
+}
+
+static inline void
+BIT_CLEAR(unsigned char *array, unsigned int index)
+{
+    unsigned int offset = index / 8;
+    unsigned int shift = index % 8;
+
+    array[offset] &= ~(1 << shift);
+}
+
+static inline unsigned int
+BIT_VALUE(unsigned char *array, unsigned int index)
+{
+    unsigned int offset = index / 8;
+    unsigned int shift = index % 8;
+
+    return (array[offset] >> shift) & 0x1;
+}
+
+#define VECTORSIZE(count) (count * (sizeof(struct iovec)))
+
+#define STRLEN_0(str) (strlen(str) + 1)
+
+#define VALIDATE_OR_GOTO(arg, label)                                           \
+    do {                                                                       \
+        if (!arg) {                                                            \
+            errno = EINVAL;                                                    \
+            gf_msg_callingfn((this ? (this->name) : "(Govinda! Govinda!)"),    \
+                             GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,       \
+                             "invalid argument: " #arg);                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_VALIDATE_OR_GOTO(name, arg, label)                                  \
+    do {                                                                       \
+        if (!arg) {                                                            \
+            errno = EINVAL;                                                    \
+            gf_msg_callingfn(name, GF_LOG_ERROR, errno, LG_MSG_INVALID_ARG,    \
+                             "invalid argument: " #arg);                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_VALIDATE_OR_GOTO_WITH_ERROR(name, arg, label, errno, error)         \
+    do {                                                                       \
+        if (!arg) {                                                            \
+            errno = error;                                                     \
+            gf_msg_callingfn(name, GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,   \
+                             "invalid argument: " #arg);                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_CHECK_ALLOC(arg, retval, label)                                     \
+    do {                                                                       \
+        if (!(arg)) {                                                          \
+            retval = -ENOMEM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_CHECK_ALLOC_AND_LOG(name, item, retval, msg, errlabel)              \
+    do {                                                                       \
+        if (!(item)) {                                                         \
+            (retval) = -ENOMEM;                                                \
+            gf_msg(name, GF_LOG_CRITICAL, ENOMEM, LG_MSG_NO_MEMORY, (msg));    \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
+
+#define GF_ASSERT_AND_GOTO_WITH_ERROR(name, arg, label, errno, error)          \
+    do {                                                                       \
+        if (!arg) {                                                            \
+            GF_ASSERT(0);                                                      \
+            errno = error;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO(name, arg, label)                    \
+    do {                                                                       \
+        GF_VALIDATE_OR_GOTO(name, arg, label);                                 \
+        if ((arg[0]) != '/') {                                                 \
+            errno = EINVAL;                                                    \
+            gf_msg_callingfn(name, GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,   \
+                             "invalid argument: " #arg);                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_REMOVE_SLASH_FROM_PATH(path, string)                                \
+    do {                                                                       \
+        int i = 0;                                                             \
+        for (i = 1; i < strlen(path); i++) {                                   \
+            string[i - 1] = path[i];                                           \
+            if (string[i - 1] == '/')                                          \
+                string[i - 1] = '-';                                           \
+        }                                                                      \
+    } while (0)
+
+#define GF_REMOVE_INTERNAL_XATTR(pattern, dict)                                \
+    do {                                                                       \
+        if (!dict) {                                                           \
+            gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_DICT_NULL,              \
+                   "dict is null");                                            \
+            break;                                                             \
+        }                                                                      \
+        dict_foreach_fnmatch(dict, pattern, dict_remove_foreach_fn, NULL);     \
+    } while (0)
+
+#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, op_errno, label)              \
+    do {                                                                       \
+        if (!dict) {                                                           \
+            gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_DICT_NULL,              \
+                   "setxattr dict is null");                                   \
+            goto label;                                                        \
+        }                                                                      \
+        if (dict_foreach_fnmatch(dict, pattern, dict_null_foreach_fn, NULL) >  \
+            0) {                                                               \
+            op_errno = EPERM;                                                  \
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, LG_MSG_NO_PERM,         \
+                   "attempt to set internal"                                   \
+                   " xattr: %s",                                               \
+                   pattern);                                                   \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GF_IF_NATIVE_XATTR_GOTO(pattern, key, op_errno, label)                 \
+    do {                                                                       \
+        if (!key) {                                                            \
+            gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_NO_KEY,                 \
+                   "no key for removexattr");                                  \
+            goto label;                                                        \
+        }                                                                      \
+        if (!fnmatch(pattern, key, 0)) {                                       \
+            op_errno = EPERM;                                                  \
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, LG_MSG_NO_PERM,         \
+                   "attempt to remove internal "                               \
+                   "xattr: %s",                                                \
+                   key);                                                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#ifdef DEBUG
+#define GF_ASSERT(x) assert(x);
+#else
+#define GF_ASSERT(x)                                                           \
+    do {                                                                       \
+        if (!(x)) {                                                            \
+            gf_msg_callingfn("", GF_LOG_ERROR, 0, LG_MSG_ASSERTION_FAILED,     \
+                             "Assertion failed: " #x);                         \
+        }                                                                      \
+    } while (0)
+#endif
+
+/* Compile-time assert, borrowed from Linux kernel. */
+#ifdef HAVE_STATIC_ASSERT
+#define GF_STATIC_ASSERT(expr, ...)                                            \
+    __gf_static_assert(expr, ##__VA_ARGS__, #expr)
+#define __gf_static_assert(expr, msg, ...) _Static_assert(expr, msg)
+#else
+#define GF_STATIC_ASSERT(expr, ...)
+#endif
+
+#define GF_ABORT(msg...)                                                       \
+    do {                                                                       \
+        gf_msg_callingfn("", GF_LOG_CRITICAL, 0, LG_MSG_ASSERTION_FAILED,      \
+                         "Assertion failed: " msg);                            \
+        abort();                                                               \
+    } while (0)
+
+#define GF_UUID_ASSERT(u)                                                      \
+    if (gf_uuid_is_null(u))                                                    \
+        GF_ASSERT(!"uuid null");
+
+#define GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, op_errno)                        \
+    (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&                            \
+      (op_errno == EEXIST || op_errno == ENOENT))                              \
+         ? 0                                                                   \
+         : 1)
+
+union gf_sock_union {
+    struct sockaddr_storage storage;
+    struct sockaddr_in6 sin6;
+    struct sockaddr_in sin;
+    struct sockaddr sa;
+};
+
+#define GF_HIDDEN_PATH ".glusterfs"
+#define GF_UNLINK_PATH GF_HIDDEN_PATH "/unlink"
+#define GF_LANDFILL_PATH GF_HIDDEN_PATH "/landfill"
+
+#define IOV_MIN(n) min(IOV_MAX, n)
+
+static inline gf_boolean_t
+gf_irrelevant_entry(struct dirent *entry)
+{
+    GF_ASSERT(entry);
+
+    return (!strcmp(entry->d_name, ".") ||
+            !fnmatch("*.tmp", entry->d_name, 0) ||
+            !strcmp(entry->d_name, ".."));
+}
+
+static inline void
+iov_free(struct iovec *vector, int count)
+{
+    int i;
+
+    for (i = 0; i < count; i++)
+        FREE(vector[i].iov_base);
+
+    GF_FREE(vector);
+}
+
+static inline int
+iov_length(const struct iovec *vector, int count)
+{
+    int i = 0;
+    size_t size = 0;
+
+    for (i = 0; i < count; i++)
+        size += vector[i].iov_len;
+
+    return size;
+}
+
+static inline struct iovec *
+iov_dup(const struct iovec *vector, int count)
+{
+    int bytecount = 0;
+    struct iovec *newvec = NULL;
+
+    bytecount = (count * sizeof(struct iovec));
+    newvec = GF_MALLOC(bytecount, gf_common_mt_iovec);
+    if (newvec != NULL) {
+        memcpy(newvec, vector, bytecount);
+    }
+
+    return newvec;
+}
+
+typedef struct _iov_iter {
+    const struct iovec *iovec;
+    void *ptr;
+    uint32_t len;
+    uint32_t count;
+} iov_iter_t;
+
+static inline bool
+iov_iter_init(iov_iter_t *iter, const struct iovec *iovec, uint32_t count,
+              uint32_t offset)
+{
+    uint32_t len;
+
+    while (count > 0) {
+        count--;
+        len = iovec->iov_len;
+        if (offset < len) {
+            iter->ptr = iovec->iov_base + offset;
+            iter->len = len - offset;
+            iter->iovec = iovec + 1;
+            iter->count = count;
+
+            return true;
+        }
+        offset -= len;
+    }
+
+    memset(iter, 0, sizeof(*iter));
+
+    return false;
+}
+
+static inline bool
+iov_iter_end(iov_iter_t *iter)
+{
+    return iter->count == 0;
+}
+
+static inline bool
+iov_iter_next(iov_iter_t *iter, uint32_t size)
+{
+    GF_ASSERT(size <= iter->len);
+
+    if (iter->len > size) {
+        iter->len -= size;
+        iter->ptr += size;
+
+        return true;
+    }
+    if (iter->count > 0) {
+        iter->count--;
+        iter->ptr = iter->iovec->iov_base;
+        iter->len = iter->iovec->iov_len;
+        iter->iovec++;
+
+        return true;
+    }
+
+    memset(iter, 0, sizeof(*iter));
+
+    return false;
+}
+
+static inline uint32_t
+iov_iter_copy(iov_iter_t *dst, iov_iter_t *src, uint32_t size)
+{
+    uint32_t len;
+
+    len = src->len;
+    if (len > dst->len) {
+        len = dst->len;
+    }
+    if (len > size) {
+        len = size;
+    }
+    memcpy(dst->ptr, src->ptr, len);
+
+    return len;
+}
+
+static inline uint32_t
+iov_iter_to_iovec(iov_iter_t *iter, struct iovec *iovec, int32_t idx,
+                  uint32_t size)
+{
+    uint32_t len;
+
+    len = iter->len;
+    if (len > size) {
+        len = size;
+    }
+    iovec[idx].iov_base = iter->ptr;
+    iovec[idx].iov_len = len;
+
+    return len;
+}
+
+static inline int
+iov_subset(struct iovec *src, int src_count, uint32_t start, uint32_t size,
+           struct iovec **dst, int32_t dst_count)
+{
+    struct iovec iovec[src_count];
+    iov_iter_t iter;
+    uint32_t len;
+    int32_t idx;
+
+    if ((size == 0) || !iov_iter_init(&iter, src, src_count, start)) {
+        return 0;
+    }
+
+    idx = 0;
+    do {
+        len = iov_iter_to_iovec(&iter, iovec, idx, size);
+        idx++;
+        size -= len;
+    } while ((size > 0) && iov_iter_next(&iter, len));
+
+    if (*dst == NULL) {
+        *dst = iov_dup(iovec, idx);
+        if (*dst == NULL) {
+            return -1;
+        }
+    } else if (idx > dst_count) {
+        return -1;
+    } else {
+        memcpy(*dst, iovec, idx * sizeof(struct iovec));
+    }
+
+    return idx;
+}
+
+static inline int
+iov_skip(struct iovec *iovec, uint32_t count, uint32_t size)
+{
+    uint32_t len, idx;
+
+    idx = 0;
+    while ((size > 0) && (idx < count)) {
+        len = iovec[idx].iov_len;
+        if (len > size) {
+            iovec[idx].iov_len -= size;
+            iovec[idx].iov_base += size;
+            break;
+        }
+        idx++;
+        size -= len;
+    }
+
+    if (idx > 0) {
+        memmove(iovec, iovec + idx, (count - idx) * sizeof(struct iovec));
+    }
+
+    return count - idx;
+}
+
+static inline size_t
+iov_range_copy(const struct iovec *dst, uint32_t dst_count, uint32_t dst_offset,
+               const struct iovec *src, uint32_t src_count, uint32_t src_offset,
+               uint32_t size)
+{
+    iov_iter_t src_iter, dst_iter;
+    uint32_t len, total;
+
+    if ((size == 0) || !iov_iter_init(&src_iter, src, src_count, src_offset) ||
+        !iov_iter_init(&dst_iter, dst, dst_count, dst_offset)) {
+        return 0;
+    }
+
+    total = 0;
+    do {
+        len = iov_iter_copy(&dst_iter, &src_iter, size);
+        total += len;
+        size -= len;
+    } while ((size > 0) && iov_iter_next(&src_iter, len) &&
+             iov_iter_next(&dst_iter, len));
+
+    return total;
+}
+
+static inline void
+iov_unload(char *buf, const struct iovec *vector, int count)
+{
+    int i;
+    int copied = 0;
+
+    for (i = 0; i < count; i++) {
+        memcpy(buf + copied, vector[i].iov_base, vector[i].iov_len);
+        copied += vector[i].iov_len;
+    }
+}
+
+static inline size_t
+iov_load(const struct iovec *vector, int count, char *buf, int size)
+{
+    size_t left = size;
+    size_t cp = 0;
+    int ret = 0;
+    int i = 0;
+
+    while (left && i < count) {
+        cp = min(vector[i].iov_len, left);
+        if (vector[i].iov_base != buf + (size - left))
+            memcpy(vector[i].iov_base, buf + (size - left), cp);
+        ret += cp;
+        left -= cp;
+        if (left)
+            i++;
+    }
+
+    return ret;
+}
+
+static inline size_t
+iov_copy(const struct iovec *dst, int dcnt, const struct iovec *src, int scnt)
+{
+    return iov_range_copy(dst, dcnt, 0, src, scnt, 0, UINT32_MAX);
+}
+
+/* based on the amusing discussion @ https://rusty.ozlabs.org/?p=560 */
+static bool
+memeqzero(const void *data, size_t length)
+{
+    const unsigned char *p = data;
+    size_t len;
+
+    /* Check first 16 bytes manually */
+    for (len = 0; len < 16; len++) {
+        if (!length)
+            return true;
+        if (*p)
+            return false;
+        p++;
+        length--;
+    }
+
+    /* Now we know that's zero, memcmp with self. */
+    return memcmp(data, p, length) == 0;
+}
+
+static inline int
+mem_0filled(const char *buf, size_t size)
+{
+    return !memeqzero(buf, size);
+}
+
+static inline int
+iov_0filled(const struct iovec *vector, int count)
+{
+    int i = 0;
+    int ret = 0;
+
+    for (i = 0; i < count; i++) {
+        ret = mem_0filled(vector[i].iov_base, vector[i].iov_len);
+        if (ret)
+            break;
+    }
+
+    return ret;
+}
+
+typedef enum {
+    gf_timefmt_default = 0,
+    gf_timefmt_FT = 0, /* YYYY-MM-DD hh:mm:ss */
+    gf_timefmt_Ymd_T,  /* YYYY/MM-DD-hh:mm:ss */
+    gf_timefmt_bdT,    /* MMM DD hh:mm:ss */
+    gf_timefmt_F_HMS,  /* YYYY-MM-DD hhmmss */
+    gf_timefmt_dirent,
+    gf_timefmt_s,
+    gf_timefmt_last
+} gf_timefmts;
+
+static inline char *
+gf_time_fmt_tv(char *dst, size_t sz_dst, struct timeval *tv, unsigned int fmt)
+{
+    extern void _gf_timestuff(const char ***, const char ***);
+    static gf_timefmts timefmt_last = (gf_timefmts)-1;
+    static const char **fmts;
+    static const char **zeros;
+    struct tm tm, *res;
+    int localtime = 0;
+    int len = 0;
+    int pos = 0;
+
+    if (timefmt_last == ((gf_timefmts)-1)) {
+        _gf_timestuff(&fmts, &zeros);
+        timefmt_last = gf_timefmt_last;
+    }
+    if (timefmt_last <= fmt) {
+        fmt = gf_timefmt_default;
+    }
+    localtime = gf_log_get_localtime();
+    res = localtime ? localtime_r(&tv->tv_sec, &tm)
+                    : gmtime_r(&tv->tv_sec, &tm);
+    if (tv->tv_sec && (res != NULL)) {
+        len = strftime(dst, sz_dst, fmts[fmt], &tm);
+        if (len == 0)
+            return dst;
+        pos += len;
+        if (tv->tv_usec >= 0) {
+            len = snprintf(dst + pos, sz_dst - pos, ".%" GF_PRI_SUSECONDS,
+                           tv->tv_usec);
+            if (len >= sz_dst - pos)
+                return dst;
+            pos += len;
+        }
+        strftime(dst + pos, sz_dst - pos, " %z", &tm);
+    } else {
+        strncpy(dst, "N/A", sz_dst);
+    }
+    return dst;
+}
+
+static inline char *
+gf_time_fmt(char *dst, size_t sz_dst, time_t utime, unsigned int fmt)
+{
+    struct timeval tv = {utime, -1};
+
+    return gf_time_fmt_tv(dst, sz_dst, &tv, fmt);
+}
+
+/* This function helps us use gfid (unique identity) to generate inode's unique
+ * number in glusterfs.
+ */
+ino_t
+gfid_to_ino(uuid_t gfid);
+
+int
+mkdir_p(char *path, mode_t mode, gf_boolean_t allow_symlinks);
+/*
+ * rounds up nr to power of two. If nr is already a power of two, just returns
+ * nr
+ */
+
+int
+gf_lstat_dir(const char *path, struct stat *stbuf_in);
+
+int32_t
+gf_roundup_power_of_two(int32_t nr);
+
+/*
+ * rounds up nr to next power of two. If nr is already a power of two, next
+ * power of two is returned.
+ */
+
+int32_t
+gf_roundup_next_power_of_two(int32_t nr);
+
+char *
+gf_trim(char *string);
+int
+gf_volume_name_validate(const char *volume_name);
+
+int
+gf_string2long(const char *str, long *n);
+int
+gf_string2ulong(const char *str, unsigned long *n);
+int
+gf_string2int(const char *str, int *n);
+int
+gf_string2uint(const char *str, unsigned int *n);
+int
+gf_string2double(const char *str, double *n);
+int
+gf_string2longlong(const char *str, long long *n);
+int
+gf_string2ulonglong(const char *str, unsigned long long *n);
+
+int
+gf_string2int8(const char *str, int8_t *n);
+int
+gf_string2int16(const char *str, int16_t *n);
+int
+gf_string2int32(const char *str, int32_t *n);
+int
+gf_string2int64(const char *str, int64_t *n);
+int
+gf_string2uint8(const char *str, uint8_t *n);
+int
+gf_string2uint16(const char *str, uint16_t *n);
+int
+gf_string2uint32(const char *str, uint32_t *n);
+int
+gf_string2uint64(const char *str, uint64_t *n);
+
+int
+gf_strstr(const char *str, const char *delim, const char *match);
+
+int
+gf_string2ulong_base10(const char *str, unsigned long *n);
+int
+gf_string2uint_base10(const char *str, unsigned int *n);
+int
+gf_string2uint8_base10(const char *str, uint8_t *n);
+int
+gf_string2uint16_base10(const char *str, uint16_t *n);
+int
+gf_string2uint32_base10(const char *str, uint32_t *n);
+int
+gf_string2uint64_base10(const char *str, uint64_t *n);
+int
+gf_string2bytesize_uint64(const char *str, uint64_t *n);
+int
+gf_string2bytesize_int64(const char *str, int64_t *n);
+int
+gf_string2percent_or_bytesize(const char *str, double *n,
+                              gf_boolean_t *is_percent);
+
+int
+gf_string2boolean(const char *str, gf_boolean_t *b);
+int
+gf_strn2boolean(const char *str, const int len, gf_boolean_t *b);
+int
+gf_string2percent(const char *str, double *n);
+int
+gf_string2time(const char *str, uint32_t *n);
+
+int
+gf_lockfd(int fd);
+int
+gf_unlockfd(int fd);
+
+int
+get_checksum_for_file(int fd, uint32_t *checksum, int op_version);
+int
+log_base2(unsigned long x);
+
+int
+get_checksum_for_path(char *path, uint32_t *checksum, int op_version);
+int
+get_file_mtime(const char *path, time_t *stamp);
+char *
+gf_resolve_path_parent(const char *path);
+
+char *
+strtail(char *str, const char *pattern);
+void
+skipwhite(char **s);
+char *
+nwstrtail(char *str, char *pattern);
+/* returns a new string with nth word of given string. n>=1 */
+
+typedef struct token_iter {
+    char *end;
+    char sep;
+} token_iter_t;
+char *
+token_iter_init(char *str, char sep, token_iter_t *tit);
+gf_boolean_t
+next_token(char **tokenp, token_iter_t *tit);
+void
+drop_token(char *token, token_iter_t *tit);
+
+gf_boolean_t
+mask_match(const uint32_t a, const uint32_t b, const uint32_t m);
+gf_boolean_t
+gf_is_ip_in_net(const char *network, const char *ip_str);
+char
+valid_host_name(char *address, int length);
+char
+valid_ipv4_address(char *address, int length, gf_boolean_t wildcard_acc);
+char
+valid_ipv6_address(char *address, int length, gf_boolean_t wildcard_acc);
+char
+valid_internet_address(char *address, gf_boolean_t wildcard_acc,
+                       gf_boolean_t cidr);
+gf_boolean_t
+valid_mount_auth_address(char *address);
+gf_boolean_t
+valid_ipv4_subnetwork(const char *address);
+gf_boolean_t
+gf_sock_union_equal_addr(union gf_sock_union *a, union gf_sock_union *b);
+char *
+gf_rev_dns_lookup(const char *ip);
+
+char *
+uuid_utoa(uuid_t uuid);
+char *
+uuid_utoa_r(uuid_t uuid, char *dst);
+char *
+lkowner_utoa(gf_lkowner_t *lkowner);
+char *
+lkowner_utoa_r(gf_lkowner_t *lkowner, char *dst, int len);
+char *
+leaseid_utoa(const char *lease_id);
+gf_boolean_t
+is_valid_lease_id(const char *lease_id);
+char *
+gf_leaseid_get(void);
+char *
+gf_existing_leaseid(void);
+
+void
+gf_array_insertionsort(void *a, int l, int r, size_t elem_size, gf_cmp cmp);
+int
+gf_is_str_int(const char *value);
+
+char *gf_uint64_2human_readable(uint64_t);
+int
+validate_brick_name(char *brick);
+char *
+get_host_name(char *word, char **host);
+char *
+get_path_name(char *word, char **path);
+void
+gf_path_strip_trailing_slashes(char *path);
+uint64_t
+get_mem_size(void);
+int
+gf_strip_whitespace(char *str, int len);
+int
+gf_canonicalize_path(char *path);
+char *
+generate_glusterfs_ctx_id(void);
+char *
+gf_get_reserved_ports(void);
+int
+gf_process_reserved_ports(unsigned char *ports, uint32_t ceiling);
+gf_boolean_t
+gf_ports_reserved(char *blocked_port, unsigned char *ports, uint32_t ceiling);
+int
+gf_get_hostname_from_ip(char *client_ip, char **hostname);
+gf_boolean_t
+gf_is_local_addr(char *hostname);
+gf_boolean_t
+gf_is_same_address(char *host1, char *host2);
+void
+gf_xxh64_wrapper(const unsigned char *data, size_t const len,
+                 unsigned long long const seed, char *xxh64);
+int
+gf_gfid_generate_from_xxh64(uuid_t gfid, char *key);
+
+int
+gf_set_timestamp(const char *src, const char *dest);
+
+int
+gf_thread_create(pthread_t *thread, const pthread_attr_t *attr,
+                 void *(*start_routine)(void *), void *arg, const char *name,
+                 ...) __attribute__((__format__(__printf__, 5, 6)));
+
+int
+gf_thread_vcreate(pthread_t *thread, const pthread_attr_t *attr,
+                  void *(*start_routine)(void *), void *arg, const char *name,
+                  va_list args);
+int
+gf_thread_create_detached(pthread_t *thread, void *(*start_routine)(void *),
+                          void *arg, const char *name, ...)
+    __attribute__((__format__(__printf__, 4, 5)));
+
+void
+gf_thread_set_name(pthread_t thread, const char *name, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+void
+gf_thread_set_vname(pthread_t thread, const char *name, va_list args);
+gf_boolean_t
+gf_is_pid_running(int pid);
+gf_boolean_t
+gf_is_service_running(char *pidfile, int *pid);
+gf_boolean_t
+gf_valid_pid(const char *pid, int length);
+int
+gf_skip_header_section(int fd, int header_len);
+
+struct iatt;
+struct _dict;
+
+gf_boolean_t
+dht_is_linkfile(struct iatt *buf, struct _dict *dict);
+
+int
+gf_check_log_format(const char *value);
+
+int
+gf_check_logger(const char *value);
+
+gf_boolean_t
+gf_compare_sockaddr(const struct sockaddr *addr1, const struct sockaddr *addr2);
+
+char *
+gf_backtrace_save(char *buf);
+
+void
+gf_backtrace_done(char *buf);
+
+gf_loglevel_t
+fop_log_level(glusterfs_fop_t fop, int op_errno);
+
+int32_t
+gf_build_absolute_path(char *current_path, char *relative_path, char **path);
+
+int
+recursive_rmdir(const char *delete_path);
+
+int
+gf_get_index_by_elem(char **array, char *elem);
+
+int
+glusterfs_is_local_pathinfo(char *pathinfo, gf_boolean_t *local);
+
+int
+gf_thread_cleanup_xint(pthread_t thread);
+
+ssize_t
+gf_nread(int fd, void *buf, size_t count);
+
+ssize_t
+gf_nwrite(int fd, const void *buf, size_t count);
+
+void
+_mask_cancellation(void);
+void
+_unmask_cancellation(void);
+
+gf_boolean_t
+gf_is_zero_filled_stat(struct iatt *buf);
+
+void
+gf_zero_fill_stat(struct iatt *buf);
+
+gf_boolean_t
+gf_is_valid_xattr_namespace(char *k);
+
+const char *
+gf_inode_type_to_str(ia_type_t type);
+
+int32_t
+gf_bits_count(uint64_t n);
+
+int32_t
+gf_bits_index(uint64_t n);
+
+const char *
+gf_fop_string(glusterfs_fop_t fop);
+
+int
+gf_fop_int(char *fop);
+
+char *
+get_ip_from_addrinfo(struct addrinfo *addr, char **ip);
+
+int
+close_fds_except(int *fdv, size_t count);
+
+int
+gf_getgrouplist(const char *user, gid_t group, gid_t **groups);
+
+int
+glusterfs_compute_sha256(const unsigned char *content, size_t size,
+                         char *sha256_hash);
+
+char *
+gf_strncpy(char *dest, const char *src, const size_t dest_size);
+
+void
+gf_strTrim(char **s);
+
+int
+gf_replace_old_iatt_in_dict(struct _dict *);
+
+int
+gf_replace_new_iatt_in_dict(struct _dict *);
+
+xlator_cmdline_option_t *
+find_xlator_option_in_cmd_args_t(const char *option_name, cmd_args_t *args);
+
+int
+gf_d_type_from_ia_type(ia_type_t type);
+
+int
+gf_syncfs(int fd);
+
+int
+gf_nanosleep(uint64_t nsec);
+
+static inline time_t
+gf_time(void)
+{
+    return time(NULL);
+}
+
+/* Return delta value in microseconds. */
+
+static inline double
+gf_tvdiff(struct timeval *start, struct timeval *end)
+{
+    struct timeval t;
+
+    if (start->tv_usec > end->tv_usec)
+        t.tv_sec = end->tv_sec - 1, t.tv_usec = end->tv_usec + 1000000;
+    else
+        t.tv_sec = end->tv_sec, t.tv_usec = end->tv_usec;
+
+    return (double)(t.tv_sec - start->tv_sec) * 1e6 +
+           (double)(t.tv_usec - start->tv_usec);
+}
+
+/* Return delta value in nanoseconds. */
+
+static inline double
+gf_tsdiff(struct timespec *start, struct timespec *end)
+{
+    struct timespec t;
+
+    if (start->tv_nsec > end->tv_nsec)
+        t.tv_sec = end->tv_sec - 1, t.tv_nsec = end->tv_nsec + 1000000000;
+    else
+        t.tv_sec = end->tv_sec, t.tv_nsec = end->tv_nsec;
+
+    return (double)(t.tv_sec - start->tv_sec) * 1e9 +
+           (double)(t.tv_nsec - start->tv_nsec);
+}
+
+#endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/glusterfs/compat-errno.h b/libglusterfs/src/glusterfs/compat-errno.h
new file mode 100644
index 00000000000..c4ab09ab0d5
--- /dev/null
+++ b/libglusterfs/src/glusterfs/compat-errno.h
@@ -0,0 +1,238 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __COMPAT_ERRNO_H__
+#define __COMPAT_ERRNO_H__
+
+#include <errno.h>
+
+#define GF_ERROR_CODE_SUCCESS 0
+#define GF_ERROR_CODE_UNKNOWN 1024
+#define GF_ERRNO_UNKNOWN 1024
+
+#define GF_ERROR_CODE_PERM 1         /* Operation not permitted */
+#define GF_ERROR_CODE_NOENT 2        /* No such file or directory */
+#define GF_ERROR_CODE_SRCH 3         /* No such process */
+#define GF_ERROR_CODE_INTR 4         /* Interrupted system call */
+#define GF_ERROR_CODE_IO 5           /* I/O error */
+#define GF_ERROR_CODE_NXIO 6         /* No such device or address */
+#define GF_ERROR_CODE_2BIG 7         /* Argument list too long */
+#define GF_ERROR_CODE_NOEXEC 8       /* Exec format error */
+#define GF_ERROR_CODE_BADF 9         /* Bad file number */
+#define GF_ERROR_CODE_CHILD 10       /* No child processes */
+#define GF_ERROR_CODE_AGAIN 11       /* Try again */
+#define GF_ERROR_CODE_NOMEM 12       /* Out of memory */
+#define GF_ERROR_CODE_ACCES 13       /* Permission denied */
+#define GF_ERROR_CODE_FAULT 14       /* Bad address */
+#define GF_ERROR_CODE_NOTBLK 15      /* Block device required */
+#define GF_ERROR_CODE_BUSY 16        /* Device or resource busy */
+#define GF_ERROR_CODE_EXIST 17       /* File exists */
+#define GF_ERROR_CODE_XDEV 18        /* Cross-device link */
+#define GF_ERROR_CODE_NODEV 19       /* No such device */
+#define GF_ERROR_CODE_NOTDIR 20      /* Not a directory */
+#define GF_ERROR_CODE_ISDIR 21       /* Is a directory */
+#define GF_ERROR_CODE_INVAL 22       /* Invalid argument */
+#define GF_ERROR_CODE_NFILE 23       /* File table overflow */
+#define GF_ERROR_CODE_MFILE 24       /* Too many open files */
+#define GF_ERROR_CODE_NOTTY 25       /* Not a typewriter */
+#define GF_ERROR_CODE_TXTBSY 26      /* Text file busy */
+#define GF_ERROR_CODE_FBIG 27        /* File too large */
+#define GF_ERROR_CODE_NOSPC 28       /* No space left on device */
+#define GF_ERROR_CODE_SPIPE 29       /* Illegal seek */
+#define GF_ERROR_CODE_ROFS 30        /* Read-only file system */
+#define GF_ERROR_CODE_MLINK 31       /* Too many links */
+#define GF_ERROR_CODE_PIPE 32        /* Broken pipe */
+#define GF_ERROR_CODE_DOM 33         /* Math argument out of domain of func */
+#define GF_ERROR_CODE_RANGE 34       /* Math result not representable */
+#define GF_ERROR_CODE_DEADLK 35      /* Resource deadlock would occur */
+#define GF_ERROR_CODE_NAMETOOLONG 36 /* File name too long */
+#define GF_ERROR_CODE_NOLCK 37       /* No record locks available */
+#define GF_ERROR_CODE_NOSYS 38       /* Function not implemented */
+#define GF_ERROR_CODE_NOTEMPTY 39    /* Directory not empty */
+#define GF_ERROR_CODE_LOOP 40        /* Too many symbolic links encountered */
+
+#define GF_ERROR_CODE_NOMSG 42    /* No message of desired type */
+#define GF_ERROR_CODE_IDRM 43     /* Identifier removed */
+#define GF_ERROR_CODE_CHRNG 44    /* Channel number out of range */
+#define GF_ERROR_CODE_L2NSYNC 45  /* Level 2 not synchronized */
+#define GF_ERROR_CODE_L3HLT 46    /* Level 3 halted */
+#define GF_ERROR_CODE_L3RST 47    /* Level 3 reset */
+#define GF_ERROR_CODE_LNRNG 48    /* Link number out of range */
+#define GF_ERROR_CODE_UNATCH 49   /* Protocol driver not attached */
+#define GF_ERROR_CODE_NOCSI 50    /* No CSI structure available */
+#define GF_ERROR_CODE_L2HLT 51    /* Level 2 halted */
+#define GF_ERROR_CODE_BADE 52     /* Invalid exchange */
+#define GF_ERROR_CODE_BADR 53     /* Invalid request descriptor */
+#define GF_ERROR_CODE_XFULL 54    /* Exchange full */
+#define GF_ERROR_CODE_NOANO 55    /* No anode */
+#define GF_ERROR_CODE_BADRQC 56   /* Invalid request code */
+#define GF_ERROR_CODE_BADSLT 57   /* Invalid slot */
+#define GF_ERROR_CODE_BFONT 59    /* Bad font file format */
+#define GF_ERROR_CODE_NOSTR 60    /* Device not a stream */
+#define GF_ERROR_CODE_NODATA 61   /* No data available */
+#define GF_ERROR_CODE_TIME 62     /* Timer expired */
+#define GF_ERROR_CODE_NOSR 63     /* Out of streams resources */
+#define GF_ERROR_CODE_NONET 64    /* Machine is not on the network */
+#define GF_ERROR_CODE_NOPKG 65    /* Package not installed */
+#define GF_ERROR_CODE_REMOTE 66   /* Object is remote */
+#define GF_ERROR_CODE_NOLINK 67   /* Link has been severed */
+#define GF_ERROR_CODE_ADV 68      /* Advertise error */
+#define GF_ERROR_CODE_SRMNT 69    /* Srmount error */
+#define GF_ERROR_CODE_COMM 70     /* Communication error on send */
+#define GF_ERROR_CODE_PROTO 71    /* Protocol error */
+#define GF_ERROR_CODE_MULTIHOP 72 /* Multihop attempted */
+#define GF_ERROR_CODE_DOTDOT 73   /* RFS specific error */
+#define GF_ERROR_CODE_BADMSG 74   /* Not a data message */
+#define GF_ERROR_CODE_OVERFLOW 75 /* Value too large for defined data type */
+#define GF_ERROR_CODE_NOTUNIQ 76  /* Name not unique on network */
+#define GF_ERROR_CODE_BADFD 77    /* File descriptor in bad state */
+#define GF_ERROR_CODE_REMCHG 78   /* Remote address changed */
+#define GF_ERROR_CODE_LIBACC 79   /* Can not access a needed shared library */
+#define GF_ERROR_CODE_LIBBAD 80   /* Accessing a corrupted shared library */
+#define GF_ERROR_CODE_LIBSCN 81   /* .lib section in a.out corrupted */
+#define GF_ERROR_CODE_LIBMAX                                                   \
+    82 /* Attempting to link in too many shared libraries */
+#define GF_ERROR_CODE_LIBEXEC 83 /* Cannot exec a shared library directly */
+#define GF_ERROR_CODE_ILSEQ 84   /* Illegal byte sequence */
+#define GF_ERROR_CODE_RESTART                                                  \
+    85 /* Interrupted system call should be restarted */
+#define GF_ERROR_CODE_STRPIPE 86        /* Streams pipe error */
+#define GF_ERROR_CODE_USERS 87          /* Too many users */
+#define GF_ERROR_CODE_NOTSOCK 88        /* Socket operation on non-socket */
+#define GF_ERROR_CODE_DESTADDRREQ 89    /* Destination address required */
+#define GF_ERROR_CODE_MSGSIZE 90        /* Message too long */
+#define GF_ERROR_CODE_PROTOTYPE 91      /* Protocol wrong type for socket */
+#define GF_ERROR_CODE_NOPROTOOPT 92     /* Protocol not available */
+#define GF_ERROR_CODE_PROTONOSUPPORT 93 /* Protocol not supported */
+#define GF_ERROR_CODE_SOCKTNOSUPPORT 94 /* Socket type not supported */
+#define GF_ERROR_CODE_OPNOTSUPP                                                \
+    95 /* Operation not supported on transport endpoint */
+#define GF_ERROR_CODE_PFNOSUPPORT 96 /* Protocol family not supported */
+#define GF_ERROR_CODE_AFNOSUPPORT                                              \
+    97 /* Address family not supported by protocol */
+#define GF_ERROR_CODE_ADDRINUSE 98    /* Address already in use */
+#define GF_ERROR_CODE_ADDRNOTAVAIL 99 /* Cannot assign requested address */
+#define GF_ERROR_CODE_NETDOWN 100     /* Network is down */
+#define GF_ERROR_CODE_NETUNREACH 101  /* Network is unreachable */
+#define GF_ERROR_CODE_NETRESET                                                 \
+    102 /* Network dropped connection because of reset */
+#define GF_ERROR_CODE_CONNABORTED 103 /* Software caused connection abort */
+#define GF_ERROR_CODE_CONNRESET 104   /* Connection reset by peer */
+#define GF_ERROR_CODE_NOBUFS 105      /* No buffer space available */
+#define GF_ERROR_CODE_ISCONN 106  /* Transport endpoint is already connected */
+#define GF_ERROR_CODE_NOTCONN 107 /* Transport endpoint is not connected */
+#define GF_ERROR_CODE_SHUTDOWN                                                 \
+    108 /* Cannot send after transport endpoint shutdown */
+#define GF_ERROR_CODE_TOOMANYREFS 109 /* Too many references: cannot splice */
+#define GF_ERROR_CODE_TIMEDOUT 110    /* Connection timed out */
+#define GF_ERROR_CODE_CONNREFUSED 111 /* Connection refused */
+#define GF_ERROR_CODE_HOSTDOWN 112    /* Host is down */
+#define GF_ERROR_CODE_HOSTUNREACH 113 /* No route to host */
+#define GF_ERROR_CODE_ALREADY 114     /* Operation already in progress */
+#define GF_ERROR_CODE_INPROGRESS 115  /* Operation now in progress */
+#define GF_ERROR_CODE_ALREADY 114     /* Operation already in progress */
+#define GF_ERROR_CODE_INPROGRESS 115  /* Operation now in progress */
+#define GF_ERROR_CODE_STALE 116       /* Stale NFS file handle */
+#define GF_ERROR_CODE_UCLEAN 117      /* Structure needs cleaning */
+#define GF_ERROR_CODE_NOTNAM 118      /* Not a XENIX named type file */
+#define GF_ERROR_CODE_NAVAIL 119      /* No XENIX semaphores available */
+#define GF_ERROR_CODE_ISNAM 120       /* Is a named type file */
+#define GF_ERROR_CODE_REMOTEIO 121    /* Remote I/O error */
+#define GF_ERROR_CODE_DQUOT 122       /* Quota exceeded */
+#define GF_ERROR_CODE_NOMEDIUM 123    /* No medium found */
+#define GF_ERROR_CODE_MEDIUMTYPE 124  /* Wrong medium type */
+#define GF_ERROR_CODE_CANCELED 125    /* Operation Canceled */
+#define GF_ERROR_CODE_NOKEY 126       /* Required key not available */
+#define GF_ERROR_CODE_KEYEXPIRED 127  /* Key has expired */
+#define GF_ERROR_CODE_KEYREVOKED 128  /* Key has been revoked */
+#define GF_ERROR_CODE_KEYREJECTED 129 /* Key was rejected by service */
+
+/* for robust mutexes */
+#define GF_ERROR_CODE_OWNERDEAD 130      /* Owner died */
+#define GF_ERROR_CODE_NOTRECOVERABLE 131 /* State not recoverable */
+
+/* Should never be seen by user programs */
+#define GF_ERROR_CODE_RESTARTSYS 512
+#define GF_ERROR_CODE_RESTARTNOINTR 513
+#define GF_ERROR_CODE_RESTARTNOHAND 514 /* restart if no handler.. */
+#define GF_ERROR_CODE_NOIOCTLCMD 515    /* No ioctl command */
+#define GF_ERROR_CODE_RESTART_RESTARTBLOCK                                     \
+    516 /* restart by calling sys_restart_syscall */
+
+/* Defined for the NFSv3 protocol */
+#define GF_ERROR_CODE_BADHANDLE 521   /* Illegal NFS file handle */
+#define GF_ERROR_CODE_NOTSYNC 522     /* Update synchronization mismatch */
+#define GF_ERROR_CODE_BADCOOKIE 523   /* Cookie is stale */
+#define GF_ERROR_CODE_NOTSUPP 524     /* Operation is not supported */
+#define GF_ERROR_CODE_TOOSMALL 525    /* Buffer or request is too small */
+#define GF_ERROR_CODE_SERVERFAULT 526 /* An untranslatable error occurred */
+#define GF_ERROR_CODE_BADTYPE 527     /* Type not supported by server */
+#define GF_ERROR_CODE_JUKEBOX                                                  \
+    528 /* Request initiated, but will not complete before timeout */
+#define GF_ERROR_CODE_IOCBQUEUED                                               \
+    529                             /* iocb queued, will get completion event */
+#define GF_ERROR_CODE_IOCBRETRY 530 /* iocb queued, will trigger a retry */
+
+/* Darwin OS X */
+#define GF_ERROR_CODE_NOPOLICY 701
+#define GF_ERROR_CODE_BADMACHO 702
+#define GF_ERROR_CODE_PWROFF 703
+#define GF_ERROR_CODE_DEVERR 704
+#define GF_ERROR_CODE_BADARCH 705
+#define GF_ERROR_CODE_BADEXEC 706
+#define GF_ERROR_CODE_SHLIBVERS 707
+
+/* Solaris */
+/*	ENOTACTIVE 73	/ * Facility is not active		*/
+#define GF_ERROR_CODE_NOTACTIVE 801
+/*	ELOCKUNMAPPED	72	/ * locked lock was unmapped */
+#define GF_ERROR_CODE_LOCKUNMAPPED 802
+
+/* BSD system */
+#define GF_ERROR_CODE_PROCLIM 901      /* Too many processes */
+#define GF_ERROR_CODE_BADRPC 902       /* RPC struct is bad */
+#define GF_ERROR_CODE_RPCMISMATCH 903  /* RPC version wrong */
+#define GF_ERROR_CODE_PROGUNAVAIL 904  /* RPC prog. not avail */
+#define GF_ERROR_CODE_PROGMISMATCH 905 /* Program version wrong */
+#define GF_ERROR_CODE_PROCUNAVAIL 905  /* Bad procedure for program */
+#define GF_ERROR_CODE_FTYPE 906        /* Inappropriate file type or format */
+#define GF_ERROR_CODE_AUTH 907         /* Authentication error */
+#define GF_ERROR_CODE_NEEDAUTH 908     /* Need authenticator */
+#define GF_ERROR_CODE_DOOFUS 909       /* Programming error */
+
+#define GF_ERROR_CODE_NOATTR GF_ERROR_CODE_NODATA /* Attribute not found */
+
+/* Either one of enodata or enoattr will be there in system */
+#ifndef ENOATTR
+#define ENOATTR ENODATA
+#endif /* ENOATTR */
+
+#ifndef ENODATA
+#define ENODATA ENOATTR
+#endif /* ENODATA */
+
+#ifndef EBADFD
+#define EBADFD EBADRPC
+#endif /* EBADFD */
+
+#if !defined(ENODATA)
+/* This happens on FreeBSD.  Value borrowed from Linux. */
+#define ENODATA 61
+#endif
+
+/* These functions are defined for all the OS flags, but content will
+ * be different for each OS flag.
+ */
+int32_t
+gf_errno_to_error(int32_t op_errno);
+int32_t
+gf_error_to_errno(int32_t error);
+
+#endif /* __COMPAT_ERRNO_H__ */
diff --git a/libglusterfs/src/glusterfs/compat-uuid.h b/libglusterfs/src/glusterfs/compat-uuid.h
new file mode 100644
index 00000000000..6e7fdefbfab
--- /dev/null
+++ b/libglusterfs/src/glusterfs/compat-uuid.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _GF_UUID_H
+#define _GF_UUID_H
+
+#include <uuid/uuid.h>
+
+static inline void
+gf_uuid_clear(uuid_t uuid)
+{
+    uuid_clear(uuid);
+}
+
+static inline int
+gf_uuid_compare(uuid_t u1, uuid_t u2)
+{
+    return uuid_compare(u1, u2);
+}
+
+static inline void
+gf_uuid_copy(uuid_t dst, const uuid_t src)
+{
+    uuid_copy(dst, src);
+}
+
+static inline void
+gf_uuid_generate(uuid_t uuid)
+{
+    uuid_generate(uuid);
+}
+
+static inline int
+gf_uuid_is_null(uuid_t uuid)
+{
+    return uuid_is_null(uuid);
+}
+
+static inline int
+gf_uuid_parse(const char *in, uuid_t uuid)
+{
+    return uuid_parse(in, uuid);
+}
+
+static inline void
+gf_uuid_unparse(const uuid_t uuid, char *out)
+{
+    uuid_unparse(uuid, out);
+}
+
+/* TODO: add more uuid APIs, use constructs like this:
+#if defined(__NetBSD__) * NetBSD libc *
+
+#include <string.h>
+
+static inline void
+gf_uuid_clear (uuid_t uuid)
+{
+        memset (uuid, 0, sizeof (uuid_t));
+}
+#endif
+*/
+
+#endif /* _GF_UUID_H */
diff --git a/libglusterfs/src/glusterfs/compat.h b/libglusterfs/src/glusterfs/compat.h
new file mode 100644
index 00000000000..bf00d903152
--- /dev/null
+++ b/libglusterfs/src/glusterfs/compat.h
@@ -0,0 +1,544 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __COMPAT_H__
+#define __COMPAT_H__
+
+#include <stdint.h>
+
+#ifndef LLONG_MAX
+#define LLONG_MAX __LONG_LONG_MAX__ /* compat with old gcc */
+#endif                              /* LLONG_MAX */
+
+#ifdef GF_LINUX_HOST_OS
+
+#define UNIX_PATH_MAX 108
+
+#include <sys/un.h>
+#include <linux/limits.h>
+#include <sys/xattr.h>
+#include <linux/xattr.h>
+#include <endian.h>
+#ifdef HAVE_LINUX_FALLOC_H
+#include <linux/falloc.h>
+#endif
+
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#endif
+
+#ifndef _PATH_UMOUNT
+#define _PATH_UMOUNT "/bin/umount"
+#endif
+#define GF_XATTR_NAME_MAX XATTR_NAME_MAX
+#endif /* GF_LINUX_HOST_OS */
+
+/*
+ * Define the fallocate flags in case we do not have the header. This also
+ * accounts for older systems that do not define FALLOC_FL_PUNCH_HOLE.
+ */
+
+#ifndef FALLOC_FL_KEEP_SIZE
+#define FALLOC_FL_KEEP_SIZE 0x01 /* default is extend size */
+#endif
+#ifndef FALLOC_FL_PUNCH_HOLE
+#define FALLOC_FL_PUNCH_HOLE 0x02 /* de-allocates range */
+#endif
+#ifndef FALLOC_FL_ZERO_RANGE
+#define FALLOC_FL_ZERO_RANGE 0x10 /* zeroes out range */
+#endif
+#ifndef FALLOC_FL_COLLAPSE_RANGE
+#define FALLOC_FL_COLLAPSE_RANGE 0x08 /* reduces the size */
+#endif
+#ifndef FALLOC_FL_INSERT_RANGE
+#define FALLOC_FL_INSERT_RANGE 0x20 /* expands the size */
+#endif
+
+#ifndef HAVE_LLISTXATTR
+
+/* This part is valid only in case of old glibc which doesn't support
+ * 'llistxattr()' system calls.
+ */
+
+#define lremovexattr(path, key) removexattr(path, key)
+#define llistxattr(path, key, size) listxattr(path, key, size)
+#define lgetxattr(path, key, value, size) getxattr(path, key, value, size)
+#define lsetxattr(path, key, value, size, flags)                               \
+    setxattr(path, key, value, size, flags)
+
+#endif /* HAVE_LLISTXATTR */
+
+#ifdef GF_DARWIN_HOST_OS
+#include <machine/endian.h>
+#include <libkern/OSByteOrder.h>
+#include <sys/xattr.h>
+
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+
+#endif
+
+#ifdef GF_BSD_HOST_OS
+/* In case of FreeBSD and NetBSD */
+
+#define UNIX_PATH_MAX 104
+#include <sys/types.h>
+
+#include <sys/un.h>
+#include <sys/endian.h>
+#include <sys/extattr.h>
+#ifdef HAVE_SYS_XATTR_H
+#include <sys/xattr.h>
+#endif /* HAVE_SYS_XATTR_H */
+#include <limits.h>
+
+#include <libgen.h>
+/*
+ * This is where things like off64_t are defined.
+ * So include it before declaring _OFF64_T_DECLARED.
+ * If the freebsd version has support for off64_t
+ * including stdio.h should be sufficient.
+ */
+#include <stdio.h>
+
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+
+#ifndef XATTR_CREATE
+enum {
+    ATTR_CREATE = 1,
+#define XATTR_CREATE ATTR_CREATE
+    ATTR_REPLACE = 2
+#define XATTR_REPLACE ATTR_REPLACE
+};
+#endif /* XATTR_CREATE */
+
+#ifndef sighandler_t
+#define sighandler_t sig_t
+#endif
+
+#ifdef __FreeBSD__
+#undef ino_t
+#define ino_t uint64_t
+#include <sys/types.h>
+#include <sys/extattr.h>
+/* Using NAME_MAX since EXTATTR_MAXNAMELEN is inside a preprocessor conditional
+ * for the kernel
+ */
+#define GF_XATTR_NAME_MAX NAME_MAX
+#endif /* __FreeBSD__ */
+
+#ifdef __NetBSD__
+#define GF_XATTR_NAME_MAX XATTR_NAME_MAX
+#endif
+
+#ifndef ino64_t
+#define ino64_t ino_t
+#endif
+
+#ifndef EUCLEAN
+#define EUCLEAN 0
+#endif
+
+#include <netinet/in.h>
+#ifndef s6_addr16
+#define s6_addr16 __u6_addr.__u6_addr16
+#endif
+#ifndef s6_addr32
+#define s6_addr32 __u6_addr.__u6_addr32
+#endif
+
+#ifndef LOGIN_NAME_MAX
+#define LOGIN_NAME_MAX 256
+#endif
+
+/* Posix dictates NAME_MAX to be used */
+#ifndef NAME_MAX
+#ifdef MAXNAMLEN
+#define NAME_MAX MAXNAMLEN
+#else
+#define NAME_MAX 255
+#endif
+#endif
+
+#define F_GETLK64 F_GETLK
+#define F_SETLK64 F_SETLK
+#define F_SETLKW64 F_SETLKW
+#define FALLOC_FL_KEEP_SIZE 0x01      /* default is extend size */
+#define FALLOC_FL_PUNCH_HOLE 0x02     /* de-allocates range */
+#define FALLOC_FL_ZERO_RANGE 0x10     /* zeroes out range */
+#define FALLOC_FL_INSERT_RANGE 0x20   /* Expands the size */
+#define FALLOC_FL_COLLAPSE_RANGE 0x08 /* Reduces the size */
+
+#ifndef _PATH_UMOUNT
+#define _PATH_UMOUNT "/sbin/umount"
+#endif
+
+void
+gf_extattr_list_reshape(char *list, ssize_t size);
+
+#endif /* GF_BSD_HOST_OS */
+
+#ifdef GF_DARWIN_HOST_OS
+#include <machine/endian.h>
+#include <libkern/OSByteOrder.h>
+
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+
+#define UNIX_PATH_MAX 104
+/* OSX Yosemite now has this defined */
+#ifndef AT_SYMLINK_NOFOLLOW
+#define AT_SYMLINK_NOFOLLOW 0x100
+#endif
+#include <sys/types.h>
+
+#include <sys/un.h>
+#include <sys/xattr.h>
+#include <limits.h>
+
+#include <libgen.h>
+
+#if __DARWIN_64_BIT_INO_T == 0
+#error '64 bit ino_t is must for GlusterFS to work, Compile with "CFLAGS=-D__DARWIN_64_BIT_INO_T"'
+#endif /* __DARWIN_64_BIT_INO_T */
+
+#if __DARWIN_64_BIT_INO_T == 0
+#error '64 bit ino_t is must for GlusterFS to work, Compile with "CFLAGS=-D__DARWIN_64_BIT_INO_T"'
+#endif /* __DARWIN_64_BIT_INO_T */
+
+#ifndef sighandler_t
+#define sighandler_t sig_t
+#endif
+
+#ifndef EUCLEAN
+#define EUCLEAN 0
+#endif
+
+#include <netinet/in.h>
+#ifndef s6_addr16
+#define s6_addr16 __u6_addr.__u6_addr16
+#endif
+#ifndef s6_addr32
+#define s6_addr32 __u6_addr.__u6_addr32
+#endif
+
+/* Posix dictates NAME_MAX to be used */
+#ifndef NAME_MAX
+#ifdef MAXNAMLEN
+#define NAME_MAX MAXNAMLEN
+#else
+#define NAME_MAX 255
+#endif
+#endif
+
+#define F_GETLK64 F_GETLK
+#define F_SETLK64 F_SETLK
+#define F_SETLKW64 F_SETLKW
+
+#ifndef FTW_CONTINUE
+#define FTW_CONTINUE 0
+#endif
+
+#ifndef _PATH_UMOUNT
+#define _PATH_UMOUNT "/sbin/umount"
+#endif
+#endif /* GF_DARWIN_HOST_OS */
+
+#ifdef GF_SOLARIS_HOST_OS
+
+#define UNIX_PATH_MAX 108
+#define EUCLEAN 117
+
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/fcntl.h>
+#include <libgen.h>
+#include <sys/mkdev.h>
+
+#ifndef lchmod
+#define lchmod chmod
+#endif
+
+#define lgetxattr(path, key, value, size)                                      \
+    solaris_getxattr(path, key, value, size)
+enum {
+    ATTR_CREATE = 1,
+#define XATTR_CREATE ATTR_CREATE
+    ATTR_REPLACE = 2
+#define XATTR_REPLACE ATTR_REPLACE
+};
+
+/* This patch is not present in Solaris 10 and before */
+#ifndef dirfd
+#define dirfd(dirp) ((dirp)->dd_fd)
+#endif
+
+/* Posix dictates NAME_MAX to be used */
+#ifndef NAME_MAX
+#ifdef MAXNAMLEN
+#define NAME_MAX MAXNAMLEN
+#else
+#define NAME_MAX 255
+#endif
+#endif
+
+#include <netinet/in.h>
+#ifndef s6_addr16
+#define S6_ADDR16(x) ((uint16_t *)((char *)&(x).s6_addr))
+#endif
+#ifndef s6_addr32
+#define s6_addr32 _S6_un._S6_u32
+#endif
+
+#define lutimes(filename, times) utimes(filename, times)
+
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#endif
+
+enum {
+    DT_UNKNOWN = 0,
+#define DT_UNKNOWN DT_UNKNOWN
+    DT_FIFO = 1,
+#define DT_FIFO DT_FIFO
+    DT_CHR = 2,
+#define DT_CHR DT_CHR
+    DT_DIR = 4,
+#define DT_DIR DT_DIR
+    DT_BLK = 6,
+#define DT_BLK DT_BLK
+    DT_REG = 8,
+#define DT_REG DT_REG
+    DT_LNK = 10,
+#define DT_LNK DT_LNK
+    DT_SOCK = 12,
+#define DT_SOCK DT_SOCK
+    DT_WHT = 14
+#define DT_WHT DT_WHT
+};
+
+#ifndef _PATH_MOUNTED
+#define _PATH_MOUNTED "/etc/mtab"
+#endif
+#ifndef _PATH_UMOUNT
+#define _PATH_UMOUNT "/sbin/umount"
+#endif
+
+#ifndef O_ASYNC
+#ifdef FASYNC
+#define O_ASYNC FASYNC
+#else
+#define O_ASYNC 0
+#endif
+#endif
+
+#ifndef FTW_CONTINUE
+#define FTW_CONTINUE 0
+#endif
+
+int
+asprintf(char **string_ptr, const char *format, ...);
+
+int
+vasprintf(char **result, const char *format, va_list args);
+char *
+strsep(char **str, const char *delims);
+int
+solaris_listxattr(const char *path, char *list, size_t size);
+int
+solaris_removexattr(const char *path, const char *key);
+int
+solaris_getxattr(const char *path, const char *key, char *value, size_t size);
+int
+solaris_setxattr(const char *path, const char *key, const char *value,
+                 size_t size, int flags);
+int
+solaris_fgetxattr(int fd, const char *key, char *value, size_t size);
+int
+solaris_fsetxattr(int fd, const char *key, const char *value, size_t size,
+                  int flags);
+int
+solaris_flistxattr(int fd, char *list, size_t size);
+
+int
+solaris_rename(const char *oldpath, const char *newpath);
+
+int
+solaris_unlink(const char *pathname);
+
+char *
+mkdtemp(char *temp);
+
+#define GF_SOLARIS_XATTR_DIR ".glusterfs_xattr_inode"
+
+int
+solaris_xattr_resolve_path(const char *real_path, char **path);
+
+#endif /* GF_SOLARIS_HOST_OS */
+
+#ifndef HAVE_ARGP
+#include "argp.h"
+#else
+#include <argp.h>
+#endif /* HAVE_ARGP */
+
+#ifndef HAVE_STRNLEN
+size_t
+strnlen(const char *string, size_t maxlen);
+#endif /* STRNLEN */
+
+#ifndef strdupa
+#define strdupa(s)                                                             \
+    (__extension__({                                                           \
+        __const char *__old = (s);                                             \
+        size_t __len = strlen(__old) + 1;                                      \
+        char *__new = (char *)__builtin_alloca(__len);                         \
+        (char *)memcpy(__new, __old, __len);                                   \
+    }))
+#endif
+
+#define GF_DIR_ALIGN(x) (((x) + sizeof(uint64_t) - 1) & ~(sizeof(uint64_t) - 1))
+
+#include <sys/types.h>
+#include <dirent.h>
+
+static inline int32_t
+dirent_size(struct dirent *entry)
+{
+    int32_t size = -1;
+
+#ifdef GF_BSD_HOST_OS
+    size = GF_DIR_ALIGN(24 /* FIX MEEEE!!! */ + entry->d_namlen);
+#endif
+#ifdef GF_DARWIN_HOST_OS
+    size = GF_DIR_ALIGN(24 /* FIX MEEEE!!! */ + entry->d_namlen);
+#endif
+#ifdef GF_LINUX_HOST_OS
+    size = GF_DIR_ALIGN(24 /* FIX MEEEE!!! */ + entry->d_reclen);
+#endif
+#ifdef GF_SOLARIS_HOST_OS
+    size = GF_DIR_ALIGN(24 /* FIX MEEEE!!! */ + entry->d_reclen);
+#endif
+    return size;
+}
+
+#ifdef THREAD_UNSAFE_BASENAME
+char *
+basename_r(const char *);
+#define basename(path) basename_r(path)
+#endif /* THREAD_UNSAFE_BASENAME */
+
+#ifdef THREAD_UNSAFE_DIRNAME
+char *
+dirname_r(char *path);
+#define dirname(path) dirname_r(path)
+#endif /* THREAD_UNSAFE_DIRNAME */
+
+#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
+/* Linux, Solaris, Cygwin */
+#define ST_ATIM_SEC(stbuf) ((stbuf)->st_atim.tv_sec)
+#define ST_CTIM_SEC(stbuf) ((stbuf)->st_ctim.tv_sec)
+#define ST_MTIM_SEC(stbuf) ((stbuf)->st_mtim.tv_sec)
+#define ST_ATIM_SEC_SET(stbuf, val) ((stbuf)->st_atim.tv_sec = (val))
+#define ST_MTIM_SEC_SET(stbuf, val) ((stbuf)->st_mtim.tv_sec = (val))
+#define ST_CTIM_SEC_SET(stbuf, val) ((stbuf)->st_ctim.tv_sec = (val))
+#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atim.tv_nsec)
+#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctim.tv_nsec)
+#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtim.tv_nsec)
+#define ST_ATIM_NSEC_SET(stbuf, val) ((stbuf)->st_atim.tv_nsec = (val))
+#define ST_MTIM_NSEC_SET(stbuf, val) ((stbuf)->st_mtim.tv_nsec = (val))
+#define ST_CTIM_NSEC_SET(stbuf, val) ((stbuf)->st_ctim.tv_nsec = (val))
+#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC)
+/* FreeBSD, NetBSD */
+#define ST_ATIM_SEC(stbuf) ((stbuf)->st_atimespec.tv_sec)
+#define ST_CTIM_SEC(stbuf) ((stbuf)->st_ctimespec.tv_sec)
+#define ST_MTIM_SEC(stbuf) ((stbuf)->st_mtimespec.tv_sec)
+#define ST_ATIM_SEC_SET(stbuf, val) ((stbuf)->st_atimespec.tv_sec = (val))
+#define ST_MTIM_SEC_SET(stbuf, val) ((stbuf)->st_mtimespec.tv_sec = (val))
+#define ST_CTIM_SEC_SET(stbuf, val) ((stbuf)->st_ctimespec.tv_sec = (val))
+#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atimespec.tv_nsec)
+#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctimespec.tv_nsec)
+#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtimespec.tv_nsec)
+#define ST_ATIM_NSEC_SET(stbuf, val) ((stbuf)->st_atimespec.tv_nsec = (val))
+#define ST_MTIM_NSEC_SET(stbuf, val) ((stbuf)->st_mtimespec.tv_nsec = (val))
+#define ST_CTIM_NSEC_SET(stbuf, val) ((stbuf)->st_ctimespec.tv_nsec = (val))
+#else
+#define ST_ATIM_NSEC(stbuf) (0)
+#define ST_CTIM_NSEC(stbuf) (0)
+#define ST_MTIM_NSEC(stbuf) (0)
+#define ST_ATIM_NSEC_SET(stbuf, val)                                           \
+    do {                                                                       \
+    } while (0);
+#define ST_MTIM_NSEC_SET(stbuf, val)                                           \
+    do {                                                                       \
+    } while (0);
+#define ST_CTIM_NSEC_SET(stbuf, val)                                           \
+    do {                                                                       \
+    } while (0);
+#endif
+
+#ifdef GF_BSD_HOST_OS
+#define CLOCK_REALTIME_COARSE CLOCK_REALTIME
+#endif
+
+#if defined(__GNUC__) && !defined(RELAX_POISONING)
+/* Use run API, see run.h */
+#include <stdlib.h> /* system(), mkostemp() */
+#include <stdio.h>  /* popen() */
+#ifdef GF_LINUX_HOST_OS
+#include <sys/sysmacros.h>
+#endif
+#pragma GCC poison system mkostemp popen
+#endif
+
+int
+gf_umount_lazy(char *xlname, char *path, int rmdir);
+
+#ifndef GF_XATTR_NAME_MAX
+#error 'Please define GF_XATTR_NAME_MAX for your OS distribution.'
+#endif
+
+#endif /* __COMPAT_H__ */
diff --git a/libglusterfs/src/glusterfs/daemon.h b/libglusterfs/src/glusterfs/daemon.h
new file mode 100644
index 00000000000..48850800b5e
--- /dev/null
+++ b/libglusterfs/src/glusterfs/daemon.h
@@ -0,0 +1,20 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DAEMON_H
+#define _DAEMON_H
+
+#define DEVNULLPATH "/dev/null"
+
+int
+os_daemon_return(int nochdir, int noclose);
+int
+os_daemon(int nochdir, int noclose);
+#endif /*_DAEMON_H */
diff --git a/libglusterfs/src/glusterfs/default-args.h b/libglusterfs/src/glusterfs/default-args.h
new file mode 100644
index 00000000000..ca7526fcab6
--- /dev/null
+++ b/libglusterfs/src/glusterfs/default-args.h
@@ -0,0 +1,455 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* libglusterfs/src/defaults.h:
+       This file contains definition of default fops and mops functions.
+*/
+
+#ifndef _DEFAULT_ARGS_H
+#define _DEFAULT_ARGS_H
+
+#include "glusterfs/xlator.h"
+
+int
+args_lookup_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, inode_t *inode, struct iatt *buf,
+                      dict_t *xdata, struct iatt *postparent);
+
+int
+args_stat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    struct iatt *buf, dict_t *xdata);
+
+int
+args_fstat_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *buf, dict_t *xdata);
+
+int
+args_truncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata);
+
+int
+args_ftruncate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata);
+
+int
+args_access_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, dict_t *xdata);
+
+int
+args_readlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, const char *path, struct iatt *stbuf,
+                        dict_t *xdata);
+
+int
+args_mknod_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
+
+int
+args_mkdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
+
+int
+args_unlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata);
+
+int
+args_rmdir_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata);
+
+int
+args_symlink_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, inode_t *inode, struct iatt *buf,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata);
+
+int
+args_rename_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *buf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata);
+
+int
+args_link_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+
+int
+args_create_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, fd_t *fd, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata);
+
+int
+args_open_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    fd_t *fd, dict_t *xdata);
+
+int
+args_readv_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iovec *vector, int32_t count, struct iatt *stbuf,
+                     struct iobref *iobref, dict_t *xdata);
+
+int
+args_writev_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata);
+
+int
+args_put_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata);
+
+int
+args_flush_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     dict_t *xdata);
+
+int
+args_fsync_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata);
+
+int
+args_opendir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+int
+args_fsyncdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata);
+
+int
+args_statfs_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                      int32_t op_errno, struct statvfs *buf, dict_t *xdata);
+
+int
+args_setxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata);
+
+int
+args_getxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *dict, dict_t *xdata);
+
+int
+args_fsetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata);
+
+int
+args_fgetxattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, dict_t *dict, dict_t *xdata);
+
+int
+args_removexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, dict_t *xdata);
+
+int
+args_fremovexattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                            int32_t op_errno, dict_t *xdata);
+
+int
+args_lk_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                  struct gf_flock *lock, dict_t *xdata);
+
+int
+args_inodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata);
+
+int
+args_finodelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata);
+
+int
+args_entrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xdata);
+
+int
+args_fentrylk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata);
+
+int
+args_readdirp_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
+
+int
+args_readdir_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
+
+int
+args_rchecksum_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, uint32_t weak_checksum,
+                         uint8_t *strong_checksum, dict_t *xdata);
+
+int
+args_xattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, dict_t *xattr, dict_t *xdata);
+
+int
+args_fxattrop_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, dict_t *xattr, dict_t *xdata);
+
+int
+args_setattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata);
+
+int
+args_fsetattr_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata);
+
+int
+args_fallocate_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                         int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata);
+
+int
+args_discard_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                       int32_t op_errno, struct iatt *statpre,
+                       struct iatt *statpost, dict_t *xdata);
+
+int
+args_zerofill_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                        int32_t op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata);
+
+int
+args_ipc_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                   dict_t *xdata);
+
+int
+args_seek_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                    off_t offset, dict_t *xdata);
+
+void
+args_lease_cbk_store(default_args_cbk_t *args, int32_t op_ret, int32_t op_errno,
+                     struct gf_lease *lease, dict_t *xdata);
+
+int
+args_copy_file_range_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                               int32_t op_errno, struct iatt *stbuf,
+                               struct iatt *prebuf_dst,
+                               struct iatt *postbuf_dst, dict_t *xdata);
+
+void
+args_cbk_wipe(default_args_cbk_t *args_cbk);
+
+void
+args_wipe(default_args_t *args);
+
+int
+args_lookup_store(default_args_t *args, loc_t *loc, dict_t *xdata);
+
+int
+args_stat_store(default_args_t *args, loc_t *loc, dict_t *xdata);
+
+int
+args_fstat_store(default_args_t *args, fd_t *fd, dict_t *xdata);
+
+int
+args_truncate_store(default_args_t *args, loc_t *loc, off_t off, dict_t *xdata);
+int
+args_ftruncate_store(default_args_t *args, fd_t *fd, off_t off, dict_t *xdata);
+
+int
+args_access_store(default_args_t *args, loc_t *loc, int32_t mask,
+                  dict_t *xdata);
+
+int
+args_readlink_store(default_args_t *args, loc_t *loc, size_t size,
+                    dict_t *xdata);
+
+int
+args_mknod_store(default_args_t *args, loc_t *loc, mode_t mode, dev_t rdev,
+                 mode_t umask, dict_t *xdata);
+
+int
+args_mkdir_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+                 dict_t *xdata);
+
+int
+args_unlink_store(default_args_t *args, loc_t *loc, int xflag, dict_t *xdata);
+
+int
+args_rmdir_store(default_args_t *args, loc_t *loc, int flags, dict_t *xdata);
+
+int
+args_symlink_store(default_args_t *args, const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata);
+
+int
+args_rename_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                  dict_t *xdata);
+
+int
+args_link_store(default_args_t *args, loc_t *oldloc, loc_t *newloc,
+                dict_t *xdata);
+
+int
+args_create_store(default_args_t *args, loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata);
+
+int
+args_open_store(default_args_t *args, loc_t *loc, int32_t flags, fd_t *fd,
+                dict_t *xdata);
+
+int
+args_readv_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                 uint32_t flags, dict_t *xdata);
+
+int
+args_writev_store(default_args_t *args, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t off, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata);
+
+int
+args_put_store(default_args_t *args, loc_t *loc, mode_t mode, mode_t umask,
+               uint32_t flags, struct iovec *vector, int32_t count, off_t off,
+               struct iobref *iobref, dict_t *xattr, dict_t *xdata);
+
+int
+args_flush_store(default_args_t *args, fd_t *fd, dict_t *xdata);
+
+int
+args_fsync_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                 dict_t *xdata);
+
+int
+args_opendir_store(default_args_t *args, loc_t *loc, fd_t *fd, dict_t *xdata);
+
+int
+args_fsyncdir_store(default_args_t *args, fd_t *fd, int32_t datasync,
+                    dict_t *xdata);
+
+int
+args_statfs_store(default_args_t *args, loc_t *loc, dict_t *xdata);
+
+int
+args_setxattr_store(default_args_t *args, loc_t *loc, dict_t *dict,
+                    int32_t flags, dict_t *xdata);
+
+int
+args_getxattr_store(default_args_t *args, loc_t *loc, const char *name,
+                    dict_t *xdata);
+
+int
+args_fsetxattr_store(default_args_t *args, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata);
+
+int
+args_fgetxattr_store(default_args_t *args, fd_t *fd, const char *name,
+                     dict_t *xdata);
+
+int
+args_removexattr_store(default_args_t *args, loc_t *loc, const char *name,
+                       dict_t *xdata);
+
+int
+args_fremovexattr_store(default_args_t *args, fd_t *fd, const char *name,
+                        dict_t *xdata);
+
+int
+args_lk_store(default_args_t *args, fd_t *fd, int32_t cmd,
+              struct gf_flock *lock, dict_t *xdata);
+
+int
+args_inodelk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int
+args_finodelk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int
+args_entrylk_store(default_args_t *args, const char *volume, loc_t *loc,
+                   const char *name, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata);
+
+int
+args_fentrylk_store(default_args_t *args, const char *volume, fd_t *fd,
+                    const char *name, entrylk_cmd cmd, entrylk_type type,
+                    dict_t *xdata);
+int
+args_readdirp_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                    dict_t *xdata);
+
+int
+args_readdir_store(default_args_t *args, fd_t *fd, size_t size, off_t off,
+                   dict_t *xdata);
+
+int
+args_rchecksum_store(default_args_t *args, fd_t *fd, off_t offset, int32_t len,
+                     dict_t *xdata);
+
+int
+args_xattrop_store(default_args_t *args, loc_t *loc, gf_xattrop_flags_t optype,
+                   dict_t *xattr, dict_t *xdata);
+
+int
+args_fxattrop_store(default_args_t *args, fd_t *fd, gf_xattrop_flags_t optype,
+                    dict_t *xattr, dict_t *xdata);
+
+int
+args_setattr_store(default_args_t *args, loc_t *loc, struct iatt *stbuf,
+                   int32_t valid, dict_t *xdata);
+
+int
+args_fsetattr_store(default_args_t *args, fd_t *fd, struct iatt *stbuf,
+                    int32_t valid, dict_t *xdata);
+
+int
+args_fallocate_store(default_args_t *args, fd_t *fd, int32_t mode, off_t offset,
+                     size_t len, dict_t *xdata);
+
+int
+args_discard_store(default_args_t *args, fd_t *fd, off_t offset, size_t len,
+                   dict_t *xdata);
+
+int
+args_zerofill_store(default_args_t *args, fd_t *fd, off_t offset, off_t len,
+                    dict_t *xdata);
+
+int
+args_ipc_store(default_args_t *args, int32_t op, dict_t *xdata);
+
+int
+args_seek_store(default_args_t *args, fd_t *fd, off_t offset,
+                gf_seek_what_t what, dict_t *xdata);
+
+void
+args_lease_store(default_args_t *args, loc_t *loc, struct gf_lease *lease,
+                 dict_t *xdata);
+
+int
+args_getactivelk_cbk_store(default_args_cbk_t *args, int32_t op_ret,
+                           int32_t op_errno, lock_migration_info_t *locklist,
+                           dict_t *xdata);
+
+int
+args_setactivelk_store(default_args_t *args, loc_t *loc,
+                       lock_migration_info_t *locklist, dict_t *xdata);
+
+int
+args_icreate_store(default_args_t *args, loc_t *loc, mode_t mode,
+                   dict_t *xdata);
+
+int
+args_namelink_store(default_args_t *args, loc_t *loc, dict_t *xdata);
+
+int
+args_copy_file_range_store(default_args_t *args, fd_t *fd_in, off64_t off_in,
+                           fd_t *fd_out, off_t off64_out, size_t len,
+                           uint32_t flags, dict_t *xdata);
+
+void
+args_cbk_init(default_args_cbk_t *args_cbk);
+#endif /* _DEFAULT_ARGS_H */
diff --git a/libglusterfs/src/glusterfs/defaults.h b/libglusterfs/src/glusterfs/defaults.h
new file mode 100644
index 00000000000..5a818eeb91a
--- /dev/null
+++ b/libglusterfs/src/glusterfs/defaults.h
@@ -0,0 +1,1275 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* libglusterfs/src/defaults.h:
+       This file contains definition of default fops and mops functions.
+*/
+
+#ifndef _DEFAULTS_H
+#define _DEFAULTS_H
+
+#include "glusterfs/xlator.h"
+
+typedef struct {
+    int op_ret;
+    int op_errno;
+    inode_t *inode;
+    struct iatt stat;
+    struct iatt prestat;
+    struct iatt poststat;
+    struct iatt preparent;   /* @preoldparent in rename_cbk */
+    struct iatt postparent;  /* @postoldparent in rename_cbk */
+    struct iatt preparent2;  /* @prenewparent in rename_cbk */
+    struct iatt postparent2; /* @postnewparent in rename_cbk */
+    const char *buf;
+    struct iovec *vector;
+    int count;
+    struct iobref *iobref;
+    fd_t *fd;
+    struct statvfs statvfs;
+    dict_t *xattr;
+    struct gf_flock lock;
+    uint32_t weak_checksum;
+    uint8_t *strong_checksum;
+    dict_t *xdata;
+    gf_dirent_t entries;
+    off_t offset; /* seek hole/data */
+    int valid;    /* If the response is valid or not. For call-stub it is
+                     always valid irrespective of this */
+    struct gf_lease lease;
+    lock_migration_info_t locklist;
+} default_args_cbk_t;
+
+typedef struct {
+    loc_t loc;    /* @old in rename(), link() */
+    loc_t loc2;   /* @new in rename(), link() */
+    fd_t *fd;     /* for all the fd based ops */
+    fd_t *fd_dst; /* Only for copy_file_range destination */
+    off_t offset;
+    /*
+     * According to the man page of copy_file_range,
+     * the offsets for source and destination file
+     * are of type loff_t. But the type loff_t is
+     * linux specific and is actual a typedef of
+     * off64_t.
+     */
+    off64_t off_in;  /* For copy_file_range source fd */
+    off64_t off_out; /* For copy_file_range destination fd only */
+    int mask;
+    size_t size;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+    int xflag;
+    int flags;
+    const char *linkname;
+    struct iovec *vector;
+    int count;
+    struct iobref *iobref;
+    int datasync;
+    dict_t *xattr;
+    const char *name;
+    int cmd;
+    struct gf_flock lock;
+    const char *volume;
+    entrylk_cmd entrylkcmd;
+    entrylk_type entrylktype;
+    gf_xattrop_flags_t optype;
+    int valid;
+    struct iatt stat;
+    gf_seek_what_t what;
+    dict_t *xdata;
+    struct gf_lease lease;
+    lock_migration_info_t locklist;
+} default_args_t;
+
+typedef struct {
+    int fop_enum;
+    unsigned int fop_length;
+    int *enum_list;
+    default_args_t *req_list;
+    dict_t *xdata;
+} compound_args_t;
+
+typedef struct {
+    int fop_enum;
+    unsigned int fop_length;
+    int *enum_list;
+    default_args_cbk_t *rsp_list;
+    dict_t *xdata;
+} compound_args_cbk_t;
+
+int32_t
+default_notify(xlator_t *this, int32_t event, void *data, ...);
+
+int32_t
+default_forget(xlator_t *this, inode_t *inode);
+
+int32_t
+default_release(xlator_t *this, fd_t *fd);
+
+int32_t
+default_releasedir(xlator_t *this, fd_t *fd);
+
+extern struct xlator_fops *default_fops;
+
+/* Management Operations */
+
+int32_t
+default_getspec(call_frame_t *frame, xlator_t *this, const char *key,
+                int32_t flag);
+
+int32_t
+default_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  int32_t len, dict_t *xdata);
+
+/* FileSystem operations */
+int32_t
+default_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+default_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+default_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata);
+
+int32_t
+default_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata);
+
+int32_t
+default_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+               dict_t *xdata);
+
+int32_t
+default_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata);
+
+int32_t
+default_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t
+default_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata);
+
+int32_t
+default_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata);
+
+int32_t
+default_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata);
+
+int32_t
+default_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+default_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata);
+
+int32_t
+default_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata);
+
+int32_t
+default_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata);
+
+int32_t
+default_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+default_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t offset,
+               uint32_t flags, struct iobref *iobref, dict_t *xdata);
+
+int32_t
+default_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+              dict_t *xdata);
+
+int32_t
+default_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata);
+
+int32_t
+default_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 int32_t datasync, dict_t *xdata);
+
+int32_t
+default_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+default_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata);
+
+int32_t
+default_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata);
+
+int32_t
+default_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int32_t flags, dict_t *xdata);
+
+int32_t
+default_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata);
+
+int32_t
+default_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata);
+
+int32_t
+default_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata);
+
+int32_t
+default_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+default_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+default_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+default_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata);
+
+int32_t
+default_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata);
+
+int32_t
+default_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata);
+
+int32_t
+default_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata);
+
+int32_t
+default_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+default_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+default_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+default_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+default_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  int32_t keep_size, off_t offset, size_t len, dict_t *xdata);
+
+int32_t
+default_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata);
+
+int32_t
+default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata);
+
+int32_t
+default_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
+
+int32_t
+default_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata);
+
+int32_t
+default_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+default_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata);
+
+int32_t
+default_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    lock_migration_info_t *locklist, dict_t *xdata);
+
+int32_t
+default_put(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+            off_t off, struct iobref *iobref, dict_t *xattr, dict_t *xdata);
+
+int32_t
+default_icreate(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dict_t *xdata);
+
+int32_t
+default_namelink(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 dict_t *xdata);
+
+int32_t
+default_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                        off64_t off_in, fd_t *fd_out, off64_t off_out,
+                        size_t len, uint32_t flags, dict_t *xdata);
+
+/* Resume */
+int32_t
+default_getspec_resume(call_frame_t *frame, xlator_t *this, const char *key,
+                       int32_t flag);
+
+int32_t
+default_rchecksum_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, int32_t len, dict_t *xdata);
+
+/* FileSystem operations */
+int32_t
+default_lookup_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata);
+
+int32_t
+default_stat_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata);
+
+int32_t
+default_fstat_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     dict_t *xdata);
+
+int32_t
+default_truncate_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        off_t offset, dict_t *xdata);
+
+int32_t
+default_ftruncate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, dict_t *xdata);
+
+int32_t
+default_access_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      int32_t mask, dict_t *xdata);
+
+int32_t
+default_readlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        size_t size, dict_t *xdata);
+
+int32_t
+default_mknod_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t
+default_mkdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     mode_t mode, mode_t umask, dict_t *xdata);
+
+int32_t
+default_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      int xflag, dict_t *xdata);
+
+int32_t
+default_rmdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                     dict_t *xdata);
+
+int32_t
+default_symlink_resume(call_frame_t *frame, xlator_t *this,
+                       const char *linkpath, loc_t *loc, mode_t umask,
+                       dict_t *xdata);
+
+int32_t
+default_rename_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                      loc_t *newloc, dict_t *xdata);
+
+int32_t
+default_link_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                    loc_t *newloc, dict_t *xdata);
+
+int32_t
+default_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                      dict_t *xdata);
+
+int32_t
+default_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t flags, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_readv_resume(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                     off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+default_writev_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iovec *vector, int32_t count, off_t offset,
+                      uint32_t flags, struct iobref *iobref, dict_t *xdata);
+
+int32_t
+default_flush_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     dict_t *xdata);
+
+int32_t
+default_fsync_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     int32_t datasync, dict_t *xdata);
+
+int32_t
+default_opendir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       fd_t *fd, dict_t *xdata);
+
+int32_t
+default_fsyncdir_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int32_t datasync, dict_t *xdata);
+
+int32_t
+default_statfs_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata);
+
+int32_t
+default_setxattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+default_getxattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        const char *name, dict_t *xdata);
+
+int32_t
+default_fsetxattr_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+default_fgetxattr_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         const char *name, dict_t *xdata);
+
+int32_t
+default_removexattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           const char *name, dict_t *xdata);
+
+int32_t
+default_fremovexattr_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            const char *name, dict_t *xdata);
+
+int32_t
+default_lk_resume(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+                  struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+default_inodelk_resume(call_frame_t *frame, xlator_t *this, const char *volume,
+                       loc_t *loc, int32_t cmd, struct gf_flock *flock,
+                       dict_t *xdata);
+
+int32_t
+default_finodelk_resume(call_frame_t *frame, xlator_t *this, const char *volume,
+                        fd_t *fd, int32_t cmd, struct gf_flock *flock,
+                        dict_t *xdata);
+
+int32_t
+default_entrylk_resume(call_frame_t *frame, xlator_t *this, const char *volume,
+                       loc_t *loc, const char *basename, entrylk_cmd cmd,
+                       entrylk_type type, dict_t *xdata);
+
+int32_t
+default_fentrylk_resume(call_frame_t *frame, xlator_t *this, const char *volume,
+                        fd_t *fd, const char *basename, entrylk_cmd cmd,
+                        entrylk_type type, dict_t *xdata);
+
+int32_t
+default_readdir_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       size_t size, off_t off, dict_t *xdata);
+
+int32_t
+default_readdirp_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata);
+
+int32_t
+default_xattrop_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+default_fxattrop_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+int32_t
+default_rchecksum_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, int32_t len, dict_t *xdata);
+
+int32_t
+default_setattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+default_fsetattr_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+default_fallocate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         int32_t keep_size, off_t offset, size_t len,
+                         dict_t *xdata);
+
+int32_t
+default_discard_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, size_t len, dict_t *xdata);
+
+int32_t
+default_zerofill_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        off_t offset, off_t len, dict_t *xdata);
+
+int32_t
+default_ipc_resume(call_frame_t *frame, xlator_t *this, int32_t op,
+                   dict_t *xdata);
+
+int32_t
+default_seek_resume(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    gf_seek_what_t what, dict_t *xdata);
+
+int32_t
+default_lease_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+default_getactivelk_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           dict_t *xdata);
+
+int32_t
+default_setactivelk_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           lock_migration_info_t *locklist, dict_t *xdata);
+
+int32_t
+default_put_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   mode_t umask, uint32_t flags, struct iovec *vector,
+                   int32_t count, off_t off, struct iobref *iobref,
+                   dict_t *xattr, dict_t *xdata);
+
+int32_t
+default_copy_file_range_resume(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                               off_t off64_in, fd_t *fd_out, off64_t off_out,
+                               size_t len, uint32_t flags, dict_t *xdata);
+
+/* _cbk_resume */
+
+int32_t
+default_lookup_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata,
+                          struct iatt *postparent);
+
+int32_t
+default_stat_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata);
+
+int32_t
+default_truncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata);
+
+int32_t
+default_ftruncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *prebuf, struct iatt *postbuf,
+                             dict_t *xdata);
+
+int32_t
+default_access_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_readlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, const char *path,
+                            struct iatt *buf, dict_t *xdata);
+
+int32_t
+default_mknod_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_mkdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_unlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata);
+
+int32_t
+default_rmdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata);
+
+int32_t
+default_symlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_rename_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
+                          dict_t *xdata);
+
+int32_t
+default_link_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_create_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, fd_t *fd,
+                          inode_t *inode, struct iatt *buf,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata);
+
+int32_t
+default_open_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata);
+
+int32_t
+default_readv_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                         int32_t count, struct iatt *stbuf,
+                         struct iobref *iobref, dict_t *xdata);
+
+int32_t
+default_writev_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_flush_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fsync_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_fstat_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         dict_t *xdata);
+
+int32_t
+default_opendir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           dict_t *xdata);
+
+int32_t
+default_fsyncdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_statfs_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                          dict_t *xdata);
+
+int32_t
+default_setxattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fsetxattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fgetxattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *dict,
+                             dict_t *xdata);
+
+int32_t
+default_getxattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata);
+
+int32_t
+default_xattrop_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata);
+
+int32_t
+default_fxattrop_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata);
+
+int32_t
+default_removexattr_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata);
+
+int32_t
+default_fremovexattr_cbk_resume(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_lk_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata);
+
+int32_t
+default_inodelk_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_finodelk_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_entrylk_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fentrylk_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_rchecksum_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             uint32_t weak_checksum, uint8_t *strong_checksum,
+                             dict_t *xdata);
+
+int32_t
+default_readdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           gf_dirent_t *entries, dict_t *xdata);
+
+int32_t
+default_readdirp_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            gf_dirent_t *entries, dict_t *xdata);
+
+int32_t
+default_setattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *statpre, struct iatt *statpost,
+                           dict_t *xdata);
+
+int32_t
+default_fsetattr_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *statpre, struct iatt *statpost,
+                            dict_t *xdata);
+
+int32_t
+default_fallocate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                             struct iatt *post, dict_t *xdata);
+
+int32_t
+default_discard_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                           struct iatt *post, dict_t *xdata);
+
+int32_t
+default_zerofill_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                            struct iatt *post, dict_t *xdata);
+int32_t
+default_ipc_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_seek_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, off_t offset,
+                        dict_t *xdata);
+
+int32_t
+default_getspec_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, char *spec_data);
+
+int32_t
+default_lease_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+default_getactivelk_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               lock_migration_info_t *locklist, dict_t *xdata);
+
+int32_t
+default_setactivelk_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata);
+
+int32_t
+default_put_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_icreate_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, dict_t *xdata);
+
+int32_t
+default_namelink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *xdata);
+
+int32_t
+default_copy_file_range_cbk_resume(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct iatt *stbuf,
+                                   struct iatt *prebuf_dst,
+                                   struct iatt *postbuf_dst, dict_t *xdata);
+
+/* _CBK */
+int32_t
+default_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent);
+
+int32_t
+default_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata);
+
+int32_t
+default_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata);
+
+int32_t
+default_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata);
+
+int32_t
+default_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata);
+
+int32_t
+default_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata);
+
+int32_t
+default_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+int32_t
+default_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata);
+
+int32_t
+default_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata);
+
+int32_t
+default_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata);
+
+int32_t
+default_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata);
+
+int32_t
+default_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata);
+
+int32_t
+default_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata);
+
+int32_t
+default_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                      uint8_t *strong_checksum, dict_t *xdata);
+
+int32_t
+default_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata);
+
+int32_t
+default_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata);
+
+int32_t
+default_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata);
+
+int32_t
+default_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata);
+
+int32_t
+default_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                      struct iatt *post, dict_t *xdata);
+
+int32_t
+default_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata);
+
+int32_t
+default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                     struct iatt *post, dict_t *xdata);
+
+int32_t
+default_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata);
+
+int32_t
+default_getspec_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, char *spec_data);
+
+int32_t
+default_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                  dict_t *xdata);
+
+int32_t
+default_getactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        lock_migration_info_t *locklist, dict_t *xdata);
+
+int32_t
+default_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int32_t
+default_put_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata);
+
+int32_t
+default_icreate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata);
+
+int32_t
+default_namelink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+
+int32_t
+default_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *stbuf, struct iatt *prebuf_dst,
+                            struct iatt *postbuf_dst, dict_t *xdata);
+
+int32_t
+default_lookup_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_stat_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_truncate_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_ftruncate_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_access_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_readlink_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_mknod_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_mkdir_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_unlink_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_rmdir_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_symlink_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_rename_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_link_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_create_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_open_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_readv_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_writev_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_flush_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fsync_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fstat_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_opendir_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fsyncdir_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_statfs_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_setxattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fsetxattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fgetxattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_getxattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_xattrop_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fxattrop_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_removexattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fremovexattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_lk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_inodelk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_finodelk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_entrylk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fentrylk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_rchecksum_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_readdir_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_readdirp_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_setattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fsetattr_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_fallocate_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_discard_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_zerofill_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_getspec_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_ipc_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_seek_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_lease_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_getactivelk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_setactivelk_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_put_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_icreate_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_namelink_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_copy_file_range_failure_cbk(call_frame_t *frame, int32_t op_errno);
+
+int32_t
+default_mem_acct_init(xlator_t *this);
+
+void
+default_fini(xlator_t *this);
+
+#endif /* _DEFAULTS_H */
diff --git a/libglusterfs/src/glusterfs/dict.h b/libglusterfs/src/glusterfs/dict.h
new file mode 100644
index 00000000000..d0467c6dfb6
--- /dev/null
+++ b/libglusterfs/src/glusterfs/dict.h
@@ -0,0 +1,420 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DICT_H
+#define _DICT_H
+
+#include <inttypes.h>
+#include <sys/uio.h>
+#include <pthread.h>
+
+#include "glusterfs/common-utils.h"
+
+typedef struct _data data_t;
+typedef struct _dict dict_t;
+typedef struct _data_pair data_pair_t;
+
+#define dict_set_sizen(this, key, value) dict_setn(this, key, SLEN(key), value)
+
+#define dict_add_sizen(this, key, value) dict_addn(this, key, SLEN(key), value)
+
+#define dict_get_sizen(this, key) dict_getn(this, key, SLEN(key))
+
+#define dict_del_sizen(this, key) dict_deln(this, key, SLEN(key))
+
+#define dict_set_str_sizen(this, key, str)                                     \
+    dict_set_strn(this, key, SLEN(key), str)
+
+#define dict_set_sizen_str_sizen(this, key, str)                               \
+    dict_set_nstrn(this, key, SLEN(key), str, SLEN(str))
+
+#define dict_set_dynstr_sizen(this, key, str)                                  \
+    dict_set_dynstrn(this, key, SLEN(key), str)
+
+#define dict_get_str_sizen(this, key, str)                                     \
+    dict_get_strn(this, key, SLEN(key), str)
+
+#define dict_get_int32_sizen(this, key, val)                                   \
+    dict_get_int32n(this, key, SLEN(key), val)
+
+#define dict_set_int32_sizen(this, key, val)                                   \
+    dict_set_int32n(this, key, SLEN(key), val)
+
+#define GF_PROTOCOL_DICT_SERIALIZE(this, from_dict, to, len, ope, labl)        \
+    do {                                                                       \
+        int _ret = 0;                                                          \
+                                                                               \
+        if (!from_dict)                                                        \
+            break;                                                             \
+                                                                               \
+        _ret = dict_allocate_and_serialize(from_dict, to, &len);               \
+        if (_ret < 0) {                                                        \
+            gf_msg(this->name, GF_LOG_WARNING, 0, LG_MSG_DICT_SERIAL_FAILED,   \
+                   "failed to get serialized dict (%s)", (#from_dict));        \
+            ope = EINVAL;                                                      \
+            goto labl;                                                         \
+        }                                                                      \
+    } while (0)
+
+#define GF_PROTOCOL_DICT_UNSERIALIZE(xl, to, buff, len, ret, ope, labl)        \
+    do {                                                                       \
+        if (!len)                                                              \
+            break;                                                             \
+        to = dict_new();                                                       \
+        GF_VALIDATE_OR_GOTO(xl->name, to, labl);                               \
+                                                                               \
+        ret = dict_unserialize(buff, len, &to);                                \
+        if (ret < 0) {                                                         \
+            gf_msg(xl->name, GF_LOG_WARNING, 0, LG_MSG_DICT_UNSERIAL_FAILED,   \
+                   "failed to unserialize dictionary (%s)", (#to));            \
+                                                                               \
+            ope = EINVAL;                                                      \
+            goto labl;                                                         \
+        }                                                                      \
+                                                                               \
+    } while (0)
+
+#define dict_foreach_inline(d, c) for (c = d->members_list; c; c = c->next)
+
+#define DICT_KEY_VALUE_MAX_SIZE 1048576
+#define DICT_MAX_FLAGS 256
+#define DICT_FLAG_SET 1
+#define DICT_FLAG_CLEAR 0
+#define DICT_HDR_LEN 4
+#define DICT_DATA_HDR_KEY_LEN 4
+#define DICT_DATA_HDR_VAL_LEN 4
+
+struct _data {
+    char *data;
+    gf_atomic_t refcount;
+    gf_dict_data_type_t data_type;
+    uint32_t len;
+    gf_boolean_t is_static;
+};
+
+struct _data_pair {
+    struct _data_pair *hash_next;
+    struct _data_pair *prev;
+    struct _data_pair *next;
+    data_t *value;
+    char *key;
+    uint32_t key_hash;
+};
+
+struct _dict {
+    uint64_t max_count;
+    int32_t hash_size;
+    int32_t count;
+    gf_atomic_t refcount;
+    data_pair_t **members;
+    data_pair_t *members_list;
+    char *extra_stdfree;
+    gf_lock_t lock;
+    data_pair_t *members_internal;
+    data_pair_t free_pair;
+    /* Variable to store total keylen + value->len */
+    uint32_t totkvlen;
+};
+
+typedef gf_boolean_t (*dict_match_t)(dict_t *d, char *k, data_t *v, void *data);
+
+int32_t
+is_data_equal(data_t *one, data_t *two);
+void
+data_destroy(data_t *data);
+
+/* function to set a key/value pair (overwrite existing if matches the key */
+int32_t
+dict_set(dict_t *this, char *key, data_t *value);
+int32_t
+dict_setn(dict_t *this, char *key, const int keylen, data_t *value);
+
+/* function to set a new key/value pair (without checking for duplicate) */
+int32_t
+dict_add(dict_t *this, char *key, data_t *value);
+int32_t
+dict_addn(dict_t *this, char *key, const int keylen, data_t *value);
+int
+dict_get_with_ref(dict_t *this, char *key, data_t **data);
+data_t *
+dict_get(dict_t *this, char *key);
+data_t *
+dict_getn(dict_t *this, char *key, const int keylen);
+void
+dict_del(dict_t *this, char *key);
+void
+dict_deln(dict_t *this, char *key, const int keylen);
+int
+dict_reset(dict_t *dict);
+
+int
+dict_key_count(dict_t *this);
+
+int32_t
+dict_serialized_length(dict_t *dict);
+int32_t
+dict_serialize(dict_t *dict, char *buf);
+int32_t
+dict_unserialize(char *buf, int32_t size, dict_t **fill);
+
+int32_t
+dict_allocate_and_serialize(dict_t *this, char **buf, u_int *length);
+
+void
+dict_unref(dict_t *dict);
+dict_t *
+dict_ref(dict_t *dict);
+data_t *
+data_ref(data_t *data);
+void
+data_unref(data_t *data);
+
+int32_t
+dict_lookup(dict_t *this, char *key, data_t **data);
+/*
+   TODO: provide converts for different byte sizes, signedness, and void *
+ */
+data_t *
+int_to_data(int64_t value);
+data_t *
+str_to_data(char *value);
+data_t *
+strn_to_data(char *value, const int vallen);
+data_t *
+data_from_dynptr(void *value, int32_t len);
+data_t *
+bin_to_data(void *value, int32_t len);
+data_t *
+static_str_to_data(char *value);
+data_t *
+static_bin_to_data(void *value);
+
+int64_t
+data_to_int64(data_t *data);
+int32_t
+data_to_int32(data_t *data);
+int16_t
+data_to_int16(data_t *data);
+int8_t
+data_to_int8(data_t *data);
+
+uint64_t
+data_to_uint64(data_t *data);
+uint32_t
+data_to_uint32(data_t *data);
+uint16_t
+data_to_uint16(data_t *data);
+uint8_t
+data_to_uint8(data_t *data);
+
+data_t *
+data_from_int64(int64_t value);
+data_t *
+data_from_int32(int32_t value);
+data_t *
+data_from_int16(int16_t value);
+data_t *
+data_from_int8(int8_t value);
+
+data_t *
+data_from_uint64(uint64_t value);
+data_t *
+data_from_uint32(uint32_t value);
+data_t *
+data_from_uint16(uint16_t value);
+
+char *
+data_to_str(data_t *data);
+void *
+data_to_bin(data_t *data);
+void *
+data_to_ptr(data_t *data);
+data_t *
+data_copy(data_t *old);
+struct iatt *
+data_to_iatt(data_t *data, char *key);
+
+int
+dict_foreach(dict_t *this,
+             int (*fn)(dict_t *this, char *key, data_t *value, void *data),
+             void *data);
+
+int
+dict_foreach_fnmatch(dict_t *dict, char *pattern,
+                     int (*fn)(dict_t *this, char *key, data_t *value,
+                               void *data),
+                     void *data);
+
+int
+dict_foreach_match(dict_t *dict,
+                   gf_boolean_t (*match)(dict_t *this, char *key, data_t *value,
+                                         void *mdata),
+                   void *match_data,
+                   int (*action)(dict_t *this, char *key, data_t *value,
+                                 void *adata),
+                   void *action_data);
+
+int
+dict_null_foreach_fn(dict_t *d, char *k, data_t *v, void *tmp);
+int
+dict_remove_foreach_fn(dict_t *d, char *k, data_t *v, void *tmp);
+dict_t *
+dict_copy(dict_t *this, dict_t *new);
+int
+dict_keys_join(void *value, int size, dict_t *dict,
+               int (*filter_fn)(char *key));
+
+/* CLEANED UP FUNCTIONS DECLARATIONS */
+GF_MUST_CHECK dict_t *
+dict_new(void);
+dict_t *
+dict_copy_with_ref(dict_t *this, dict_t *new);
+
+GF_MUST_CHECK int
+dict_reset(dict_t *dict);
+
+GF_MUST_CHECK int
+dict_get_int8(dict_t *this, char *key, int8_t *val);
+GF_MUST_CHECK int
+dict_set_int8(dict_t *this, char *key, int8_t val);
+
+GF_MUST_CHECK int
+dict_get_int16(dict_t *this, char *key, int16_t *val);
+GF_MUST_CHECK int
+dict_set_int16(dict_t *this, char *key, int16_t val);
+
+GF_MUST_CHECK int
+dict_get_int32(dict_t *this, char *key, int32_t *val);
+GF_MUST_CHECK int
+dict_get_int32n(dict_t *this, char *key, const int keylen, int32_t *val);
+GF_MUST_CHECK int
+dict_set_int32(dict_t *this, char *key, int32_t val);
+GF_MUST_CHECK int
+dict_set_int32n(dict_t *this, char *key, const int keylen, int32_t val);
+
+GF_MUST_CHECK int
+dict_get_int64(dict_t *this, char *key, int64_t *val);
+GF_MUST_CHECK int
+dict_set_int64(dict_t *this, char *key, int64_t val);
+
+GF_MUST_CHECK int
+dict_get_uint16(dict_t *this, char *key, uint16_t *val);
+GF_MUST_CHECK int
+dict_set_uint16(dict_t *this, char *key, uint16_t val);
+
+GF_MUST_CHECK int
+dict_get_uint32(dict_t *this, char *key, uint32_t *val);
+GF_MUST_CHECK int
+dict_set_uint32(dict_t *this, char *key, uint32_t val);
+
+GF_MUST_CHECK int
+dict_get_uint64(dict_t *this, char *key, uint64_t *val);
+GF_MUST_CHECK int
+dict_set_uint64(dict_t *this, char *key, uint64_t val);
+
+GF_MUST_CHECK int
+dict_check_flag(dict_t *this, char *key, int flag);
+GF_MUST_CHECK int
+dict_set_flag(dict_t *this, char *key, int flag);
+GF_MUST_CHECK int
+dict_clear_flag(dict_t *this, char *key, int flag);
+
+GF_MUST_CHECK int
+dict_get_double(dict_t *this, char *key, double *val);
+GF_MUST_CHECK int
+dict_set_double(dict_t *this, char *key, double val);
+
+GF_MUST_CHECK int
+dict_set_static_ptr(dict_t *this, char *key, void *ptr);
+GF_MUST_CHECK int
+dict_get_ptr(dict_t *this, char *key, void **ptr);
+GF_MUST_CHECK int
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len);
+GF_MUST_CHECK int
+dict_set_dynptr(dict_t *this, char *key, void *ptr, size_t size);
+
+GF_MUST_CHECK int
+dict_get_bin(dict_t *this, char *key, void **ptr);
+GF_MUST_CHECK int
+dict_set_bin(dict_t *this, char *key, void *ptr, size_t size);
+GF_MUST_CHECK int
+dict_set_static_bin(dict_t *this, char *key, void *ptr, size_t size);
+
+GF_MUST_CHECK int
+dict_set_option(dict_t *this, char *key, char *str);
+GF_MUST_CHECK int
+dict_set_str(dict_t *this, char *key, char *str);
+GF_MUST_CHECK int
+dict_set_strn(dict_t *this, char *key, const int keylen, char *str);
+GF_MUST_CHECK int
+dict_set_nstrn(dict_t *this, char *key, const int keylen, char *str,
+               const int vallen);
+GF_MUST_CHECK int
+dict_set_dynstr(dict_t *this, char *key, char *str);
+GF_MUST_CHECK int
+dict_set_dynstrn(dict_t *this, char *key, const int keylen, char *str);
+GF_MUST_CHECK int
+dict_set_dynstr_with_alloc(dict_t *this, char *key, const char *str);
+GF_MUST_CHECK int
+dict_add_dynstr_with_alloc(dict_t *this, char *key, char *str);
+GF_MUST_CHECK int
+dict_get_str(dict_t *this, char *key, char **str);
+GF_MUST_CHECK int
+dict_get_strn(dict_t *this, char *key, const int keylen, char **str);
+
+GF_MUST_CHECK int
+dict_get_str_boolean(dict_t *this, char *key, int default_val);
+GF_MUST_CHECK int
+dict_rename_key(dict_t *this, char *key, char *replace_key);
+GF_MUST_CHECK int
+dict_serialize_value_with_delim(dict_t *this, char *buf, int32_t *serz_len,
+                                char delimiter);
+
+GF_MUST_CHECK int
+dict_set_gfuuid(dict_t *this, char *key, uuid_t uuid, bool is_static);
+GF_MUST_CHECK int
+dict_get_gfuuid(dict_t *this, char *key, uuid_t *uuid);
+
+GF_MUST_CHECK int
+dict_set_iatt(dict_t *this, char *key, struct iatt *iatt, bool is_static);
+GF_MUST_CHECK int
+dict_get_iatt(dict_t *this, char *key, struct iatt *iatt);
+GF_MUST_CHECK int
+dict_set_mdata(dict_t *this, char *key, struct mdata_iatt *mdata,
+               bool is_static);
+GF_MUST_CHECK int
+dict_get_mdata(dict_t *this, char *key, struct mdata_iatt *mdata);
+
+void
+dict_dump_to_statedump(dict_t *dict, char *dict_name, char *domain);
+
+void
+dict_dump_to_log(dict_t *dict);
+
+int
+dict_dump_to_str(dict_t *dict, char *dump, int dumpsize, char *format);
+gf_boolean_t
+dict_match_everything(dict_t *d, char *k, data_t *v, void *data);
+
+dict_t *
+dict_for_key_value(const char *name, const char *value, size_t size,
+                   gf_boolean_t is_static);
+
+gf_boolean_t
+are_dicts_equal(dict_t *one, dict_t *two,
+                gf_boolean_t (*match)(dict_t *d, char *k, data_t *v,
+                                      void *data),
+                gf_boolean_t (*value_ignore)(char *k));
+int
+dict_has_key_from_array(dict_t *dict, char **strings, gf_boolean_t *result);
+
+int
+dict_serialized_length_lk(dict_t *this);
+#endif
diff --git a/libglusterfs/src/glusterfs/event-history.h b/libglusterfs/src/glusterfs/event-history.h
new file mode 100644
index 00000000000..f0e0422418e
--- /dev/null
+++ b/libglusterfs/src/glusterfs/event-history.h
@@ -0,0 +1,40 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _EH_H
+#define _EH_H
+
+#include <pthread.h>              // for pthread_mutex_t
+#include <stddef.h>               // for size_t
+#include "glusterfs/circ-buff.h"  // for buffer_t, circular_buffer_t
+#include "glusterfs/glusterfs.h"  // for gf_boolean_t
+
+struct event_hist {
+    buffer_t *buffer;
+    pthread_mutex_t lock;
+};
+
+typedef struct event_hist eh_t;
+
+void
+eh_dump(eh_t *event, void *data,
+        int(fn)(circular_buffer_t *buffer, void *data));
+
+eh_t *
+eh_new(size_t buffer_size, gf_boolean_t use_buffer_once,
+       void (*destroy_data)(void *data));
+
+int
+eh_save_history(eh_t *history, void *string);
+
+int
+eh_destroy(eh_t *history);
+
+#endif /* _EH_H */
diff --git a/libglusterfs/src/glusterfs/events.h b/libglusterfs/src/glusterfs/events.h
new file mode 100644
index 00000000000..74c5326427b
--- /dev/null
+++ b/libglusterfs/src/glusterfs/events.h
@@ -0,0 +1,34 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EVENTS_H__
+#define __EVENTS_H__
+
+#include "eventtypes.h"
+
+#ifdef USE_EVENTS
+int
+_gf_event(eventtypes_t event, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+#else
+__attribute__((__format__(__printf__, 2, 3))) static inline int
+_gf_event(eventtypes_t event, const char *fmt, ...)
+{
+    return 0;
+}
+#endif /* USE_EVENTS */
+
+#define gf_event(event, fmt...)                                                \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+        _gf_event(event, ##fmt);                                               \
+    } while (0)
+
+#endif /* __EVENTS_H__ */
diff --git a/libglusterfs/src/glusterfs/fd-lk.h b/libglusterfs/src/glusterfs/fd-lk.h
new file mode 100644
index 00000000000..76cc680306a
--- /dev/null
+++ b/libglusterfs/src/glusterfs/fd-lk.h
@@ -0,0 +1,59 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _FD_LK_H
+#define _FD_LK_H
+
+#include "glusterfs/fd.h"
+#include "glusterfs/locking.h"
+#include "glusterfs/list.h"
+#include "glusterfs/glusterfs.h"
+
+#define get_lk_type(type)                                                      \
+    type == F_UNLCK ? "F_UNLCK" : (type == F_RDLCK ? "F_RDLCK" : "F_WRLCK")
+
+#define get_lk_cmd(cmd)                                                        \
+    cmd == F_SETLKW ? "F_SETLKW" : (cmd == F_SETLK ? "F_SETLK" : "F_GETLK")
+
+struct _fd;
+
+struct fd_lk_ctx {
+    struct list_head lk_list;
+    gf_atomic_t ref;
+    gf_lock_t lock;
+};
+typedef struct fd_lk_ctx fd_lk_ctx_t;
+
+struct fd_lk_ctx_node {
+    int32_t cmd;
+    struct gf_flock user_flock;
+    off_t fl_start;
+    off_t fl_end;
+    short fl_type;
+    struct list_head next;
+};
+typedef struct fd_lk_ctx_node fd_lk_ctx_node_t;
+
+fd_lk_ctx_t *
+fd_lk_ctx_ref(fd_lk_ctx_t *lk_ctx);
+
+fd_lk_ctx_t *
+fd_lk_ctx_create(void);
+
+int
+fd_lk_insert_and_merge(struct _fd *lk_ctx, int32_t cmd, struct gf_flock *flock);
+
+int
+fd_lk_ctx_unref(fd_lk_ctx_t *lk_ctx);
+
+gf_boolean_t
+fd_lk_ctx_empty(fd_lk_ctx_t *lk_ctx);
+
+#endif /* _FD_LK_H */
diff --git a/libglusterfs/src/glusterfs/fd.h b/libglusterfs/src/glusterfs/fd.h
new file mode 100644
index 00000000000..3ffaaa60504
--- /dev/null
+++ b/libglusterfs/src/glusterfs/fd.h
@@ -0,0 +1,169 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _FD_H
+#define _FD_H
+
+#include "glusterfs/list.h"
+#include <sys/types.h>
+#include <unistd.h>
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/locking.h"
+#include "glusterfs/fd-lk.h"
+
+#define GF_ANON_FD_NO -2
+#define GF_ANON_FD_FLAGS (O_RDWR | O_LARGEFILE)
+
+struct _inode;
+struct _dict;
+struct fd_lk_ctx;
+
+struct _fd_ctx {
+    union {
+        uint64_t key;
+        void *xl_key;
+    };
+    union {
+        uint64_t value1;
+        void *ptr1;
+    };
+};
+
+struct _fd {
+    uint64_t pid;
+    int32_t flags;
+    gf_atomic_t refcount;
+    struct list_head inode_list;
+    struct _inode *inode;
+    gf_lock_t lock; /* used ONLY for manipulating
+                       'struct _fd_ctx' array (_ctx).*/
+    struct _fd_ctx *_ctx;
+    int xl_count; /* Number of xl referred in this fd */
+    struct fd_lk_ctx *lk_ctx;
+    gf_boolean_t anonymous; /* fd which does not have counterpart open
+                               fd on backend (server for client, posix
+                               for server). */
+};
+typedef struct _fd fd_t;
+
+struct fd_table_entry {
+    fd_t *fd;
+    int next_free;
+};
+typedef struct fd_table_entry fdentry_t;
+
+struct _fdtable {
+    int refcount;
+    uint32_t max_fds;
+    pthread_rwlock_t lock;
+    fdentry_t *fdentries;
+    int first_free;
+};
+typedef struct _fdtable fdtable_t;
+
+/* Signifies no more entries in the fd table. */
+#define GF_FDTABLE_END -1
+
+/* This is used to invalidated
+ * the next_free value in an fdentry that has been allocated
+ */
+#define GF_FDENTRY_ALLOCATED -2
+
+#include "glusterfs/logging.h"
+#include "glusterfs/xlator.h"
+
+void
+gf_fd_put(fdtable_t *fdtable, int32_t fd);
+
+fd_t *
+gf_fd_fdptr_get(fdtable_t *fdtable, int64_t fd);
+
+fdtable_t *
+gf_fd_fdtable_alloc(void);
+
+int
+gf_fd_unused_get(fdtable_t *fdtable, fd_t *fdptr);
+
+fdentry_t *
+gf_fd_fdtable_get_all_fds(fdtable_t *fdtable, uint32_t *count);
+
+void
+gf_fd_fdtable_destroy(fdtable_t *fdtable);
+
+fd_t *
+__fd_ref(fd_t *fd);
+
+fd_t *
+fd_ref(fd_t *fd);
+
+void
+fd_unref(fd_t *fd);
+
+void
+fd_close(fd_t *fd);
+
+fd_t *
+fd_create(struct _inode *inode, pid_t pid);
+
+fd_t *
+fd_create_uint64(struct _inode *inode, uint64_t pid);
+
+fd_t *
+fd_lookup(struct _inode *inode, pid_t pid);
+
+fd_t *
+fd_lookup_uint64(struct _inode *inode, uint64_t pid);
+
+fd_t *
+fd_lookup_anonymous(inode_t *inode, int32_t flags);
+
+fd_t *
+fd_anonymous(inode_t *inode);
+
+fd_t *
+fd_anonymous_with_flags(inode_t *inode, int32_t flags);
+
+gf_boolean_t
+fd_is_anonymous(fd_t *fd);
+
+uint8_t
+fd_list_empty(struct _inode *inode);
+
+fd_t *
+fd_bind(fd_t *fd);
+
+int
+fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value);
+
+int
+fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value);
+
+int
+fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value);
+
+int
+__fd_ctx_del(fd_t *fd, xlator_t *xlator, uint64_t *value);
+
+int
+__fd_ctx_set(fd_t *fd, xlator_t *xlator, uint64_t value);
+
+int
+__fd_ctx_get(fd_t *fd, xlator_t *xlator, uint64_t *value);
+
+void
+fd_ctx_dump(fd_t *fd, char *prefix);
+
+fdentry_t *
+gf_fd_fdtable_copy_all_fds(fdtable_t *fdtable, uint32_t *count);
+
+void
+gf_fdptr_put(fdtable_t *fdtable, fd_t *fd);
+
+#endif /* _FD_H */
diff --git a/libglusterfs/src/glusterfs/gf-dirent.h b/libglusterfs/src/glusterfs/gf-dirent.h
new file mode 100644
index 00000000000..e358da30f58
--- /dev/null
+++ b/libglusterfs/src/glusterfs/gf-dirent.h
@@ -0,0 +1,71 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_DIRENT_H
+#define _GF_DIRENT_H
+
+#include "glusterfs/iatt.h"
+#include "glusterfs/inode.h"
+
+#define gf_dirent_size(name) (sizeof(gf_dirent_t) + strlen(name) + 1)
+
+int
+gf_deitransform(xlator_t *this, uint64_t y);
+
+int
+gf_itransform(xlator_t *this, uint64_t x, uint64_t *y_p, int client_id);
+
+uint64_t
+gf_dirent_orig_offset(xlator_t *this, uint64_t offset);
+
+struct _dir_entry {
+    struct _dir_entry *next;
+    char *name;
+    char *link;
+    struct iatt buf;
+};
+
+struct _gf_dirent {
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_dirent *next;
+            struct _gf_dirent *prev;
+        };
+    };
+    uint64_t d_ino;
+    uint64_t d_off;
+    uint32_t d_len;
+    uint32_t d_type;
+    struct iatt d_stat;
+    dict_t *dict;
+    inode_t *inode;
+    char d_name[];
+};
+
+#define DT_ISDIR(mode) (mode == DT_DIR)
+
+gf_dirent_t *
+gf_dirent_for_name(const char *name);
+gf_dirent_t *
+entry_copy(gf_dirent_t *source);
+void
+gf_dirent_entry_free(gf_dirent_t *entry);
+void
+gf_dirent_free(gf_dirent_t *entries);
+int
+gf_link_inodes_from_dirent(xlator_t *this, inode_t *parent,
+                           gf_dirent_t *entries);
+int
+gf_fill_iatt_for_dirent(gf_dirent_t *entry, inode_t *parent, xlator_t *subvol);
+
+void
+gf_link_inode_from_dirent(xlator_t *this, inode_t *parent, gf_dirent_t *entry);
+#endif /* _GF_DIRENT_H */
diff --git a/libglusterfs/src/glusterfs/gf-event.h b/libglusterfs/src/glusterfs/gf-event.h
new file mode 100644
index 00000000000..40f8fbdf10a
--- /dev/null
+++ b/libglusterfs/src/glusterfs/gf-event.h
@@ -0,0 +1,140 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_EVENT_H_
+#define _GF_EVENT_H_
+
+#include <pthread.h>
+#include "common-utils.h"
+#include "list.h"
+
+struct event_pool;
+struct event_ops;
+struct event_slot_poll;
+struct event_slot_epoll;
+struct event_data {
+    int idx;
+    int gen;
+} __attribute__((__packed__, __may_alias__));
+
+typedef void (*event_handler_t)(int fd, int idx, int gen, void *data,
+                                int poll_in, int poll_out, int poll_err,
+                                char event_thread_exit);
+
+#define EVENT_EPOLL_TABLES 1024
+#define EVENT_EPOLL_SLOTS 1024
+#define EVENT_MAX_THREADS 1024
+
+/* See rpcsvc.h to check why. */
+GF_STATIC_ASSERT(EVENT_MAX_THREADS % __BITS_PER_LONG == 0);
+
+struct event_pool {
+    struct event_ops *ops;
+
+    int fd;
+    int breaker[2];
+
+    int count;
+    struct event_slot_poll *reg;
+    struct event_slot_epoll *ereg[EVENT_EPOLL_TABLES];
+    int slots_used[EVENT_EPOLL_TABLES];
+
+    struct list_head poller_death;
+    int poller_death_sliced; /* track whether the list of fds interested
+                              * poller_death is sliced. If yes, new thread death
+                              * notification has to wait till the list is added
+                              * back
+                              */
+    int poller_gen;
+    int used;
+    int changed;
+
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+
+    void *evcache;
+    int evcache_size;
+
+    /* NOTE: Currently used only when event processing is done using
+     * epoll. */
+    int eventthreadcount; /* number of event threads to execute. */
+    pthread_t pollers[EVENT_MAX_THREADS]; /* poller thread_id store, and live
+                                             status */
+    int destroy;
+    int activethreadcount;
+
+    /*
+     * Number of threads created by auto-scaling, *in addition to* the
+     * configured number of threads.  This is only applicable on the
+     * server, where we try to keep the number of threads around the number
+     * of bricks.  In that case, the configured number is just "extra"
+     * threads to handle requests in excess of one per brick (including
+     * requests on the GlusterD connection).  For clients or GlusterD, this
+     * number will always be zero, so the "extra" is all we have.
+     *
+     * TBD: consider auto-scaling for clients as well
+     */
+    int auto_thread_count;
+};
+
+struct event_destroy_data {
+    int readfd;
+    struct event_pool *pool;
+};
+
+struct event_ops {
+    struct event_pool *(*new)(int count, int eventthreadcount);
+
+    int (*event_register)(struct event_pool *event_pool, int fd,
+                          event_handler_t handler, void *data, int poll_in,
+                          int poll_out, char notify_poller_death);
+
+    int (*event_select_on)(struct event_pool *event_pool, int fd, int idx,
+                           int poll_in, int poll_out);
+
+    int (*event_unregister)(struct event_pool *event_pool, int fd, int idx);
+
+    int (*event_unregister_close)(struct event_pool *event_pool, int fd,
+                                  int idx);
+
+    int (*event_dispatch)(struct event_pool *event_pool);
+
+    int (*event_reconfigure_threads)(struct event_pool *event_pool,
+                                     int newcount);
+    int (*event_pool_destroy)(struct event_pool *event_pool);
+    int (*event_handled)(struct event_pool *event_pool, int fd, int idx,
+                         int gen);
+};
+
+struct event_pool *
+gf_event_pool_new(int count, int eventthreadcount);
+int
+gf_event_select_on(struct event_pool *event_pool, int fd, int idx, int poll_in,
+                   int poll_out);
+int
+gf_event_register(struct event_pool *event_pool, int fd,
+                  event_handler_t handler, void *data, int poll_in,
+                  int poll_out, char notify_poller_death);
+int
+gf_event_unregister(struct event_pool *event_pool, int fd, int idx);
+int
+gf_event_unregister_close(struct event_pool *event_pool, int fd, int idx);
+int
+gf_event_dispatch(struct event_pool *event_pool);
+int
+gf_event_reconfigure_threads(struct event_pool *event_pool, int value);
+int
+gf_event_pool_destroy(struct event_pool *event_pool);
+int
+gf_event_dispatch_destroy(struct event_pool *event_pool);
+int
+gf_event_handled(struct event_pool *event_pool, int fd, int idx, int gen);
+
+#endif /* _GF_EVENT_H_ */
diff --git a/libglusterfs/src/glusterfs/gidcache.h b/libglusterfs/src/glusterfs/gidcache.h
new file mode 100644
index 00000000000..ddaabd765b5
--- /dev/null
+++ b/libglusterfs/src/glusterfs/gidcache.h
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __GIDCACHE_H__
+#define __GIDCACHE_H__
+
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/locking.h"
+
+/*
+ * TBD: make the cache size tunable
+ *
+ * The current size represents a pretty trivial amount of memory, and should
+ * provide good hit rates even for quite busy systems.  If we ever want to
+ * support really large cache sizes, we'll need to do dynamic allocation
+ * instead of just defining an array within a private structure. It doesn't make
+ * a whole lot of sense to change the associativity, because it won't improve
+ * hit rates all that much and will increase the maintenance cost as we have
+ * to scan more entries with every lookup/update.
+ */
+
+#define AUX_GID_CACHE_ASSOC 4
+#define AUX_GID_CACHE_BUCKETS 256
+#define AUX_GID_CACHE_SIZE (AUX_GID_CACHE_ASSOC * AUX_GID_CACHE_BUCKETS)
+
+typedef struct {
+    uint64_t gl_id;
+    uint64_t gl_uid;
+    uint64_t gl_gid;
+    int gl_count;
+    gid_t *gl_list;
+    time_t gl_deadline;
+} gid_list_t;
+
+typedef struct {
+    gf_lock_t gc_lock;
+    uint32_t gc_max_age;
+    unsigned int gc_nbuckets;
+    gid_list_t gc_cache[AUX_GID_CACHE_SIZE];
+} gid_cache_t;
+
+int
+gid_cache_init(gid_cache_t *, uint32_t);
+int
+gid_cache_reconf(gid_cache_t *, uint32_t);
+const gid_list_t *
+gid_cache_lookup(gid_cache_t *, uint64_t, uint64_t, uint64_t);
+void
+gid_cache_release(gid_cache_t *, const gid_list_t *);
+int
+gid_cache_add(gid_cache_t *, gid_list_t *);
+
+#endif /* __GIDCACHE_H__ */
diff --git a/libglusterfs/src/glusterfs/glfs-message-id.h b/libglusterfs/src/glusterfs/glfs-message-id.h
new file mode 100644
index 00000000000..a1a16ca1efb
--- /dev/null
+++ b/libglusterfs/src/glusterfs/glfs-message-id.h
@@ -0,0 +1,102 @@
+/*
+  Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_MESSAGE_ID_H_
+#define _GLFS_MESSAGE_ID_H_
+
+/* Base of all message IDs, all message IDs would be
+ * greater than this */
+#define GLFS_MSGID_BASE 100000
+
+/* Segment size of allocated range. Any component needing more than this
+ * segment size should take multiple segments (at times non contiguous,
+ * if extensions are being made post the next segment already allocated) */
+#define GLFS_MSGID_SEGMENT 1000
+
+/* Macro to define a range of messages for a component. The first argument is
+ * the name of the component. The second argument is the number of segments
+ * to allocate. The defined values will be GLFS_MSGID_COMP_<name> and
+ * GLFS_MSGID_COMP_<name>_END. */
+#define GLFS_MSGID_COMP(_name, _blocks)                                        \
+    GLFS_MSGID_COMP_##_name,                                                   \
+        GLFS_MSGID_COMP_##_name##_END = (GLFS_MSGID_COMP_##_name +             \
+                                         (GLFS_MSGID_SEGMENT * (_blocks)) - 1)
+
+#define GLFS_MSGID(_name, _msgs...)                                            \
+    enum _msgid_table_##_name                                                  \
+    {                                                                          \
+        GLFS_##_name##_COMP_BASE = GLFS_MSGID_COMP_##_name, ##_msgs,           \
+        GLGS_##_name##_COMP_END                                                \
+    }
+
+/* Per module message segments allocated */
+/* NOTE: For any new module add to the end the modules */
+enum _msgid_comp {
+    GLFS_MSGID_RESERVED = GLFS_MSGID_BASE - 1,
+
+    GLFS_MSGID_COMP(GLUSTERFSD, 1),
+    GLFS_MSGID_COMP(LIBGLUSTERFS, 1),
+    GLFS_MSGID_COMP(RPC_LIB, 1),
+    GLFS_MSGID_COMP(RPC_TRANS_RDMA, 1),
+    GLFS_MSGID_COMP(API, 1),
+    GLFS_MSGID_COMP(CLI, 1),
+    /* glusterd has a lot of messages, taking 2 segments for the same */
+    GLFS_MSGID_COMP(GLUSTERD, 2),
+    GLFS_MSGID_COMP(AFR, 1),
+    GLFS_MSGID_COMP(DHT, 1),
+    /* there is no component called 'common', however reserving this segment
+     * for common actions/errors like dict_{get/set}, memory accounting*/
+    GLFS_MSGID_COMP(COMMON, 1),
+    GLFS_MSGID_COMP(UPCALL, 1),
+    GLFS_MSGID_COMP(NFS, 1),
+    GLFS_MSGID_COMP(POSIX, 1),
+    GLFS_MSGID_COMP(PC, 1),
+    GLFS_MSGID_COMP(PS, 1),
+    GLFS_MSGID_COMP(BITROT_STUB, 1),
+    GLFS_MSGID_COMP(CHANGELOG, 1),
+    GLFS_MSGID_COMP(BITROT_BITD, 1),
+    GLFS_MSGID_COMP(RPC_TRANS_SOCKET, 1),
+    GLFS_MSGID_COMP(QUOTA, 1),
+    GLFS_MSGID_COMP(CTR, 1),
+    GLFS_MSGID_COMP(EC, 1),
+    GLFS_MSGID_COMP(IO_CACHE, 1),
+    GLFS_MSGID_COMP(IO_THREADS, 1),
+    GLFS_MSGID_COMP(MD_CACHE, 1),
+    GLFS_MSGID_COMP(OPEN_BEHIND, 1),
+    GLFS_MSGID_COMP(QUICK_READ, 1),
+    GLFS_MSGID_COMP(READ_AHEAD, 1),
+    GLFS_MSGID_COMP(READDIR_AHEAD, 1),
+    GLFS_MSGID_COMP(SYMLINK_CACHE, 1),
+    GLFS_MSGID_COMP(WRITE_BEHIND, 1),
+    GLFS_MSGID_COMP(CHANGELOG_LIB, 1),
+    GLFS_MSGID_COMP(SHARD, 1),
+    GLFS_MSGID_COMP(JBR, 1),
+    GLFS_MSGID_COMP(PL, 1),
+    GLFS_MSGID_COMP(DC, 1),
+    GLFS_MSGID_COMP(LEASES, 1),
+    GLFS_MSGID_COMP(INDEX, 1),
+    GLFS_MSGID_COMP(POSIX_ACL, 1),
+    GLFS_MSGID_COMP(NLC, 1),
+    GLFS_MSGID_COMP(SL, 1),
+    GLFS_MSGID_COMP(HAM, 1),
+    GLFS_MSGID_COMP(SDFS, 1),
+    GLFS_MSGID_COMP(QUIESCE, 1),
+    GLFS_MSGID_COMP(TA, 1),
+    GLFS_MSGID_COMP(SNAPVIEW_CLIENT, 1),
+    GLFS_MSGID_COMP(TEMPLATE, 1),
+    GLFS_MSGID_COMP(UTIME, 1),
+    GLFS_MSGID_COMP(SNAPVIEW_SERVER, 1),
+    GLFS_MSGID_COMP(CVLT, 1),
+    /* --- new segments for messages goes above this line --- */
+
+    GLFS_MSGID_END
+};
+
+#endif /* !_GLFS_MESSAGE_ID_H_ */
diff --git a/libglusterfs/src/glusterfs/globals.h b/libglusterfs/src/glusterfs/globals.h
new file mode 100644
index 00000000000..b22eaae6c2f
--- /dev/null
+++ b/libglusterfs/src/glusterfs/globals.h
@@ -0,0 +1,188 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLOBALS_H
+#define _GLOBALS_H
+
+#define GF_DEFAULT_BASE_PORT 24007
+#define GF_DEFAULT_VOLFILE_TRANSPORT "tcp"
+
+#define GF_GLOBAL_XLATOR_NAME "global"
+#define GD_OP_VERSION_KEY "operating-version"
+#define GD_MIN_OP_VERSION_KEY "minimum-operating-version"
+#define GD_MAX_OP_VERSION_KEY "maximum-operating-version"
+
+#define GF_PROTECT_FROM_EXTERNAL_WRITES "trusted.glusterfs.protect.writes"
+#define GF_AVOID_OVERWRITE "glusterfs.avoid.overwrite"
+#define GF_CLEAN_WRITE_PROTECTION "glusterfs.clean.writexattr"
+
+/* Gluster versions - OP-VERSION mapping
+ *
+ * 3.3.x                - 1
+ * 3.4.x                - 2
+ * 3.5.0                - 3
+ * 3.5.1                - 30501
+ * 3.6.0                - 30600
+ * 3.7.0                - 30700
+ * 3.7.1                - 30701
+ * 3.7.2                - 30702
+ *
+ * Starting with Gluster v3.6, the op-version will be multi-digit integer values
+ * based on the Glusterfs version, instead of a simply incrementing integer
+ * value. The op-version for a given X.Y.Z release will be an integer XYZ, with
+ * Y and Z 2 digit always 2 digits wide and padded with 0 when needed. This
+ * should allow for some gaps between two Y releases for backports of features
+ * in Z releases.
+ */
+#define GD_OP_VERSION_MIN                                                      \
+    1 /* MIN is the fresh start op-version, mostly                             \
+         should not change */
+#define GD_OP_VERSION_MAX                                                      \
+    GD_OP_VERSION_9_0 /* MAX VERSION is the maximum                            \
+                         count in VME table, should                            \
+                         keep changing with                                    \
+                         introduction of newer                                 \
+                         versions */
+
+#define GD_OP_VERSION_3_6_0 30600 /* Op-Version for GlusterFS 3.6.0 */
+
+#define GD_OP_VERSION_3_7_0 30700 /* Op-version for GlusterFS 3.7.0 */
+
+#define GD_OP_VERSION_3_7_1 30701 /* Op-version for GlusterFS 3.7.1 */
+
+#define GD_OP_VERSION_3_7_2 30702 /* Op-version for GlusterFS 3.7.2 */
+
+#define GD_OP_VERSION_3_7_3 30703 /* Op-version for GlusterFS 3.7.3 */
+
+#define GD_OP_VERSION_3_7_4 30704 /* Op-version for GlusterFS 3.7.4 */
+
+#define GD_OP_VERSION_3_7_5 30705 /* Op-version for GlusterFS 3.7.5 */
+
+#define GD_OP_VERSION_3_7_6 30706 /* Op-version for GlusterFS 3.7.6 */
+
+#define GD_OP_VERSION_3_7_7 30707 /* Op-version for GlusterFS 3.7.7 */
+
+#define GD_OP_VERSION_3_7_10 30710 /* Op-version for GlusterFS 3.7.10 */
+
+#define GD_OP_VERSION_3_7_12 30712 /* Op-version for GlusterFS 3.7.12 */
+
+#define GD_OP_VERSION_3_8_0 30800 /* Op-version for GlusterFS 3.8.0 */
+
+#define GD_OP_VERSION_3_8_3 30803 /* Op-version for GlusterFS 3.8.3 */
+
+#define GD_OP_VERSION_3_8_4 30804 /* Op-version for GlusterFS 3.8.4 */
+
+#define GD_OP_VERSION_3_9_0 30900 /* Op-version for GlusterFS 3.9.0 */
+
+#define GD_OP_VERSION_3_9_1 30901 /* Op-version for GlusterFS 3.9.1 */
+
+#define GD_OP_VERSION_3_10_0 31000 /* Op-version for GlusterFS 3.10.0 */
+
+#define GD_OP_VERSION_3_10_1 31001 /* Op-version for GlusterFS 3.10.1 */
+
+#define GD_OP_VERSION_3_10_2 31002 /* Op-version for GlusterFS 3.10.2 */
+
+#define GD_OP_VERSION_3_11_0 31100 /* Op-version for GlusterFS 3.11.0 */
+
+#define GD_OP_VERSION_3_11_1 31101 /* Op-version for GlusterFS 3.11.1 */
+
+#define GD_OP_VERSION_3_12_0 31200 /* Op-version for GlusterFS 3.12.0 */
+
+#define GD_OP_VERSION_3_12_2 31202 /* Op-version for GlusterFS 3.12.2 */
+
+#define GD_OP_VERSION_3_12_3 31203 /* Op-version for GlusterFS 3.12.3 */
+
+#define GD_OP_VERSION_3_13_0 31300 /* Op-version for GlusterFS 3.13.0 */
+
+#define GD_OP_VERSION_3_13_1 31301 /* Op-version for GlusterFS 3.13.1 */
+
+#define GD_OP_VERSION_3_13_2 31302 /* Op-version for GlusterFS 3.13.2 */
+
+#define GD_OP_VERSION_4_0_0 40000 /* Op-version for GlusterFS 4.0.0 */
+
+#define GD_OP_VERSION_4_1_0 40100 /* Op-version for GlusterFS 4.1.0 */
+
+#define GD_OP_VERSION_5_0 50000 /* Op-version for GlusterFS 5.0 */
+
+#define GD_OP_VERSION_5_4 50400 /* Op-version for GlusterFS 5.4 */
+
+#define GD_OP_VERSION_6_0 60000 /* Op-version for GlusterFS 6.0 */
+
+#define GD_OP_VERSION_7_0 70000 /* Op-version for GlusterFS 7.0 */
+#define GD_OP_VERSION_7_1 70100 /* Op-version for GlusterFS 7.1 */
+#define GD_OP_VERSION_7_2 70200 /* Op-version for GlusterFS 7.2 */
+#define GD_OP_VERSION_7_3 70300 /* Op-version for GlusterFS 7.3 */
+
+#define GD_OP_VERSION_8_0 80000 /* Op-version for GlusterFS 8.0 */
+
+#define GD_OP_VERSION_9_0 90000 /* Op-version for GlusterFS 9.0 */
+
+#define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/options.h"
+
+/* THIS */
+#define THIS (*__glusterfs_this_location())
+#define DECLARE_OLD_THIS xlator_t *old_THIS = THIS
+
+xlator_t **
+__glusterfs_this_location(void);
+xlator_t *
+glusterfs_this_get(void);
+void
+glusterfs_this_set(xlator_t *);
+
+extern xlator_t global_xlator;
+extern struct volume_options global_xl_options[];
+
+/* syncopctx */
+void *
+syncopctx_getctx(void);
+
+/* task */
+void *
+synctask_get(void);
+void
+synctask_set(void *);
+
+/* uuid_buf */
+char *
+glusterfs_uuid_buf_get(void);
+/* lkowner_buf */
+char *
+glusterfs_lkowner_buf_get(void);
+/* leaseid buf */
+char *
+glusterfs_leaseid_buf_get(void);
+char *
+glusterfs_leaseid_exist(void);
+
+/* init */
+int
+glusterfs_globals_init(glusterfs_ctx_t *ctx);
+
+void
+gf_thread_needs_cleanup(void);
+
+struct tvec_base *
+glusterfs_ctx_tw_get(glusterfs_ctx_t *ctx);
+void
+glusterfs_ctx_tw_put(glusterfs_ctx_t *ctx);
+
+extern const char *gf_fop_list[];
+extern const char *gf_upcall_list[];
+
+/* mem acct enable/disable */
+int
+gf_global_mem_acct_enable_get(void);
+int
+gf_global_mem_acct_enable_set(int val);
+#endif /* !_GLOBALS_H */
diff --git a/libglusterfs/src/glusterfs/glusterfs-acl.h b/libglusterfs/src/glusterfs/glusterfs-acl.h
new file mode 100644
index 00000000000..987bf5fab0b
--- /dev/null
+++ b/libglusterfs/src/glusterfs/glusterfs-acl.h
@@ -0,0 +1,162 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERFS_ACL_H
+#define _GLUSTERFS_ACL_H
+
+/* WARNING: Much if this code is restricted to Linux usage.
+ *
+ * It would be much cleaner to replace the code with something that is based on
+ * libacl (or its libc implementation on *BSD).
+ *
+ * Initial work for replacing this Linux specific implementation has been
+ * started as part of the "Improve POSIX ACLs" feature. Functionality for this
+ * feature has been added to the end of this file.
+ */
+
+#include <stdint.h>
+#include <sys/types.h> /* For uid_t */
+
+#include "glusterfs/locking.h" /* For gf_lock_t in struct posix_acl_conf */
+
+#define ACL_PROGRAM 100227
+#define ACLV3_VERSION 3
+
+#define POSIX_ACL_MINIMAL_ACE_COUNT 3
+
+#define POSIX_ACL_READ (0x04)
+#define POSIX_ACL_WRITE (0x02)
+#define POSIX_ACL_EXECUTE (0x01)
+
+#define POSIX_ACL_UNDEFINED_TAG (0x00)
+#define POSIX_ACL_USER_OBJ (0x01)
+#define POSIX_ACL_USER (0x02)
+#define POSIX_ACL_GROUP_OBJ (0x04)
+#define POSIX_ACL_GROUP (0x08)
+#define POSIX_ACL_MASK (0x10)
+#define POSIX_ACL_OTHER (0x20)
+
+#define POSIX_ACL_UNDEFINED_ID (-1)
+
+#define POSIX_ACL_XATTR_VERSION (0x02)
+
+#define POSIX_ACL_ACCESS_XATTR "system.posix_acl_access"
+#define POSIX_ACL_DEFAULT_XATTR "system.posix_acl_default"
+
+struct posix_acl_xattr_entry {
+    uint16_t tag;
+    uint16_t perm;
+    uint32_t id;
+};
+
+struct posix_acl_xattr_header {
+    uint32_t version;
+    struct posix_acl_xattr_entry entries[];
+};
+
+typedef struct posix_acl_xattr_entry posix_acl_xattr_entry;
+typedef struct posix_acl_xattr_header posix_acl_xattr_header;
+
+static inline size_t
+posix_acl_xattr_size(unsigned int count)
+{
+    return (sizeof(posix_acl_xattr_header) +
+            (count * sizeof(posix_acl_xattr_entry)));
+}
+
+static inline ssize_t
+posix_acl_xattr_count(size_t size)
+{
+    if (size < sizeof(posix_acl_xattr_header))
+        return (-1);
+    size -= sizeof(posix_acl_xattr_header);
+    if (size % sizeof(posix_acl_xattr_entry))
+        return (-1);
+    return (size / sizeof(posix_acl_xattr_entry));
+}
+
+struct posix_ace {
+    uint16_t tag;
+    uint16_t perm;
+    uint32_t id;
+};
+
+struct posix_acl {
+    int refcnt;
+    int count;
+    struct posix_ace entries[];
+};
+
+struct posix_acl_ctx {
+    uid_t uid;
+    gid_t gid;
+    mode_t perm;
+    glusterfs_fop_t fop;
+    struct posix_acl *acl_access;
+    struct posix_acl *acl_default;
+};
+
+struct posix_acl_conf {
+    gf_lock_t acl_lock;
+    uid_t super_uid;
+    struct posix_acl *minimal_acl;
+};
+
+/* Above this comment, the legacy POSIX ACL support is kept until it is not
+ * used anymore. Below you will find the more portable version to support POSIX
+ * ACls based on the implementation of libacl (see sys/acl.h). */
+
+/* virtual xattrs passed over RPC, not stored on disk */
+#define GF_POSIX_ACL_ACCESS "glusterfs.posix.acl"
+#define GF_POSIX_ACL_DEFAULT "glusterfs.posix.default_acl"
+#define GF_POSIX_ACL_REQUEST(key)                                              \
+    (!strncmp(key, GF_POSIX_ACL_ACCESS, SLEN(GF_POSIX_ACL_ACCESS)) ||          \
+     !strncmp(key, GF_POSIX_ACL_DEFAULT, SLEN(GF_POSIX_ACL_DEFAULT)))
+
+#ifdef HAVE_SYS_ACL_H /* only NetBSD does not support POSIX ACLs */
+
+#include <sys/acl.h>
+
+static inline const char *
+gf_posix_acl_get_key(const acl_type_t type)
+{
+    char *acl_key = NULL;
+
+    switch (type) {
+        case ACL_TYPE_ACCESS:
+            acl_key = GF_POSIX_ACL_ACCESS;
+            break;
+        case ACL_TYPE_DEFAULT:
+            acl_key = GF_POSIX_ACL_DEFAULT;
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    return acl_key;
+}
+
+static inline acl_type_t
+gf_posix_acl_get_type(const char *key)
+{
+    acl_type_t type = 0;
+
+    if (!strncmp(key, GF_POSIX_ACL_ACCESS, SLEN(GF_POSIX_ACL_ACCESS)))
+        type = ACL_TYPE_ACCESS;
+    else if (!strncmp(key, GF_POSIX_ACL_DEFAULT, SLEN(GF_POSIX_ACL_DEFAULT)))
+        type = ACL_TYPE_DEFAULT;
+    else
+        errno = EINVAL;
+
+    return type;
+}
+
+#endif /* HAVE_SYS_ACL_H */
+#endif /* _GLUSTERFS_ACL_H */
diff --git a/libglusterfs/src/glusterfs/glusterfs-fops.h b/libglusterfs/src/glusterfs/glusterfs-fops.h
new file mode 100644
index 00000000000..030b2701608
--- /dev/null
+++ b/libglusterfs/src/glusterfs/glusterfs-fops.h
@@ -0,0 +1,241 @@
+/*
+  Copyright (c) 2008-2019 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERFS_FOPS_H_
+#define _GLUSTERFS_FOPS_H_
+
+#include <glusterfs/compat.h>
+
+enum glusterfs_fop_t {
+    GF_FOP_NULL = 0,
+    GF_FOP_STAT = 0 + 1,
+    GF_FOP_READLINK = 0 + 2,
+    GF_FOP_MKNOD = 0 + 3,
+    GF_FOP_MKDIR = 0 + 4,
+    GF_FOP_UNLINK = 0 + 5,
+    GF_FOP_RMDIR = 0 + 6,
+    GF_FOP_SYMLINK = 0 + 7,
+    GF_FOP_RENAME = 0 + 8,
+    GF_FOP_LINK = 0 + 9,
+    GF_FOP_TRUNCATE = 0 + 10,
+    GF_FOP_OPEN = 0 + 11,
+    GF_FOP_READ = 0 + 12,
+    GF_FOP_WRITE = 0 + 13,
+    GF_FOP_STATFS = 0 + 14,
+    GF_FOP_FLUSH = 0 + 15,
+    GF_FOP_FSYNC = 0 + 16,
+    GF_FOP_SETXATTR = 0 + 17,
+    GF_FOP_GETXATTR = 0 + 18,
+    GF_FOP_REMOVEXATTR = 0 + 19,
+    GF_FOP_OPENDIR = 0 + 20,
+    GF_FOP_FSYNCDIR = 0 + 21,
+    GF_FOP_ACCESS = 0 + 22,
+    GF_FOP_CREATE = 0 + 23,
+    GF_FOP_FTRUNCATE = 0 + 24,
+    GF_FOP_FSTAT = 0 + 25,
+    GF_FOP_LK = 0 + 26,
+    GF_FOP_LOOKUP = 0 + 27,
+    GF_FOP_READDIR = 0 + 28,
+    GF_FOP_INODELK = 0 + 29,
+    GF_FOP_FINODELK = 0 + 30,
+    GF_FOP_ENTRYLK = 0 + 31,
+    GF_FOP_FENTRYLK = 0 + 32,
+    GF_FOP_XATTROP = 0 + 33,
+    GF_FOP_FXATTROP = 0 + 34,
+    GF_FOP_FGETXATTR = 0 + 35,
+    GF_FOP_FSETXATTR = 0 + 36,
+    GF_FOP_RCHECKSUM = 0 + 37,
+    GF_FOP_SETATTR = 0 + 38,
+    GF_FOP_FSETATTR = 0 + 39,
+    GF_FOP_READDIRP = 0 + 40,
+    GF_FOP_FORGET = 0 + 41,
+    GF_FOP_RELEASE = 0 + 42,
+    GF_FOP_RELEASEDIR = 0 + 43,
+    GF_FOP_GETSPEC = 0 + 44,
+    GF_FOP_FREMOVEXATTR = 0 + 45,
+    GF_FOP_FALLOCATE = 0 + 46,
+    GF_FOP_DISCARD = 0 + 47,
+    GF_FOP_ZEROFILL = 0 + 48,
+    GF_FOP_IPC = 0 + 49,
+    GF_FOP_SEEK = 0 + 50,
+    GF_FOP_LEASE = 0 + 51,
+    GF_FOP_COMPOUND = 0 + 52,
+    GF_FOP_GETACTIVELK = 0 + 53,
+    GF_FOP_SETACTIVELK = 0 + 54,
+    GF_FOP_PUT = 0 + 55,
+    GF_FOP_ICREATE = 0 + 56,
+    GF_FOP_NAMELINK = 0 + 57,
+    GF_FOP_COPY_FILE_RANGE = 0 + 58,
+    GF_FOP_MAXVALUE = 0 + 59,
+};
+typedef enum glusterfs_fop_t glusterfs_fop_t;
+
+enum glusterfs_event_t {
+    GF_EVENT_PARENT_UP = 1,
+    GF_EVENT_POLLIN = 1 + 1,
+    GF_EVENT_POLLOUT = 1 + 2,
+    GF_EVENT_POLLERR = 1 + 3,
+    GF_EVENT_CHILD_UP = 1 + 4,
+    GF_EVENT_CHILD_DOWN = 1 + 5,
+    GF_EVENT_CHILD_CONNECTING = 1 + 6,
+    GF_EVENT_CLEANUP = 9,
+    GF_EVENT_TRANSPORT_CONNECTED = 9 + 1,
+    GF_EVENT_VOLFILE_MODIFIED = 9 + 2,
+    GF_EVENT_GRAPH_NEW = 9 + 3,
+    GF_EVENT_TRANSLATOR_INFO = 9 + 4,
+    GF_EVENT_TRANSLATOR_OP = 9 + 5,
+    GF_EVENT_AUTH_FAILED = 9 + 6,
+    GF_EVENT_VOLUME_DEFRAG = 9 + 7,
+    GF_EVENT_PARENT_DOWN = 9 + 8,
+    GF_EVENT_VOLUME_BARRIER_OP = 9 + 9,
+    GF_EVENT_UPCALL = 9 + 10,
+    GF_EVENT_SCRUB_STATUS = 9 + 11,
+    GF_EVENT_SOME_DESCENDENT_DOWN = 9 + 12,
+    GF_EVENT_SCRUB_ONDEMAND = 9 + 13,
+    GF_EVENT_SOME_DESCENDENT_UP = 9 + 14,
+    GF_EVENT_CHILD_PING = 9 + 15,
+    GF_EVENT_MAXVAL = 9 + 16,
+};
+typedef enum glusterfs_event_t glusterfs_event_t;
+
+enum gf_op_type_t {
+    GF_OP_TYPE_NULL = 0,
+    GF_OP_TYPE_FOP = 0 + 1,
+    GF_OP_TYPE_MGMT = 0 + 2,
+    GF_OP_TYPE_MAX = 0 + 3,
+};
+typedef enum gf_op_type_t gf_op_type_t;
+
+enum glusterfs_lk_cmds_t {
+    GF_LK_GETLK = 0,
+    GF_LK_SETLK = 0 + 1,
+    GF_LK_SETLKW = 0 + 2,
+    GF_LK_RESLK_LCK = 0 + 3,
+    GF_LK_RESLK_LCKW = 0 + 4,
+    GF_LK_RESLK_UNLCK = 0 + 5,
+    GF_LK_GETLK_FD = 0 + 6,
+};
+typedef enum glusterfs_lk_cmds_t glusterfs_lk_cmds_t;
+
+enum glusterfs_lk_types_t {
+    GF_LK_F_RDLCK = 0,
+    GF_LK_F_WRLCK = 0 + 1,
+    GF_LK_F_UNLCK = 0 + 2,
+    GF_LK_EOL = 0 + 3,
+};
+typedef enum glusterfs_lk_types_t glusterfs_lk_types_t;
+
+enum gf_lease_types_t {
+    NONE = 0,
+    GF_RD_LEASE = 1,
+    GF_RW_LEASE = 2,
+    GF_LEASE_MAX_TYPE = 2 + 1,
+};
+typedef enum gf_lease_types_t gf_lease_types_t;
+
+enum gf_lease_cmds_t {
+    GF_GET_LEASE = 1,
+    GF_SET_LEASE = 2,
+    GF_UNLK_LEASE = 3,
+};
+typedef enum gf_lease_cmds_t gf_lease_cmds_t;
+
+#define LEASE_ID_SIZE 16 /* 128bits */
+
+struct gf_lease {
+    gf_lease_cmds_t cmd;
+    gf_lease_types_t lease_type;
+    char lease_id[LEASE_ID_SIZE];
+    u_int lease_flags;
+};
+typedef struct gf_lease gf_lease;
+
+enum glusterfs_lk_recovery_cmds_t {
+    F_RESLK_LCK = 200,
+    F_RESLK_LCKW = 200 + 1,
+    F_RESLK_UNLCK = 200 + 2,
+    F_GETLK_FD = 200 + 3,
+};
+typedef enum glusterfs_lk_recovery_cmds_t glusterfs_lk_recovery_cmds_t;
+
+enum gf_lk_domain_t {
+    GF_LOCK_POSIX = 0,
+    GF_LOCK_INTERNAL = 1,
+};
+typedef enum gf_lk_domain_t gf_lk_domain_t;
+
+enum entrylk_cmd {
+    ENTRYLK_LOCK = 0,
+    ENTRYLK_UNLOCK = 1,
+    ENTRYLK_LOCK_NB = 2,
+};
+typedef enum entrylk_cmd entrylk_cmd;
+
+enum entrylk_type {
+    ENTRYLK_RDLCK = 0,
+    ENTRYLK_WRLCK = 1,
+};
+typedef enum entrylk_type entrylk_type;
+#define GF_MAX_LOCK_OWNER_LEN 1024 /* 1kB as per NLM */
+#define GF_LKOWNER_BUF_SIZE                                                    \
+    ((GF_MAX_LOCK_OWNER_LEN * 2) + (GF_MAX_LOCK_OWNER_LEN / 8))
+
+struct gf_lkowner_t {
+    int len;
+    char data[GF_MAX_LOCK_OWNER_LEN];
+};
+typedef struct gf_lkowner_t gf_lkowner_t;
+
+enum gf_xattrop_flags_t {
+    GF_XATTROP_ADD_ARRAY = 0,
+    GF_XATTROP_ADD_ARRAY64 = 1,
+    GF_XATTROP_OR_ARRAY = 2,
+    GF_XATTROP_AND_ARRAY = 3,
+    GF_XATTROP_GET_AND_SET = 4,
+    GF_XATTROP_ADD_ARRAY_WITH_DEFAULT = 5,
+    GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT = 6,
+};
+typedef enum gf_xattrop_flags_t gf_xattrop_flags_t;
+
+enum gf_seek_what_t {
+    GF_SEEK_DATA = 0,
+    GF_SEEK_HOLE = 1,
+};
+typedef enum gf_seek_what_t gf_seek_what_t;
+
+enum gf_upcall_flags_t {
+    GF_UPCALL_NULL = 0,
+    GF_UPCALL = 1,
+    GF_UPCALL_CI_STAT = 2,
+    GF_UPCALL_CI_XATTR = 3,
+    GF_UPCALL_CI_RENAME = 4,
+    GF_UPCALL_CI_NLINK = 5,
+    GF_UPCALL_CI_FORGET = 6,
+    GF_UPCALL_LEASE_RECALL = 7,
+    GF_UPCALL_FLAGS_MAXVALUE = 8,
+};
+typedef enum gf_upcall_flags_t gf_upcall_flags_t;
+
+enum gf_dict_data_type_t {
+    GF_DATA_TYPE_UNKNOWN = 0,
+    GF_DATA_TYPE_STR_OLD = 1,
+    GF_DATA_TYPE_INT = 2,
+    GF_DATA_TYPE_UINT = 3,
+    GF_DATA_TYPE_DOUBLE = 4,
+    GF_DATA_TYPE_STR = 5,
+    GF_DATA_TYPE_PTR = 6,
+    GF_DATA_TYPE_GFUUID = 7,
+    GF_DATA_TYPE_IATT = 8,
+    GF_DATA_TYPE_MDATA = 9,
+    GF_DATA_TYPE_MAX = 10,
+};
+typedef enum gf_dict_data_type_t gf_dict_data_type_t;
+
+#endif /* !_GLUSTERFS_FOPS_H */
diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h
new file mode 100644
index 00000000000..e6425618b7f
--- /dev/null
+++ b/libglusterfs/src/glusterfs/glusterfs.h
@@ -0,0 +1,838 @@
+/*
+  Copyright (c) 2008-2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERFS_H
+#define _GLUSTERFS_H
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <netdb.h>
+#include <errno.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <arpa/inet.h>
+#include <sys/poll.h>
+#include <pthread.h>
+#include <limits.h> /* For PATH_MAX */
+#include <openssl/sha.h>
+
+#include "glusterfs/glusterfs-fops.h"
+#include "glusterfs/list.h"
+#include "glusterfs/locking.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/lkowner.h"
+#include "glusterfs/compat-uuid.h"
+#include "glusterfs/refcount.h"
+#include "glusterfs/atomic.h"
+
+#define GF_YES 1
+#define GF_NO 0
+
+#define IS_ERROR(ret) ((ret) < 0)
+#define IS_SUCCESS(ret) ((ret) >= 0)
+
+#ifndef O_LARGEFILE
+/* savannah bug #20053, patch for compiling on darwin */
+#define O_LARGEFILE 0100000 /* from bits/fcntl.h */
+#endif
+
+#ifndef O_FMODE_EXEC
+/* redhat bug 843080, added from linux/fs.h */
+#define O_FMODE_EXEC 040  // 0x20
+#endif
+
+#ifndef O_DIRECT
+/* savannah bug #20050, #20052 */
+#define O_DIRECT 0 /* From asm/fcntl.h */
+#endif
+
+#ifndef O_DIRECTORY
+/* FreeBSD does not need O_DIRECTORY */
+#define O_DIRECTORY 0
+#endif
+
+#ifndef EBADFD
+/* Mac OS X does not have EBADFD */
+#define EBADFD EBADF
+#endif
+
+#ifndef FNM_EXTMATCH
+#define FNM_EXTMATCH 0
+#endif
+
+/*gets max-offset on all architectures correctly*/
+#define GF_OFF_MAX ((1ULL << (sizeof(off_t) * 8 - 1)) - 1ULL)
+
+#define GLUSTERD_MAX_SNAP_NAME 255
+#define GLUSTERFS_SOCKET_LISTEN_BACKLOG 1024
+#define GLUSTERD_BRICK_SERVERS "cluster.brick-vol-servers"
+#define SLEN(str) (sizeof(str) - 1)
+
+#define ZR_MOUNTPOINT_OPT "mountpoint"
+#define ZR_ATTR_TIMEOUT_OPT "attribute-timeout"
+#define ZR_ENTRY_TIMEOUT_OPT "entry-timeout"
+#define ZR_NEGATIVE_TIMEOUT_OPT "negative-timeout"
+#define ZR_DIRECT_IO_OPT "direct-io-mode"
+#define ZR_STRICT_VOLFILE_CHECK "strict-volfile-check"
+#define ZR_DUMP_FUSE "dump-fuse"
+#define ZR_FUSE_MOUNTOPTS "fuse-mountopts"
+#define IO_THREADS_QUEUE_SIZE_KEY "io-thread-queue-size"
+
+#define GF_XATTR_CLRLK_CMD "glusterfs.clrlk"
+#define GF_XATTR_PATHINFO_KEY "trusted.glusterfs.pathinfo"
+#define GF_XATTR_NODE_UUID_KEY "trusted.glusterfs.node-uuid"
+#define GF_XATTR_LIST_NODE_UUIDS_KEY "trusted.glusterfs.list-node-uuids"
+#define GF_REBAL_FIND_LOCAL_SUBVOL "glusterfs.find-local-subvol"
+#define GF_REBAL_OLD_FIND_LOCAL_SUBVOL "glusterfs.old-find-local-subvol"
+#define GF_XATTR_VOL_ID_KEY "trusted.glusterfs.volume-id"
+#define GF_XATTR_LOCKINFO_KEY "trusted.glusterfs.lockinfo"
+#define GF_META_LOCK_KEY "glusterfs.lock-migration-meta-lock"
+#define GF_META_UNLOCK_KEY "glusterfs.lock-migration-meta-unlock"
+#define GF_XATTR_GET_REAL_FILENAME_KEY "glusterfs.get_real_filename:"
+#define GF_XATTR_USER_PATHINFO_KEY "glusterfs.pathinfo"
+#define GF_INTERNAL_IGNORE_DEEM_STATFS "ignore-deem-statfs"
+#define GF_XATTR_IOSTATS_DUMP_KEY "trusted.io-stats-dump"
+
+#define GF_READDIR_SKIP_DIRS "readdir-filter-directories"
+#define GF_MDC_LOADED_KEY_NAMES "glusterfs.mdc.loaded.key.names"
+
+#define BD_XATTR_KEY "user.glusterfs"
+#define GF_PREOP_PARENT_KEY "glusterfs.preop.parent.key"
+#define GF_PREOP_CHECK_FAILED "glusterfs.preop.check.failed"
+
+#define XATTR_IS_PATHINFO(x)                                                   \
+    ((strncmp(x, GF_XATTR_PATHINFO_KEY, strlen(x)) == 0) ||                    \
+     (strncmp(x, GF_XATTR_USER_PATHINFO_KEY, strlen(x)) == 0))
+#define XATTR_IS_NODE_UUID(x)                                                  \
+    (strncmp(x, GF_XATTR_NODE_UUID_KEY, SLEN(GF_XATTR_NODE_UUID_KEY)) == 0)
+#define XATTR_IS_NODE_UUID_LIST(x)                                             \
+    (strncmp(x, GF_XATTR_LIST_NODE_UUIDS_KEY,                                  \
+             SLEN(GF_XATTR_LIST_NODE_UUIDS_KEY)) == 0)
+#define XATTR_IS_LOCKINFO(x)                                                   \
+    (strncmp(x, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0)
+
+#define XATTR_IS_BD(x) (strncmp(x, BD_XATTR_KEY, SLEN(BD_XATTR_KEY)) == 0)
+
+#define GF_XATTR_LINKINFO_KEY "trusted.distribute.linkinfo"
+#define GFID_XATTR_KEY "trusted.gfid"
+#define PGFID_XATTR_KEY_PREFIX "trusted.pgfid."
+#define GFID2PATH_VIRT_XATTR_KEY "glusterfs.gfidtopath"
+#define GFID2PATH_XATTR_KEY_PREFIX "trusted.gfid2path."
+#define GFID2PATH_XATTR_KEY_PREFIX_LENGTH 18
+#define VIRTUAL_GFID_XATTR_KEY_STR "glusterfs.gfid.string"
+#define VIRTUAL_GFID_XATTR_KEY "glusterfs.gfid"
+#define GF_XATTR_MDATA_KEY "trusted.glusterfs.mdata"
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GET_ANCESTRY_PATH_KEY "glusterfs.ancestry.path"
+#define GET_ANCESTRY_DENTRY_KEY "glusterfs.ancestry.dentry"
+
+#define BITROT_DEFAULT_CURRENT_VERSION (unsigned long)1
+#define BITROT_DEFAULT_SIGNING_VERSION (unsigned long)0
+
+/* on-disk object signature keys */
+#define BITROT_OBJECT_BAD_KEY "trusted.bit-rot.bad-file"
+#define BITROT_CURRENT_VERSION_KEY "trusted.bit-rot.version"
+#define BITROT_SIGNING_VERSION_KEY "trusted.bit-rot.signature"
+
+/* globally usable bad file marker */
+#define GLUSTERFS_BAD_INODE "glusterfs.bad-inode"
+
+/* on-disk size of signing xattr (not the signature itself) */
+#define BITROT_SIGNING_XATTR_SIZE_KEY "trusted.glusterfs.bit-rot.size"
+
+/* GET/SET object signature */
+#define GLUSTERFS_GET_OBJECT_SIGNATURE "trusted.glusterfs.get-signature"
+#define GLUSTERFS_SET_OBJECT_SIGNATURE "trusted.glusterfs.set-signature"
+
+/* operation needs to be durable on-disk */
+#define GLUSTERFS_DURABLE_OP "trusted.glusterfs.durable-op"
+
+/* key for version exchange b/w bitrot stub and changelog */
+#define GLUSTERFS_VERSION_XCHG_KEY "glusterfs.version.xchg"
+
+#define GLUSTERFS_INTERNAL_FOP_KEY "glusterfs-internal-fop"
+
+#define GF_ENFORCE_MANDATORY_LOCK "trusted.glusterfs.enforce-mandatory-lock"
+
+/* GlusterFS Internal FOP Indicator flags
+ * (To pass information on the context in which a paritcular
+ *  fop is performed between translators)
+ * The presence of a particular flag must be treated as an
+ * indicator of the context, however the flag is added only in
+ * a scenario where there is a need for such context across translators.
+ * So it cannot be an absolute information on context.
+ */
+#define GF_INTERNAL_CTX_KEY "glusterfs.internal-ctx"
+
+/*
+ * Always append entries to end of the enum, do not delete entries.
+ * Currently dict_set_flag allows to set up to 256 flag, if the enum
+ * needs to grow beyond this dict_set_flag has to be changed accordingly
+ */
+enum gf_internal_fop_indicator {
+    GF_DHT_HEAL_DIR /* Index 0 in bit array*/
+};
+
+/* Todo:
+ * Add GF_FOP_LINK_FILE         0x2ULL
+ * address GLUSTERFS_MARKER_DONT_ACCOUNT_KEY and
+ * GLUSTERFS_INTERNAL_FOP_KEY with this flag
+ */
+
+#define DHT_CHANGELOG_RENAME_OP_KEY "changelog.rename-op"
+
+#define GLUSTERFS_WRITE_IS_APPEND "glusterfs.write-is-append"
+#define GLUSTERFS_WRITE_UPDATE_ATOMIC "glusterfs.write-update-atomic"
+#define GLUSTERFS_OPEN_FD_COUNT "glusterfs.open-fd-count"
+#define GLUSTERFS_ACTIVE_FD_COUNT "glusterfs.open-active-fd-count"
+#define GLUSTERFS_INODELK_COUNT "glusterfs.inodelk-count"
+#define GLUSTERFS_ENTRYLK_COUNT "glusterfs.entrylk-count"
+#define GLUSTERFS_POSIXLK_COUNT "glusterfs.posixlk-count"
+#define GLUSTERFS_PARENT_ENTRYLK "glusterfs.parent-entrylk"
+#define GLUSTERFS_INODELK_DOM_COUNT "glusterfs.inodelk-dom-count"
+#define GLUSTERFS_INODELK_DOM_PREFIX "glusterfs.inodelk-dom-prefix"
+#define GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS "glusterfs.multi-dom-lk-cnt-req"
+#define GFID_TO_PATH_KEY "glusterfs.gfid2path"
+#define GF_XATTR_STIME_PATTERN "trusted.glusterfs.*.stime"
+#define GF_XATTR_XTIME_PATTERN "trusted.glusterfs.*.xtime"
+#define GF_XATTR_TRIGGER_SYNC "glusterfs.geo-rep.trigger-sync"
+
+/* quota xattrs */
+#define QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
+#define QUOTA_LIMIT_KEY "trusted.glusterfs.quota.limit-set"
+#define QUOTA_LIMIT_OBJECTS_KEY "trusted.glusterfs.quota.limit-objects"
+#define VIRTUAL_QUOTA_XATTR_CLEANUP_KEY "glusterfs.quota-xattr-cleanup"
+#define QUOTA_READ_ONLY_KEY "trusted.glusterfs.quota.read-only"
+
+/* ctime related */
+#define CTIME_MDATA_XDATA_KEY "set-ctime-mdata"
+
+/* afr related */
+#define AFR_XATTR_PREFIX "trusted.afr"
+
+/* Index xlator related */
+#define GF_XATTROP_INDEX_GFID "glusterfs.xattrop_index_gfid"
+#define GF_XATTROP_ENTRY_CHANGES_GFID "glusterfs.xattrop_entry_changes_gfid"
+#define GF_XATTROP_INDEX_COUNT "glusterfs.xattrop_index_count"
+#define GF_XATTROP_DIRTY_GFID "glusterfs.xattrop_dirty_gfid"
+#define GF_XATTROP_DIRTY_COUNT "glusterfs.xattrop_dirty_count"
+#define GF_XATTROP_ENTRY_IN_KEY "glusterfs.xattrop-entry-create"
+#define GF_XATTROP_ENTRY_OUT_KEY "glusterfs.xattrop-entry-delete"
+#define GF_INDEX_IA_TYPE_GET_REQ "glusterfs.index-ia-type-get-req"
+#define GF_INDEX_IA_TYPE_GET_RSP "glusterfs.index-ia-type-get-rsp"
+
+#define GF_HEAL_INFO "glusterfs.heal-info"
+#define GF_AFR_HEAL_SBRAIN "glusterfs.heal-sbrain"
+#define GF_AFR_SBRAIN_STATUS "replica.split-brain-status"
+#define GF_AFR_SBRAIN_CHOICE "replica.split-brain-choice"
+#define GF_AFR_SPB_CHOICE_TIMEOUT "replica.split-brain-choice-timeout"
+#define GF_AFR_SBRAIN_RESOLVE "replica.split-brain-heal-finalize"
+#define GF_AFR_ADD_BRICK "trusted.add-brick"
+#define GF_AFR_REPLACE_BRICK "trusted.replace-brick"
+#define GF_AFR_DIRTY "trusted.afr.dirty"
+#define GF_XATTROP_ENTRY_OUT "glusterfs.xattrop-entry-delete"
+#define GF_XATTROP_PURGE_INDEX "glusterfs.xattrop-purge-index"
+
+#define GF_GFIDLESS_LOOKUP "gfidless-lookup"
+/* replace-brick and pump related internal xattrs */
+#define RB_PUMP_CMD_START "glusterfs.pump.start"
+#define RB_PUMP_CMD_PAUSE "glusterfs.pump.pause"
+#define RB_PUMP_CMD_COMMIT "glusterfs.pump.commit"
+#define RB_PUMP_CMD_ABORT "glusterfs.pump.abort"
+#define RB_PUMP_CMD_STATUS "glusterfs.pump.status"
+
+#define GLUSTERFS_MARKER_DONT_ACCOUNT_KEY "glusters.marker.dont-account"
+#define GLUSTERFS_RDMA_INLINE_THRESHOLD (2048)
+#define GLUSTERFS_RDMA_MAX_HEADER_SIZE                                         \
+    (228) /* (sizeof (rdma_header_t)                                           \
+             + RDMA_MAX_SEGMENTS                                               \
+             * sizeof (rdma_read_chunk_t))                                     \
+             */
+
+#define GLUSTERFS_RPC_REPLY_SIZE 24
+
+#define STARTING_EVENT_THREADS 2
+
+#define DEFAULT_VAR_RUN_DIRECTORY DATADIR "/run/gluster"
+#define DEFAULT_GLUSTERFSD_MISC_DIRETORY DATADIR "/lib/misc/glusterfsd"
+#ifdef GF_LINUX_HOST_OS
+#define GLUSTERD_DEFAULT_WORKDIR DATADIR "/lib/glusterd"
+#else
+#define GLUSTERD_DEFAULT_WORKDIR DATADIR "/db/glusterd"
+#endif
+#define GF_REPLICATE_TRASH_DIR ".landfill"
+
+/* GlusterFS's maximum supported Auxiliary GIDs */
+#define GF_MAX_AUX_GROUPS 65535
+
+#define GF_UUID_BUF_SIZE 37          /* UUID_CANONICAL_FORM_LEN + NULL */
+#define GF_UUID_BNAME_BUF_SIZE (320) /* (64 + 256) */
+
+#define GF_REBALANCE_TID_KEY "rebalance-id"
+#define GF_REMOVE_BRICK_TID_KEY "remove-brick-id"
+#define GF_TIER_TID_KEY "tier-id"
+#define GF_TIER_ENABLED "tier-enabled"
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+/* Adding this here instead of any glusterd*.h files as it is also required by
+ * cli
+ */
+#define DEFAULT_GLUSTERD_SOCKFILE DATADIR "/run/glusterd.socket"
+
+/* features/marker-quota also needs to have knowledge of link-files so as to
+ * exclude them from accounting.
+ */
+#define DHT_LINKFILE_MODE (S_ISVTX)
+
+#define IS_DHT_LINKFILE_MODE(iabuf)                                            \
+    ((st_mode_from_ia((iabuf)->ia_prot, (iabuf)->ia_type) & ~S_IFMT) ==        \
+     DHT_LINKFILE_MODE)
+#define DHT_LINKFILE_STR "linkto"
+#define DHT_COMMITHASH_STR "commithash"
+
+#define DHT_SKIP_NON_LINKTO_UNLINK "unlink-only-if-dht-linkto-file"
+#define TIER_SKIP_NON_LINKTO_UNLINK "unlink-only-if-tier-linkto-file"
+#define DHT_SKIP_OPEN_FD_UNLINK "dont-unlink-for-open-fd"
+#define DHT_IATT_IN_XDATA_KEY "dht-get-iatt-in-xattr"
+#define DHT_MODE_IN_XDATA_KEY "dht-get-mode-in-xattr"
+#define GET_LINK_COUNT "get-link-count"
+#define GF_GET_SIZE "get-size"
+#define GF_PRESTAT "virt-gf-prestat"
+#define GF_POSTSTAT "virt-gf-poststat"
+
+/*CTR and Marker requires inode dentry link count from posix*/
+#define GF_RESPONSE_LINK_COUNT_XDATA "gf_response_link_count"
+#define GF_REQUEST_LINK_COUNT_XDATA "gf_request_link_count"
+
+#define GF_GET_FILE_BLOCK_COUNT "gf_get_file_block_count"
+
+#define CTR_ATTACH_TIER_LOOKUP "ctr_attach_tier_lookup"
+
+#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
+#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
+
+#define GF_LOG_LRU_BUFSIZE_DEFAULT 5
+#define GF_LOG_LRU_BUFSIZE_MIN 0
+#define GF_LOG_LRU_BUFSIZE_MAX 20
+#define GF_LOG_LRU_BUFSIZE_MIN_STR "0"
+#define GF_LOG_LRU_BUFSIZE_MAX_STR "20"
+
+#define GF_LOG_FLUSH_TIMEOUT_DEFAULT 120
+#define GF_LOG_FLUSH_TIMEOUT_MIN 30
+#define GF_LOG_FLUSH_TIMEOUT_MAX 300
+#define GF_LOG_FLUSH_TIMEOUT_MIN_STR "30"
+#define GF_LOG_FLUSH_TIMEOUT_MAX_STR "300"
+#define GF_LOG_LOCALTIME_DEFAULT 0
+
+#define GF_NETWORK_TIMEOUT 42
+
+#define GF_BACKTRACE_LEN 4096
+#define GF_BACKTRACE_FRAME_COUNT 7
+
+#define GF_LK_ADVISORY 0  /* maps to GLFS_LK_ADVISORY from libgfapi*/
+#define GF_LK_MANDATORY 1 /* maps to GLFS_LK_MANDATORY from libgfapi*/
+#define GF_LOCK_MODE "glusterfs.lk.lkmode"
+
+#define GF_CHECK_XATTR_KEY_AND_GOTO(key, cmpkey, errval, lbl)                  \
+    do {                                                                       \
+        if (key && strcmp(key, cmpkey) == 0) {                                 \
+            errval = -EINVAL;                                                  \
+            goto lbl;                                                          \
+        }                                                                      \
+    } while (0)
+
+#define GF_CS_OBJECT_SIZE "trusted.glusterfs.cs.object_size"
+#define GF_CS_BLOCK_SIZE "trusted.glusterfs.cs.block_size"
+#define GF_CS_NUM_BLOCKS "trusted.glusterfs.cs.num_blocks"
+
+#define GF_CS_XATTR_ARCHIVE_UUID "trusted.cloudsync.uuid"
+
+#define GF_CS_OBJECT_UPLOAD_COMPLETE "trusted.glusterfs.csou.complete"
+#define GF_CS_OBJECT_REMOTE "trusted.glusterfs.cs.remote"
+#define GF_CS_OBJECT_DOWNLOADING "trusted.glusterfs.cs.downloading"
+#define GF_CS_OBJECT_DOWNLOADED "trusted.glusterfs.cs.downloaded"
+#define GF_CS_OBJECT_STATUS "trusted.glusterfs.cs.status"
+#define GF_CS_OBJECT_REPAIR "trusted.glusterfs.cs.repair"
+
+#define gf_boolean_t bool
+#define _gf_false false
+#define _gf_true true
+
+typedef enum {
+    GF_CS_LOCAL = 1,
+    GF_CS_REMOTE = 2,
+    GF_CS_REPAIR = 4,
+    GF_CS_DOWNLOADING = 8,
+    GF_CS_ERROR = 16,
+} gf_cs_obj_state;
+
+typedef enum {
+    GF_FOP_PRI_UNSPEC = -1, /* Priority not specified */
+    GF_FOP_PRI_HI = 0,      /* low latency */
+    GF_FOP_PRI_NORMAL,      /* normal */
+    GF_FOP_PRI_LO,          /* bulk */
+    GF_FOP_PRI_LEAST,       /* least */
+    GF_FOP_PRI_MAX,         /* Highest */
+} gf_fop_pri_t;
+
+typedef enum {
+    /* The 'component' (xlator / option) is not yet setting the flag */
+    GF_UNCLASSIFIED = 0,
+    /* The 'component' is experimental, should not be recommened
+       in production mode */
+    GF_EXPERIMENTAL,
+    /* The 'component' is tech preview, ie, it is 'mostly' working as
+       expected, but can have some of the corner cases, which is not
+       handled. */
+    GF_TECH_PREVIEW,
+    /* The 'component' is good to run. Has good enough test and
+       documentation coverage. */
+    GF_MAINTAINED,
+    /* The component is:
+       - no more a focus
+       - no more solving a valid use case
+       - no more maintained, no volunteers to maintain
+       - there is 'maintained' or 'tech-preview' feature,
+       which does the same thing, better.
+    */
+    GF_DEPRECATED,
+    /* The 'component' is no more 'built'. */
+    GF_OBSOLETE,
+    /* The 'component' exist for Documentation purposes.
+       No real usecase */
+    GF_DOCUMENT_PURPOSE,
+} gf_category_t;
+
+static const char *const FOP_PRI_STRINGS[] = {"HIGH", "NORMAL", "LOW", "LEAST"};
+
+static inline const char *
+fop_pri_to_string(gf_fop_pri_t pri)
+{
+    if (IS_ERROR(pri))
+        return "UNSPEC";
+
+    if (pri >= GF_FOP_PRI_MAX)
+        return "INVALID";
+
+    return FOP_PRI_STRINGS[pri];
+}
+
+const char *
+fop_enum_to_pri_string(glusterfs_fop_t fop);
+
+#define GF_SET_IF_NOT_PRESENT 0x1 /* default behaviour */
+#define GF_SET_OVERWRITE 0x2      /* Overwrite with the buf given */
+#define GF_SET_DIR_ONLY 0x4
+#define GF_SET_EPOCH_TIME 0x8    /* used by afr dir lookup selfheal */
+#define GF_AUXILLARY_PARGFID 0xd /* RIO dummy parent gfid */
+
+/* key value which quick read uses to get small files in lookup cbk */
+#define GF_CONTENT_KEY "glusterfs.content"
+
+struct _xlator_cmdline_option {
+    struct list_head cmd_args;
+    char *volume;
+    char *key;
+    char *value;
+};
+typedef struct _xlator_cmdline_option xlator_cmdline_option_t;
+
+struct _server_cmdline {
+    struct list_head list;
+    char *volfile_server;
+    char *transport;
+    int port;
+};
+typedef struct _server_cmdline server_cmdline_t;
+
+#define GF_OPTION_ENABLE _gf_true
+#define GF_OPTION_DISABLE _gf_false
+#define GF_OPTION_DEFERRED 2
+
+typedef enum { _gf_none, _gf_memcheck, _gf_drd } gf_valgrind_tool;
+
+struct _cmd_args {
+    /* basic options */
+    char *volfile_server;
+    server_cmdline_t *curr_server;
+    /* List of backup  volfile servers, including original */
+    struct list_head volfile_servers;
+    char *volfile;
+    char *log_server;
+    gf_loglevel_t log_level;
+    char *log_file;
+    char *log_ident;
+    gf_log_logger_t logger;
+    gf_log_format_t log_format;
+    uint32_t log_buf_size;
+    uint32_t log_flush_timeout;
+    int32_t max_connect_attempts;
+    char *print_exports;
+    char *print_netgroups;
+    int print_xlatordir;
+    int print_statedumpdir;
+    int print_logdir;
+    int print_libexecdir;
+    /* advanced options */
+    uint32_t volfile_server_port;
+    char *volfile_server_transport;
+    uint32_t log_server_port;
+    char *pid_file;
+    char *sock_file;
+    int no_daemon_mode;
+    char *run_id;
+    int debug_mode;
+    int read_only;
+    int acl;
+    int selinux;
+    int capability;
+    int enable_ino32;
+    int worm;
+    int mac_compat;
+    int fopen_keep_cache;
+    int gid_timeout;
+    char gid_timeout_set;
+    int aux_gfid_mount;
+
+    /* need a process wide timer-wheel? */
+    int global_timer_wheel;
+
+    /* list of xlator_option_t */
+    struct list_head xlator_options;
+
+    /* fuse options */
+    int fuse_direct_io_mode;
+    char *use_readdirp;
+    int no_root_squash;
+    int volfile_check;
+    double fuse_entry_timeout;
+    double fuse_negative_timeout;
+    double fuse_attribute_timeout;
+    char *volume_name;
+    int fuse_nodev;
+    int fuse_nosuid;
+    char *dump_fuse;
+    pid_t client_pid;
+    int client_pid_set;
+    unsigned uid_map_root;
+    int32_t lru_limit;
+    int32_t invalidate_limit;
+    int background_qlen;
+    int congestion_threshold;
+    char *fuse_mountopts;
+    int mem_acct;
+    int resolve_gids;
+
+    /* key args */
+    char *mount_point;
+    char *volfile_id;
+
+    /* required for portmap */
+    int brick_port;
+    char *brick_name;
+    int brick_port2;
+
+    /* Should management connections use SSL? */
+    int secure_mgmt;
+
+    /* Linux-only OOM killer adjustment */
+#ifdef GF_LINUX_HOST_OS
+    char *oom_score_adj;
+#endif
+
+    /* Run this process with valgrind? Might want to prevent calling
+     * functions that prevent valgrind from working correctly, like
+     * dlclose(). */
+    gf_valgrind_tool vgtool;
+
+    int localtime_logging;
+
+    /* For the subdir mount */
+    char *subdir_mount;
+
+    char *process_name;
+    char *event_history;
+    int thin_client;
+    uint32_t reader_thread_count;
+
+    /* FUSE writeback cache support */
+    int kernel_writeback_cache;
+    uint32_t attr_times_granularity;
+
+    int fuse_flush_handle_interrupt;
+    int fuse_auto_inval;
+
+    bool global_threading;
+    bool brick_mux;
+
+    uint32_t fuse_dev_eperm_ratelimit_ns;
+};
+typedef struct _cmd_args cmd_args_t;
+
+struct _glusterfs_graph {
+    struct list_head list;
+    struct timeval dob;
+    void *first;
+    void *top; /* selected by -n */
+    int xl_count;
+    int id;   /* Used in logging */
+    int used; /* Should be set when fuse gets
+                        first CHILD_UP */
+    uint32_t volfile_checksum;
+    uint32_t leaf_count;
+    void *last_xl; /* Stores the last xl of the graph, as of now only populated
+                      in client multiplexed code path */
+    pthread_mutex_t mutex;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int parent_down;
+    char graph_uuid[128];
+    char volume_id[GF_UUID_BUF_SIZE];
+};
+typedef struct _glusterfs_graph glusterfs_graph_t;
+
+typedef int32_t (*glusterfsd_mgmt_event_notify_fn_t)(int32_t event, void *data,
+                                                     ...);
+
+typedef enum {
+    MGMT_SSL_NEVER = 0,
+    MGMT_SSL_COPY_IO,
+    MGMT_SSL_ALWAYS
+} mgmt_ssl_t;
+
+struct tvec_base;
+
+/* reference counting for the global (per ctx) timer-wheel */
+struct gf_ctx_tw {
+    GF_REF_DECL;
+    struct tvec_base *timer_wheel; /* global timer-wheel instance */
+};
+
+struct _glusterfs_ctx {
+    cmd_args_t cmd_args;
+    char *process_uuid;
+    FILE *pidfp;
+    char fin;
+    void *timer;
+    void *ib;
+    struct call_pool *pool;
+    void *event_pool;
+    void *iobuf_pool;
+    void *logbuf_pool;
+    gf_lock_t lock;
+    size_t page_size;
+
+    /* one per volfile parse */
+    struct list_head graphs;
+
+    /* the latest graph in use */
+    glusterfs_graph_t *active;
+
+    /* fuse or nfs (but not protocol/server) */
+    void *master;
+
+    /* xlator implementing MOPs for centralized logging, volfile server */
+    void *mgmt;
+
+    /* listener of the commands from glusterd */
+    void *listener;
+
+    /* toggle switch for latency measurement */
+    unsigned char measure_latency;
+    pthread_t sigwaiter;
+    char *cmdlinestr;
+    struct mem_pool *stub_mem_pool;
+    unsigned char cleanup_started;
+    int graph_id;        /* Incremented per graph, value should
+                            indicate how many times the graph has
+                            got changed */
+    pid_t mnt_pid;       /* pid of the mount agent */
+    int process_mode;    /*mode in which process is runninng*/
+    struct syncenv *env; /* The env pointer to the synctasks */
+
+    struct list_head mempool_list; /* used to keep a global list of
+                                      mempools, used to log details of
+                                      mempool in statedump */
+    char *statedump_path;
+
+    struct mem_pool *dict_pool;
+    struct mem_pool *dict_pair_pool;
+    struct mem_pool *dict_data_pool;
+
+    glusterfsd_mgmt_event_notify_fn_t notify; /* Used for xlators to make
+                                                 call to fsd-mgmt */
+    gf_log_handle_t log; /* all logging related variables */
+
+    int mem_acct_enable;
+
+    int daemon_pipe[2];
+
+    struct clienttable *clienttable;
+
+    /*
+     * Should management connections use SSL?  This is the only place we
+     * can put it where both daemon-startup and socket code will see it.
+     *
+     * Why is it an int?  Because we're included before common-utils.h,
+     * which defines gf_boolean_t (what we really want).  It doesn't make
+     * any sense, but it's not worth turning the codebase upside-down to
+     * fix it.  Thus, an int.
+     */
+    int secure_mgmt;
+
+    /* The option is use to set cert_depth while management connection
+       use SSL
+     */
+    int ssl_cert_depth;
+
+    /*
+     * Should *our* server/inbound connections use SSL?  This is only true
+     * if we're glusterd and secure_mgmt is set, or if we're glusterfsd
+     * and SSL is set on the I/O path.  It should never be set e.g. for
+     * NFS.
+     */
+    mgmt_ssl_t secure_srvr;
+    /* Buffer to 'save' backtrace even under OOM-kill like situations*/
+    char btbuf[GF_BACKTRACE_LEN];
+
+    pthread_mutex_t notify_lock;
+    pthread_mutex_t cleanup_lock;
+    pthread_cond_t notify_cond;
+    int notifying;
+
+    struct gf_ctx_tw *tw; /* refcounted timer_wheel */
+
+    gf_lock_t volfile_lock;
+
+    /* configuration related elements, which gets changed
+       from global xlator */
+    struct {
+        char *metrics_dumppath;
+    } config;
+
+    struct {
+        gf_atomic_t max_dict_pairs;
+        gf_atomic_t total_pairs_used;
+        gf_atomic_t total_dicts_used;
+    } stats;
+
+    struct list_head volfile_list;
+    /* Add members to manage janitor threads for cleanup fd */
+    struct list_head janitor_fds;
+    pthread_cond_t fd_cond;
+    pthread_mutex_t fd_lock;
+    pthread_t janitor;
+    /* The variable is use to save total posix xlator count */
+    uint32_t pxl_count;
+
+    char volume_id[GF_UUID_BUF_SIZE]; /* Used only in protocol/client */
+};
+typedef struct _glusterfs_ctx glusterfs_ctx_t;
+
+typedef struct {
+    char volfile_checksum[SHA256_DIGEST_LENGTH];
+    char vol_id[NAME_MAX + 1];
+    struct list_head volfile_list;
+    glusterfs_graph_t *graph;
+    FILE *pidfp;
+} gf_volfile_t;
+
+glusterfs_ctx_t *
+glusterfs_ctx_new(void);
+
+struct gf_flock {
+    short l_type;
+    short l_whence;
+    off_t l_start;
+    off_t l_len;
+    pid_t l_pid;
+    gf_lkowner_t l_owner;
+};
+
+typedef struct lock_migration_info {
+    struct list_head list;
+    struct gf_flock flock;
+    char *client_uid;
+    uint32_t lk_flags;
+} lock_migration_info_t;
+
+#define GF_MUST_CHECK __attribute__((warn_unused_result))
+/*
+ * Some macros (e.g. ALLOC_OR_GOTO) set variables in function scope, but the
+ * calling function might not only declare the variable to keep the macro happy
+ * and not use it otherwise.  In such cases, the following can be used to
+ * suppress the "set but not used" warning that would otherwise occur.
+ */
+#define GF_UNUSED __attribute__((unused))
+
+/*
+ * If present, this has the following effects:
+ *
+ *      glusterd enables privileged commands over TCP
+ *
+ *      all code enables SSL for outbound connections to management port
+ *
+ *      glusterd enables SSL for inbound connections
+ *
+ * Servers and clients enable/disable SSL among themselves by other means.
+ * Making secure management connections conditional on a file is a bit of a
+ * hack, but we don't have any other place for such global settings across
+ * all of the affected components.  Making it a compile-time option would
+ * reduce functionality, both for users and for testing (which can now be
+ * done using secure connections for all tests without change elsewhere).
+ *
+ */
+#define SECURE_ACCESS_FILE GLUSTERD_DEFAULT_WORKDIR "/secure-access"
+
+int
+glusterfs_graph_prepare(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
+                        char *volume_name);
+int
+glusterfs_graph_destroy_residual(glusterfs_graph_t *graph);
+int
+glusterfs_graph_deactivate(glusterfs_graph_t *graph);
+int
+glusterfs_graph_destroy(glusterfs_graph_t *graph);
+int
+glusterfs_get_leaf_count(glusterfs_graph_t *graph);
+int
+glusterfs_graph_activate(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx);
+glusterfs_graph_t *
+glusterfs_graph_construct(FILE *fp);
+int
+glusterfs_graph_init(glusterfs_graph_t *graph);
+glusterfs_graph_t *
+glusterfs_graph_new(void);
+int
+glusterfs_graph_reconfigure(glusterfs_graph_t *oldgraph,
+                            glusterfs_graph_t *newgraph);
+int
+glusterfs_graph_attach(glusterfs_graph_t *orig_graph, char *path,
+                       glusterfs_graph_t **newgraph);
+int
+glusterfs_graph_parent_up(glusterfs_graph_t *graph);
+
+void
+gf_free_mig_locks(lock_migration_info_t *locks);
+
+int
+glusterfs_read_secure_access_file(void);
+int
+glusterfs_graph_fini(glusterfs_graph_t *graph);
+#endif /* _GLUSTERFS_H */
diff --git a/libglusterfs/src/glusterfs/graph-utils.h b/libglusterfs/src/glusterfs/graph-utils.h
new file mode 100644
index 00000000000..247f1a55d5a
--- /dev/null
+++ b/libglusterfs/src/glusterfs/graph-utils.h
@@ -0,0 +1,20 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GRAPH_H_
+#define _GRAPH_H_
+
+int
+glusterfs_graph_print_file(FILE *file, glusterfs_graph_t *graph);
+int
+glusterfs_xlator_link(xlator_t *pxl, xlator_t *cxl);
+void
+glusterfs_graph_set_first(glusterfs_graph_t *graph, xlator_t *xl);
+#endif
diff --git a/libglusterfs/src/glusterfs/hashfn.h b/libglusterfs/src/glusterfs/hashfn.h
new file mode 100644
index 00000000000..6e92e706d8c
--- /dev/null
+++ b/libglusterfs/src/glusterfs/hashfn.h
@@ -0,0 +1,23 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __HASHFN_H__
+#define __HASHFN_H__
+
+#include <sys/types.h>
+#include <stdint.h>
+
+uint32_t
+SuperFastHash(const char *data, int32_t len);
+
+uint32_t
+gf_dm_hashfn(const char *msg, int len);
+
+#endif /* __HASHFN_H__ */
diff --git a/libglusterfs/src/glusterfs/iatt.h b/libglusterfs/src/glusterfs/iatt.h
new file mode 100644
index 00000000000..f03d68b02f0
--- /dev/null
+++ b/libglusterfs/src/glusterfs/iatt.h
@@ -0,0 +1,489 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _IATT_H
+#define _IATT_H
+
+#if defined(GF_LINUX_HOST_OS)
+#include <sys/sysmacros.h> /* for makedev(3), major(3), minor(3) */
+#endif
+#include <sys/types.h>
+#include <sys/stat.h> /* for iatt <--> stat conversions */
+#include <unistd.h>
+
+#include "glusterfs/compat.h"
+#include "glusterfs/compat-uuid.h"
+
+typedef enum {
+    IA_INVAL = 0,
+    IA_IFREG,
+    IA_IFDIR,
+    IA_IFLNK,
+    IA_IFBLK,
+    IA_IFCHR,
+    IA_IFIFO,
+    IA_IFSOCK
+} ia_type_t;
+
+typedef struct {
+    uint8_t suid : 1;
+    uint8_t sgid : 1;
+    uint8_t sticky : 1;
+    struct {
+        uint8_t read : 1;
+        uint8_t write : 1;
+        uint8_t exec : 1;
+    } owner, group, other;
+} ia_prot_t;
+
+struct iatt {
+    uint64_t ia_flags;
+    uint64_t ia_ino;     /* inode number */
+    uint64_t ia_dev;     /* backing device ID */
+    uint64_t ia_rdev;    /* device ID (if special file) */
+    uint64_t ia_size;    /* file size in bytes */
+    uint32_t ia_nlink;   /* Link count */
+    uint32_t ia_uid;     /* user ID of owner */
+    uint32_t ia_gid;     /* group ID of owner */
+    uint32_t ia_blksize; /* blocksize for filesystem I/O */
+    uint64_t ia_blocks;  /* number of 512B blocks allocated */
+    int64_t ia_atime;    /* last access time */
+    int64_t ia_mtime;    /* last modification time */
+    int64_t ia_ctime;    /* last status change time */
+    int64_t ia_btime;    /* creation time. Fill using statx */
+    uint32_t ia_atime_nsec;
+    uint32_t ia_mtime_nsec;
+    uint32_t ia_ctime_nsec;
+    uint32_t ia_btime_nsec;
+    uint64_t ia_attributes;      /* chattr related:compressed, immutable,
+                                  * append only, encrypted etc.*/
+    uint64_t ia_attributes_mask; /* Mask for the attributes */
+
+    uuid_t ia_gfid;
+    ia_type_t ia_type; /* type of file */
+    ia_prot_t ia_prot; /* protection */
+};
+
+struct old_iatt {
+    uint64_t ia_ino; /* inode number */
+    uuid_t ia_gfid;
+    uint64_t ia_dev;     /* backing device ID */
+    ia_type_t ia_type;   /* type of file */
+    ia_prot_t ia_prot;   /* protection */
+    uint32_t ia_nlink;   /* Link count */
+    uint32_t ia_uid;     /* user ID of owner */
+    uint32_t ia_gid;     /* group ID of owner */
+    uint64_t ia_rdev;    /* device ID (if special file) */
+    uint64_t ia_size;    /* file size in bytes */
+    uint32_t ia_blksize; /* blocksize for filesystem I/O */
+    uint64_t ia_blocks;  /* number of 512B blocks allocated */
+    uint32_t ia_atime;   /* last access time */
+    uint32_t ia_atime_nsec;
+    uint32_t ia_mtime; /* last modification time */
+    uint32_t ia_mtime_nsec;
+    uint32_t ia_ctime; /* last status change time */
+    uint32_t ia_ctime_nsec;
+};
+
+struct mdata_iatt {
+    int64_t ia_atime; /* last access time */
+    int64_t ia_mtime; /* last modification time */
+    int64_t ia_ctime; /* last status change time */
+    uint32_t ia_atime_nsec;
+    uint32_t ia_mtime_nsec;
+    uint32_t ia_ctime_nsec;
+};
+
+/* 64-bit mask for valid members in struct iatt. */
+#define IATT_TYPE 0x0000000000000001U
+#define IATT_MODE 0x0000000000000002U
+#define IATT_NLINK 0x0000000000000004U
+#define IATT_UID 0x0000000000000008U
+#define IATT_GID 0x0000000000000010U
+#define IATT_ATIME 0x0000000000000020U
+#define IATT_MTIME 0x0000000000000040U
+#define IATT_CTIME 0x0000000000000080U
+#define IATT_INO 0x0000000000000100U
+#define IATT_SIZE 0x0000000000000200U
+#define IATT_BLOCKS 0x0000000000000400U
+#define IATT_BTIME 0x0000000000000800U
+#define IATT_GFID 0x0000000000001000U
+
+/* Macros for checking validity of struct iatt members.*/
+#define IATT_TYPE_VALID(iaflags) (iaflags & IATT_TYPE)
+#define IATT_MODE_VALID(iaflags) (iaflags & IATT_MODE)
+#define IATT_NLINK_VALID(iaflags) (iaflags & IATT_NLINK)
+#define IATT_UID_VALID(iaflags) (iaflags & IATT_UID)
+#define IATT_GID_VALID(iaflags) (iaflags & IATT_GID)
+#define IATT_ATIME_VALID(iaflags) (iaflags & IATT_ATIME)
+#define IATT_MTIME_VALID(iaflags) (iaflags & IATT_MTIME)
+#define IATT_CTIME_VALID(iaflags) (iaflags & IATT_CTIME)
+#define IATT_INO_VALID(iaflags) (iaflags & IATT_INO)
+#define IATT_SIZE_VALID(iaflags) (iaflags & IATT_SIZE)
+#define IATT_BLOCKS_VALID(iaflags) (iaflags & IATT_BLOCKS)
+#define IATT_BTIME_VALID(iaflags) (iaflags & IATT_BTIME)
+#define IATT_GFID_VALID(iaflags) (iaflags & IATT_GFID)
+
+#define IA_ISREG(t) (t == IA_IFREG)
+#define IA_ISDIR(t) (t == IA_IFDIR)
+#define IA_ISLNK(t) (t == IA_IFLNK)
+#define IA_ISBLK(t) (t == IA_IFBLK)
+#define IA_ISCHR(t) (t == IA_IFCHR)
+#define IA_ISFIFO(t) (t == IA_IFIFO)
+#define IA_ISSOCK(t) (t == IA_IFSOCK)
+#define IA_ISINVAL(t) (t == IA_INVAL)
+
+#define IA_PROT_RUSR(prot) ((prot).owner.read == 1)
+#define IA_PROT_WUSR(prot) ((prot).owner.write == 1)
+#define IA_PROT_XUSR(prot) ((prot).owner.exec == 1)
+
+#define IA_PROT_RGRP(prot) ((prot).group.read == 1)
+#define IA_PROT_WGRP(prot) ((prot).group.write == 1)
+#define IA_PROT_XGRP(prot) ((prot).group.exec == 1)
+
+#define IA_PROT_ROTH(prot) ((prot).other.read == 1)
+#define IA_PROT_WOTH(prot) ((prot).other.write == 1)
+#define IA_PROT_XOTH(prot) ((prot).other.exec == 1)
+
+#define IA_PROT_SUID(prot) ((prot).suid == 1)
+#define IA_PROT_SGID(prot) ((prot).sgid == 1)
+#define IA_PROT_STCKY(prot) ((prot).sticky == 1)
+
+#define IA_FILE_OR_DIR(t) (IA_ISREG(t) || IA_ISDIR(t))
+
+static inline uint32_t
+ia_major(uint64_t ia_dev)
+{
+    return (uint32_t)(ia_dev >> 32);
+}
+
+static inline uint32_t
+ia_minor(uint64_t ia_dev)
+{
+    return (uint32_t)(ia_dev & 0xffffffff);
+}
+
+static inline uint64_t
+ia_makedev(uint32_t ia_maj, uint32_t ia_min)
+{
+    return ((((uint64_t)ia_maj) << 32) | ia_min);
+}
+
+static inline ia_prot_t
+ia_prot_from_st_mode(mode_t mode)
+{
+    ia_prot_t ia_prot = {
+        0,
+    };
+
+    if (mode & S_ISUID)
+        ia_prot.suid = 1;
+    if (mode & S_ISGID)
+        ia_prot.sgid = 1;
+    if (mode & S_ISVTX)
+        ia_prot.sticky = 1;
+
+    if (mode & S_IRUSR)
+        ia_prot.owner.read = 1;
+    if (mode & S_IWUSR)
+        ia_prot.owner.write = 1;
+    if (mode & S_IXUSR)
+        ia_prot.owner.exec = 1;
+
+    if (mode & S_IRGRP)
+        ia_prot.group.read = 1;
+    if (mode & S_IWGRP)
+        ia_prot.group.write = 1;
+    if (mode & S_IXGRP)
+        ia_prot.group.exec = 1;
+
+    if (mode & S_IROTH)
+        ia_prot.other.read = 1;
+    if (mode & S_IWOTH)
+        ia_prot.other.write = 1;
+    if (mode & S_IXOTH)
+        ia_prot.other.exec = 1;
+
+    return ia_prot;
+}
+
+static inline ia_type_t
+ia_type_from_st_mode(mode_t mode)
+{
+    ia_type_t type = IA_INVAL;
+
+    if (S_ISREG(mode))
+        type = IA_IFREG;
+    if (S_ISDIR(mode))
+        type = IA_IFDIR;
+    if (S_ISLNK(mode))
+        type = IA_IFLNK;
+    if (S_ISBLK(mode))
+        type = IA_IFBLK;
+    if (S_ISCHR(mode))
+        type = IA_IFCHR;
+    if (S_ISFIFO(mode))
+        type = IA_IFIFO;
+    if (S_ISSOCK(mode))
+        type = IA_IFSOCK;
+
+    return type;
+}
+
+static inline uint32_t
+st_mode_prot_from_ia(ia_prot_t prot)
+{
+    uint32_t prot_bit = 0;
+
+    if (prot.suid)
+        prot_bit |= S_ISUID;
+    if (prot.sgid)
+        prot_bit |= S_ISGID;
+    if (prot.sticky)
+        prot_bit |= S_ISVTX;
+
+    if (prot.owner.read)
+        prot_bit |= S_IRUSR;
+    if (prot.owner.write)
+        prot_bit |= S_IWUSR;
+    if (prot.owner.exec)
+        prot_bit |= S_IXUSR;
+
+    if (prot.group.read)
+        prot_bit |= S_IRGRP;
+    if (prot.group.write)
+        prot_bit |= S_IWGRP;
+    if (prot.group.exec)
+        prot_bit |= S_IXGRP;
+
+    if (prot.other.read)
+        prot_bit |= S_IROTH;
+    if (prot.other.write)
+        prot_bit |= S_IWOTH;
+    if (prot.other.exec)
+        prot_bit |= S_IXOTH;
+
+    return prot_bit;
+}
+
+static inline uint32_t
+st_mode_type_from_ia(ia_type_t type)
+{
+    uint32_t type_bit = 0;
+
+    switch (type) {
+        case IA_IFREG:
+            type_bit = S_IFREG;
+            break;
+        case IA_IFDIR:
+            type_bit = S_IFDIR;
+            break;
+        case IA_IFLNK:
+            type_bit = S_IFLNK;
+            break;
+        case IA_IFBLK:
+            type_bit = S_IFBLK;
+            break;
+        case IA_IFCHR:
+            type_bit = S_IFCHR;
+            break;
+        case IA_IFIFO:
+            type_bit = S_IFIFO;
+            break;
+        case IA_IFSOCK:
+            type_bit = S_IFSOCK;
+            break;
+        case IA_INVAL:
+            break;
+    }
+
+    return type_bit;
+}
+
+static inline mode_t
+st_mode_from_ia(ia_prot_t prot, ia_type_t type)
+{
+    mode_t st_mode = 0;
+    uint32_t type_bit = 0;
+    uint32_t prot_bit = 0;
+
+    type_bit = st_mode_type_from_ia(type);
+    prot_bit = st_mode_prot_from_ia(prot);
+
+    st_mode = (type_bit | prot_bit);
+
+    return st_mode;
+}
+
+static inline void
+iatt_to_mdata(struct mdata_iatt *mdata, struct iatt *iatt)
+{
+    mdata->ia_atime = iatt->ia_atime;
+    mdata->ia_atime_nsec = iatt->ia_atime_nsec;
+    mdata->ia_mtime = iatt->ia_mtime;
+    mdata->ia_mtime_nsec = iatt->ia_mtime_nsec;
+    mdata->ia_ctime = iatt->ia_ctime;
+    mdata->ia_ctime_nsec = iatt->ia_ctime_nsec;
+}
+
+static inline int
+iatt_from_stat(struct iatt *iatt, struct stat *stat)
+{
+    iatt->ia_dev = stat->st_dev;
+    iatt->ia_ino = stat->st_ino;
+
+    iatt->ia_type = ia_type_from_st_mode(stat->st_mode);
+    iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+
+    iatt->ia_nlink = stat->st_nlink;
+    iatt->ia_uid = stat->st_uid;
+    iatt->ia_gid = stat->st_gid;
+
+    iatt->ia_rdev = ia_makedev(major(stat->st_rdev), minor(stat->st_rdev));
+
+    iatt->ia_size = stat->st_size;
+    iatt->ia_blksize = stat->st_blksize;
+    iatt->ia_blocks = stat->st_blocks;
+
+    /* There is a possibility that the backend FS (like XFS) can
+       allocate blocks beyond EOF for better performance reasons, which
+       results in 'st_blocks' with higher values than what is consumed by
+       the file descriptor. This would break few logic inside GlusterFS,
+       like quota behavior etc, thus we need the exact number of blocks
+       which are consumed by the file to the higher layers inside GlusterFS.
+       Currently, this logic won't work for sparse files (ie, file with
+       holes)
+    */
+    {
+        uint64_t maxblocks;
+
+        maxblocks = (iatt->ia_size + 511) / 512;
+
+        if (iatt->ia_blocks > maxblocks)
+            iatt->ia_blocks = maxblocks;
+    }
+
+    iatt->ia_atime = stat->st_atime;
+    iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+
+    iatt->ia_mtime = stat->st_mtime;
+    iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+
+    iatt->ia_ctime = stat->st_ctime;
+    iatt->ia_ctime_nsec = ST_CTIM_NSEC(stat);
+
+    /* Setting IATT_INO in ia_flags is done in posix_fill_ino_from_gfid. */
+    iatt->ia_flags = iatt->ia_flags | IATT_TYPE | IATT_MODE | IATT_NLINK |
+                     IATT_UID | IATT_GID | IATT_SIZE | IATT_BLOCKS |
+                     IATT_ATIME | IATT_MTIME | IATT_CTIME;
+
+    return 0;
+}
+
+static inline int
+iatt_to_stat(struct iatt *iatt, struct stat *stat)
+{
+    stat->st_dev = iatt->ia_dev;
+    stat->st_ino = iatt->ia_ino;
+
+    stat->st_mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
+
+    stat->st_nlink = iatt->ia_nlink;
+    stat->st_uid = iatt->ia_uid;
+    stat->st_gid = iatt->ia_gid;
+
+    stat->st_rdev = makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev));
+
+    stat->st_size = iatt->ia_size;
+    stat->st_blksize = iatt->ia_blksize;
+    stat->st_blocks = iatt->ia_blocks;
+
+    stat->st_atime = iatt->ia_atime;
+    ST_ATIM_NSEC_SET(stat, iatt->ia_atime_nsec);
+
+    stat->st_mtime = iatt->ia_mtime;
+    ST_MTIM_NSEC_SET(stat, iatt->ia_mtime_nsec);
+
+    stat->st_ctime = iatt->ia_ctime;
+    ST_CTIM_NSEC_SET(stat, iatt->ia_ctime_nsec);
+
+    return 0;
+}
+
+static inline void
+oldiatt_from_iatt(struct old_iatt *o_iatt, struct iatt *c_iatt)
+{
+    o_iatt->ia_dev = c_iatt->ia_dev;
+    o_iatt->ia_ino = c_iatt->ia_ino;
+    o_iatt->ia_type = c_iatt->ia_type;
+    o_iatt->ia_prot = c_iatt->ia_prot;
+    o_iatt->ia_nlink = c_iatt->ia_nlink;
+    o_iatt->ia_uid = c_iatt->ia_uid;
+    o_iatt->ia_gid = c_iatt->ia_gid;
+    o_iatt->ia_rdev = c_iatt->ia_rdev;
+    o_iatt->ia_size = c_iatt->ia_size;
+    o_iatt->ia_blksize = c_iatt->ia_blksize;
+    o_iatt->ia_blocks = c_iatt->ia_blocks;
+    o_iatt->ia_atime = c_iatt->ia_atime;
+    o_iatt->ia_atime_nsec = c_iatt->ia_atime_nsec;
+    o_iatt->ia_mtime = c_iatt->ia_mtime;
+    o_iatt->ia_mtime_nsec = c_iatt->ia_mtime_nsec;
+    o_iatt->ia_ctime = c_iatt->ia_ctime;
+    o_iatt->ia_ctime_nsec = c_iatt->ia_ctime_nsec;
+
+    gf_uuid_copy(o_iatt->ia_gfid, c_iatt->ia_gfid);
+
+    return;
+}
+
+static inline void
+iatt_from_oldiatt(struct iatt *c_iatt, struct old_iatt *o_iatt)
+{
+    c_iatt->ia_dev = o_iatt->ia_dev;
+    c_iatt->ia_ino = o_iatt->ia_ino;
+    c_iatt->ia_type = o_iatt->ia_type;
+    c_iatt->ia_prot = o_iatt->ia_prot;
+    c_iatt->ia_nlink = o_iatt->ia_nlink;
+    c_iatt->ia_uid = o_iatt->ia_uid;
+    c_iatt->ia_gid = o_iatt->ia_gid;
+    c_iatt->ia_rdev = o_iatt->ia_rdev;
+    c_iatt->ia_size = o_iatt->ia_size;
+    c_iatt->ia_blksize = o_iatt->ia_blksize;
+    c_iatt->ia_blocks = o_iatt->ia_blocks;
+    c_iatt->ia_atime = o_iatt->ia_atime;
+    c_iatt->ia_atime_nsec = o_iatt->ia_atime_nsec;
+    c_iatt->ia_mtime = o_iatt->ia_mtime;
+    c_iatt->ia_mtime_nsec = o_iatt->ia_mtime_nsec;
+    c_iatt->ia_ctime = o_iatt->ia_ctime;
+    c_iatt->ia_ctime_nsec = o_iatt->ia_ctime_nsec;
+
+    gf_uuid_copy(c_iatt->ia_gfid, o_iatt->ia_gfid);
+
+    c_iatt->ia_attributes = 0;
+
+    c_iatt->ia_flags = IATT_TYPE | IATT_MODE | IATT_NLINK | IATT_INO |
+                       IATT_UID | IATT_GID | IATT_SIZE | IATT_BLOCKS |
+                       IATT_ATIME | IATT_MTIME | IATT_CTIME | IATT_GFID;
+
+    return;
+}
+
+static inline int
+is_same_mode(ia_prot_t prot1, ia_prot_t prot2)
+{
+    int ret = 0;
+
+    if (st_mode_prot_from_ia(prot1) != st_mode_prot_from_ia(prot2))
+        ret = -1;
+
+    return ret;
+}
+
+#endif /* _IATT_H */
diff --git a/libglusterfs/src/glusterfs/inode.h b/libglusterfs/src/glusterfs/inode.h
new file mode 100644
index 00000000000..4b28da510c7
--- /dev/null
+++ b/libglusterfs/src/glusterfs/inode.h
@@ -0,0 +1,306 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _INODE_H
+#define _INODE_H
+
+#include <stdint.h>
+#include <sys/types.h>
+
+#define LOOKUP_NEEDED 1
+#define LOOKUP_NOT_NEEDED 2
+
+#define DEFAULT_INODE_MEMPOOL_ENTRIES 32 * 1024
+#define INODE_PATH_FMT "<gfid:%s>"
+struct _inode_table;
+typedef struct _inode_table inode_table_t;
+
+struct _inode;
+typedef struct _inode inode_t;
+
+struct _dentry;
+typedef struct _dentry dentry_t;
+
+#include "glusterfs/list.h"
+#include "glusterfs/iatt.h"
+#include "glusterfs/compat-uuid.h"
+#include "glusterfs/fd.h"
+
+struct _inode_table {
+    pthread_mutex_t lock;
+    size_t hashsize;    /* bucket size of inode hash and dentry hash */
+    char *name;         /* name of the inode table, just for gf_log() */
+    inode_t *root;      /* root directory inode, with number 1 */
+    xlator_t *xl;       /* xlator to be called to do purge */
+    uint32_t lru_limit; /* maximum LRU cache size */
+    struct list_head *inode_hash; /* buckets for inode hash table */
+    struct list_head *name_hash;  /* buckets for dentry hash table */
+    struct list_head active; /* list of inodes currently active (in an fop) */
+    uint32_t active_size;    /* count of inodes in active list */
+    struct list_head lru;    /* list of inodes recently used.
+                                lru.next most recent */
+    uint32_t lru_size;       /* count of inodes in lru list  */
+    struct list_head purge;  /* list of inodes to be purged soon */
+    uint32_t purge_size;     /* count of inodes in purge list */
+
+    struct mem_pool *inode_pool;  /* memory pool for inodes */
+    struct mem_pool *dentry_pool; /* memory pool for dentrys */
+    struct mem_pool *fd_mem_pool; /* memory pool for fd_t */
+    int ctxcount;                 /* number of slots in inode->ctx */
+
+    /* This is required for 'invalidation' when 'nlookup' would be used,
+       specially in case of fuse-bridge */
+    int32_t (*invalidator_fn)(xlator_t *, inode_t *);
+    xlator_t *invalidator_xl;
+    struct list_head invalidate; /* inodes which are in invalidation queue */
+    uint32_t invalidate_size;    /* count of inodes in invalidation list */
+
+    /* flag to indicate whether the cleanup of the inode
+       table started or not */
+    gf_boolean_t cleanup_started;
+};
+
+struct _dentry {
+    struct list_head inode_list; /* list of dentries of inode */
+    struct list_head hash;       /* hash table pointers */
+    inode_t *inode;              /* inode of this directory entry */
+    char *name;                  /* name of the directory entry */
+    inode_t *parent;             /* directory of the entry */
+};
+
+struct _inode_ctx {
+    union {
+        uint64_t key;
+        xlator_t *xl_key;
+    };
+    /* if value1 is 0, then field is not set.. */
+    union {
+        uint64_t value1;
+        void *ptr1;
+    };
+    /* if value2 is 0, then field is not set.. */
+    union {
+        uint64_t value2;
+        void *ptr2;
+    };
+    int ref; /* This is for debugging inode ref leaks,
+                basically helps in identifying the xlator
+                causing th ref leak, it is printed in
+                statedump */
+};
+
+struct _inode {
+    inode_table_t *table; /* the table this inode belongs to */
+    uuid_t gfid;
+    gf_lock_t lock;
+    gf_atomic_t nlookup;
+    uint32_t fd_count;            /* Open fd count */
+    uint32_t active_fd_count;     /* Active open fd count */
+    uint32_t ref;                 /* reference count on this inode */
+    ia_type_t ia_type;            /* what kind of file */
+    struct list_head fd_list;     /* list of open files on this inode */
+    struct list_head dentry_list; /* list of directory entries for this inode */
+    struct list_head hash;        /* hash table pointers */
+    struct list_head list;        /* active/lru/purge */
+
+    struct _inode_ctx *_ctx; /* replacement for dict_t *(inode->ctx) */
+    bool in_invalidate_list; /* Set if inode is in table invalidate list */
+    bool invalidate_sent;    /* Set it if invalidator_fn is called for inode */
+};
+
+#define UUID0_STR "00000000-0000-0000-0000-000000000000"
+#define GFID_STR_PFX "<gfid:" UUID0_STR ">"
+#define GFID_STR_PFX_LEN (sizeof(GFID_STR_PFX) - 1)
+
+inode_table_t *
+inode_table_new(uint32_t lru_limit, xlator_t *xl);
+
+inode_table_t *
+inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
+                             int32_t (*invalidator_fn)(xlator_t *, inode_t *),
+                             xlator_t *invalidator_xl);
+
+void
+inode_table_destroy_all(glusterfs_ctx_t *ctx);
+
+void
+inode_table_destroy(inode_table_t *inode_table);
+
+inode_t *
+inode_new(inode_table_t *table);
+
+inode_t *
+inode_link(inode_t *inode, inode_t *parent, const char *name,
+           struct iatt *stbuf);
+
+void
+inode_unlink(inode_t *inode, inode_t *parent, const char *name);
+
+inode_t *
+inode_parent(inode_t *inode, uuid_t pargfid, const char *name);
+
+inode_t *
+inode_ref(inode_t *inode);
+
+inode_t *
+inode_unref(inode_t *inode);
+
+int
+inode_lookup(inode_t *inode);
+
+int
+inode_forget(inode_t *inode, uint64_t nlookup);
+int
+inode_forget_with_unref(inode_t *inode, uint64_t nlookup);
+
+int
+inode_ref_reduce_by_n(inode_t *inode, uint64_t nref);
+
+int
+inode_invalidate(inode_t *inode);
+
+int
+inode_rename(inode_table_t *table, inode_t *olddir, const char *oldname,
+             inode_t *newdir, const char *newname, inode_t *inode,
+             struct iatt *stbuf);
+
+inode_t *
+inode_grep(inode_table_t *table, inode_t *parent, const char *name);
+
+int
+inode_grep_for_gfid(inode_table_t *table, inode_t *parent, const char *name,
+                    uuid_t gfid, ia_type_t *type);
+
+inode_t *
+inode_find(inode_table_t *table, uuid_t gfid);
+
+int
+inode_path(inode_t *inode, const char *name, char **bufp);
+
+int
+__inode_path(inode_t *inode, const char *name, char **bufp);
+
+inode_t *
+inode_from_path(inode_table_t *table, const char *path);
+
+inode_t *
+inode_resolve(inode_table_t *table, char *path);
+
+/* deal with inode ctx's both values */
+
+int
+inode_ctx_set2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+               uint64_t *value2);
+int
+__inode_ctx_set2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                 uint64_t *value2);
+
+int
+inode_ctx_get2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+               uint64_t *value2);
+int
+__inode_ctx_get2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                 uint64_t *value2);
+
+int
+inode_ctx_del2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+               uint64_t *value2);
+
+int
+inode_ctx_reset2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                 uint64_t *value2);
+
+/* deal with inode ctx's 1st value */
+
+int
+inode_ctx_set0(inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+__inode_ctx_set0(inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+inode_ctx_get0(inode_t *inode, xlator_t *xlator, uint64_t *value1);
+int
+__inode_ctx_get0(inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+inode_ctx_reset0(inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+/* deal with inode ctx's 2st value */
+
+int
+inode_ctx_set1(inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+__inode_ctx_set1(inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+inode_ctx_get1(inode_t *inode, xlator_t *xlator, uint64_t *value2);
+int
+__inode_ctx_get1(inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+inode_ctx_reset1(inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+static inline int
+__inode_ctx_put(inode_t *inode, xlator_t *this, uint64_t v)
+{
+    return __inode_ctx_set0(inode, this, &v);
+}
+
+static inline int
+inode_ctx_put(inode_t *inode, xlator_t *this, uint64_t v)
+{
+    return inode_ctx_set0(inode, this, &v);
+}
+
+#define __inode_ctx_set(i, x, v_p) __inode_ctx_set0(i, x, v_p)
+
+#define inode_ctx_set(i, x, v_p) inode_ctx_set0(i, x, v_p)
+
+#define inode_ctx_reset(i, x, v) inode_ctx_reset0(i, x, v)
+
+#define __inode_ctx_get(i, x, v) __inode_ctx_get0(i, x, v)
+
+#define inode_ctx_get(i, x, v) inode_ctx_get0(i, x, v)
+
+#define inode_ctx_del(i, x, v) inode_ctx_del2(i, x, v, 0)
+#define inode_ctx_del1(i, x, v) inode_ctx_del2(i, x, 0, v)
+
+gf_boolean_t
+__is_root_gfid(uuid_t gfid);
+
+void
+__inode_table_set_lru_limit(inode_table_t *table, uint32_t lru_limit);
+
+void
+inode_table_set_lru_limit(inode_table_t *table, uint32_t lru_limit);
+
+void
+inode_ctx_merge(fd_t *fd, inode_t *inode, inode_t *linked_inode);
+
+int
+inode_is_linked(inode_t *inode);
+
+void
+inode_set_need_lookup(inode_t *inode, xlator_t *this);
+
+gf_boolean_t
+inode_needs_lookup(inode_t *inode, xlator_t *this);
+
+int
+inode_has_dentry(inode_t *inode);
+
+size_t
+inode_ctx_size(inode_t *inode);
+
+void
+inode_find_directory_name(inode_t *inode, const char **name);
+#endif /* _INODE_H */
diff --git a/libglusterfs/src/glusterfs/iobuf.h b/libglusterfs/src/glusterfs/iobuf.h
new file mode 100644
index 00000000000..4bd443efd5e
--- /dev/null
+++ b/libglusterfs/src/glusterfs/iobuf.h
@@ -0,0 +1,194 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _IOBUF_H_
+#define _IOBUF_H_
+
+#include <stddef.h>  // for size_t
+#include <sys/mman.h>
+#include "glusterfs/atomic.h"   // for gf_atomic_t
+#include <sys/uio.h>            // for struct iovec
+#include "glusterfs/locking.h"  // for gf_lock_t
+#include "glusterfs/list.h"
+
+#define GF_VARIABLE_IOBUF_COUNT 32
+
+#define GF_RDMA_DEVICE_COUNT 8
+
+/* Lets try to define the new anonymous mapping
+ * flag, in case the system is still using the
+ * now deprecated MAP_ANON flag.
+ *
+ * Also, this should ideally be in a centralized/common
+ * header which can be used by other source files also.
+ */
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#define GF_ALIGN_BUF(ptr, bound)                                               \
+    ((void *)((unsigned long)(ptr + bound - 1) & (unsigned long)(~(bound - 1))))
+
+#define GF_IOBUF_ALIGN_SIZE 512
+
+/* one allocatable unit for the consumers of the IOBUF API */
+/* each unit hosts @page_size bytes of memory */
+struct iobuf;
+
+/* one region of memory mapped from the operating system */
+/* each region MMAPs @arena_size bytes of memory */
+/* each arena hosts @arena_size / @page_size IOBUFs */
+struct iobuf_arena;
+
+/* expandable and contractable pool of memory, internally broken into arenas */
+struct iobuf_pool;
+
+struct iobuf_init_config {
+    size_t pagesize;
+    int32_t num_pages;
+};
+
+struct iobuf {
+    union {
+        struct list_head list;
+        struct {
+            struct iobuf *next;
+            struct iobuf *prev;
+        };
+    };
+    struct iobuf_arena *iobuf_arena;
+
+    gf_lock_t lock;  /* for ->ptr and ->ref */
+    gf_atomic_t ref; /* 0 == passive, >0 == active */
+
+    void *ptr; /* usable memory region by the consumer */
+
+    void *free_ptr; /* in case of stdalloc, this is the
+                       one to be freed */
+};
+
+struct iobuf_arena {
+    union {
+        struct list_head list;
+        struct {
+            struct iobuf_arena *next;
+            struct iobuf_arena *prev;
+        };
+    };
+
+    struct list_head all_list;
+    size_t page_size; /* size of all iobufs in this arena */
+    size_t arena_size;
+    /* this is equal to rounded_size * num_iobufs.
+       (rounded_size comes with gf_iobuf_get_pagesize().) */
+    size_t page_count;
+
+    struct iobuf_pool *iobuf_pool;
+
+    void *mem_base;
+    struct iobuf *iobufs; /* allocated iobufs list */
+
+    struct iobuf active;  /* head node iobuf
+                             (unused by itself) */
+    struct iobuf passive; /* head node iobuf
+                             (unused by itself) */
+    uint64_t alloc_cnt;   /* total allocs in this pool */
+    int active_cnt;
+    int passive_cnt;
+    int max_active; /* max active buffers at a given time */
+};
+
+struct iobuf_pool {
+    pthread_mutex_t mutex;
+    size_t arena_size;        /* size of memory region in
+                                 arena */
+    size_t default_page_size; /* default size of iobuf */
+
+    struct list_head all_arenas;
+    struct list_head arenas[GF_VARIABLE_IOBUF_COUNT];
+    /* array of arenas. Each element of the array is a list of arenas
+       holding iobufs of particular page_size */
+
+    struct list_head filled[GF_VARIABLE_IOBUF_COUNT];
+    /* array of arenas without free iobufs */
+
+    struct list_head purge[GF_VARIABLE_IOBUF_COUNT];
+    /* array of of arenas which can be purged */
+
+    uint64_t request_misses; /* mostly the requests for higher
+                               value of iobufs */
+    int arena_cnt;
+    int rdma_device_count;
+    struct list_head *mr_list[GF_RDMA_DEVICE_COUNT];
+    void *device[GF_RDMA_DEVICE_COUNT];
+    int (*rdma_registration)(void **, void *);
+    int (*rdma_deregistration)(struct list_head **, struct iobuf_arena *);
+};
+
+struct iobuf_pool *
+iobuf_pool_new(void);
+void
+iobuf_pool_destroy(struct iobuf_pool *iobuf_pool);
+struct iobuf *
+iobuf_get(struct iobuf_pool *iobuf_pool);
+void
+iobuf_unref(struct iobuf *iobuf);
+struct iobuf *
+iobuf_ref(struct iobuf *iobuf);
+void
+iobuf_pool_destroy(struct iobuf_pool *iobuf_pool);
+void
+iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
+
+#define iobuf_ptr(iob) ((iob)->ptr)
+#define iobpool_default_pagesize(iobpool) ((iobpool)->default_page_size)
+#define iobuf_pagesize(iob) (iob->iobuf_arena->page_size)
+
+struct iobref {
+    gf_lock_t lock;
+    gf_atomic_t ref;
+    struct iobuf **iobrefs;
+    int allocated;
+    int used;
+};
+
+struct iobref *
+iobref_new(void);
+struct iobref *
+iobref_ref(struct iobref *iobref);
+void
+iobref_unref(struct iobref *iobref);
+int
+iobref_add(struct iobref *iobref, struct iobuf *iobuf);
+int
+iobref_merge(struct iobref *to, struct iobref *from);
+void
+iobref_clear(struct iobref *iobref);
+
+size_t
+iobuf_size(struct iobuf *iobuf);
+size_t
+iobref_size(struct iobref *iobref);
+void
+iobuf_stats_dump(struct iobuf_pool *iobuf_pool);
+
+struct iobuf *
+iobuf_get2(struct iobuf_pool *iobuf_pool, size_t page_size);
+
+struct iobuf *
+iobuf_get_page_aligned(struct iobuf_pool *iobuf_pool, size_t page_size,
+                       size_t align_size);
+
+int
+iobuf_copy(struct iobuf_pool *iobuf_pool, const struct iovec *iovec_src,
+           int iovcnt, struct iobref **iobref, struct iobuf **iobuf,
+           struct iovec *iov_dst);
+
+#endif /* !_IOBUF_H_ */
diff --git a/libglusterfs/src/glusterfs/latency.h b/libglusterfs/src/glusterfs/latency.h
new file mode 100644
index 00000000000..4d601bbcbd6
--- /dev/null
+++ b/libglusterfs/src/glusterfs/latency.h
@@ -0,0 +1,33 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __LATENCY_H__
+#define __LATENCY_H__
+
+#include <inttypes.h>
+#include <time.h>
+
+typedef struct _gf_latency {
+    uint64_t min;   /* min time for the call (nanoseconds) */
+    uint64_t max;   /* max time for the call (nanoseconds) */
+    uint64_t total; /* total time (nanoseconds) */
+    uint64_t count;
+} gf_latency_t;
+
+gf_latency_t *
+gf_latency_new(size_t n);
+
+void
+gf_latency_reset(gf_latency_t *lat);
+
+void
+gf_latency_update(gf_latency_t *lat, struct timespec *begin,
+                  struct timespec *end);
+#endif /* __LATENCY_H__ */
diff --git a/libglusterfs/src/glusterfs/libglusterfs-messages.h b/libglusterfs/src/glusterfs/libglusterfs-messages.h
new file mode 100644
index 00000000000..cb31dd7614b
--- /dev/null
+++ b/libglusterfs/src/glusterfs/libglusterfs-messages.h
@@ -0,0 +1,245 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _LG_MESSAGES_H_
+#define _LG_MESSAGES_H_
+
+#include "glusterfs/glfs-message-id.h"
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    LIBGLUSTERFS, LG_MSG_ASPRINTF_FAILED, LG_MSG_INVALID_ENTRY,
+    LG_MSG_COUNT_LESS_THAN_ZERO, LG_MSG_COUNT_LESS_THAN_DATA_PAIRS,
+    LG_MSG_VALUE_LENGTH_LESS_THAN_ZERO, LG_MSG_PAIRS_LESS_THAN_COUNT,
+    LG_MSG_KEY_OR_VALUE_NULL, LG_MSG_FAILED_TO_LOG_DICT,
+    LG_MSG_NULL_VALUE_IN_DICT, LG_MSG_DIR_OP_FAILED,
+    LG_MSG_STORE_HANDLE_CREATE_FAILED, LG_MSG_FILE_OP_FAILED,
+    LG_MSG_FILE_STAT_FAILED, LG_MSG_LOCK_FAILED, LG_MSG_UNLOCK_FAILED,
+    LG_MSG_DICT_SERIAL_FAILED, LG_MSG_DICT_UNSERIAL_FAILED, LG_MSG_NO_MEMORY,
+    LG_MSG_VOLUME_ERROR, LG_MSG_SUB_VOLUME_ERROR, LG_MSG_SYNTAX_ERROR,
+    LG_MSG_BACKTICK_PARSE_FAILED, LG_MSG_BUFFER_ERROR, LG_MSG_STRDUP_ERROR,
+    LG_MSG_HASH_FUNC_ERROR, LG_MSG_GET_BUCKET_FAILED, LG_MSG_INSERT_FAILED,
+    LG_MSG_OUT_OF_RANGE, LG_MSG_VALIDATE_RETURNS, LG_MSG_VALIDATE_REC_FAILED,
+    LG_MSG_RB_TABLE_CREATE_FAILED, LG_MSG_PATH_NOT_FOUND,
+    LG_MSG_EXPAND_FD_TABLE_FAILED, LG_MSG_MAPPING_FAILED,
+    LG_MSG_INIT_IOBUF_FAILED, LG_MSG_PAGE_SIZE_EXCEEDED, LG_MSG_ARENA_NOT_FOUND,
+    LG_MSG_IOBUF_NOT_FOUND, LG_MSG_POOL_NOT_FOUND, LG_MSG_SET_ATTRIBUTE_FAILED,
+    LG_MSG_READ_ATTRIBUTE_FAILED, LG_MSG_UNMOUNT_FAILED,
+    LG_MSG_LATENCY_MEASUREMENT_STATE, LG_MSG_NO_PERM, LG_MSG_NO_KEY,
+    LG_MSG_DICT_NULL, LG_MSG_INIT_TIMER_FAILED, LG_MSG_FD_ANONYMOUS_FAILED,
+    LG_MSG_FD_CREATE_FAILED, LG_MSG_BUFFER_FULL, LG_MSG_FWRITE_FAILED,
+    LG_MSG_PRINT_FAILED, LG_MSG_MEM_POOL_DESTROY,
+    LG_MSG_EXPAND_CLIENT_TABLE_FAILED, LG_MSG_DISCONNECT_CLIENT,
+    LG_MSG_PIPE_CREATE_FAILED, LG_MSG_SET_PIPE_FAILED,
+    LG_MSG_REGISTER_PIPE_FAILED, LG_MSG_POLL_IGNORE_MULTIPLE_THREADS,
+    LG_MSG_INDEX_NOT_FOUND, LG_MSG_EPOLL_FD_CREATE_FAILED,
+    LG_MSG_SLOT_NOT_FOUND, LG_MSG_STALE_FD_FOUND, LG_MSG_GENERATION_MISMATCH,
+    LG_MSG_PTHREAD_KEY_CREATE_FAILED, LG_MSG_TRANSLATOR_INIT_FAILED,
+    LG_MSG_UUID_BUF_INIT_FAILED, LG_MSG_LKOWNER_BUF_INIT_FAILED,
+    LG_MSG_SYNCTASK_INIT_FAILED, LG_MSG_SYNCOPCTX_INIT_FAILED,
+    LG_MSG_GLOBAL_INIT_FAILED, LG_MSG_PTHREAD_FAILED, LG_MSG_DIR_IS_SYMLINK,
+    LG_MSG_RESOLVE_HOSTNAME_FAILED, LG_MSG_GETADDRINFO_FAILED,
+    LG_MSG_GETNAMEINFO_FAILED, LG_MSG_PATH_ERROR, LG_MSG_INET_PTON_FAILED,
+    LG_MSG_NEGATIVE_NUM_PASSED, LG_MSG_GETHOSTNAME_FAILED,
+    LG_MSG_RESERVED_PORTS_ERROR, LG_MSG_INVALID_PORT, LG_MSG_INVALID_FAMILY,
+    LG_MSG_CONVERSION_FAILED, LG_MSG_SKIP_HEADER_FAILED, LG_MSG_INVALID_LOG,
+    LG_MSG_UTIMES_FAILED, LG_MSG_BACKTRACE_SAVE_FAILED, LG_MSG_INIT_FAILED,
+    LG_MSG_VALIDATION_FAILED, LG_MSG_GRAPH_ERROR, LG_MSG_UNKNOWN_OPTIONS_FAILED,
+    LG_MSG_CTX_NULL, LG_MSG_TMPFILE_CREATE_FAILED, LG_MSG_DLOPEN_FAILED,
+    LG_MSG_LOAD_FAILED, LG_MSG_DLSYM_ERROR, LG_MSG_TREE_NOT_FOUND,
+    LG_MSG_PER_DENTRY, LG_MSG_DENTRY, LG_MSG_GETIFADDRS_FAILED,
+    LG_MSG_REGEX_OP_FAILED, LG_MSG_FRAME_ERROR, LG_MSG_SET_PARAM_FAILED,
+    LG_MSG_GET_PARAM_FAILED, LG_MSG_PREPARE_FAILED, LG_MSG_EXEC_FAILED,
+    LG_MSG_BINDING_FAILED, LG_MSG_DELETE_FAILED, LG_MSG_GET_ID_FAILED,
+    LG_MSG_CREATE_FAILED, LG_MSG_PARSE_FAILED, LG_MSG_GETCONTEXT_FAILED,
+    LG_MSG_UPDATE_FAILED, LG_MSG_QUERY_CALL_BACK_FAILED,
+    LG_MSG_GET_RECORD_FAILED, LG_MSG_DB_ERROR, LG_MSG_CONNECTION_ERROR,
+    LG_MSG_NOT_MULTITHREAD_MODE, LG_MSG_SKIP_PATH, LG_MSG_INVALID_FOP,
+    LG_MSG_QUERY_FAILED, LG_MSG_CLEAR_COUNTER_FAILED, LG_MSG_LOCK_LIST_FAILED,
+    LG_MSG_UNLOCK_LIST_FAILED, LG_MSG_ADD_TO_LIST_FAILED, LG_MSG_INIT_DB_FAILED,
+    LG_MSG_DELETE_FROM_LIST_FAILED, LG_MSG_CLOSE_CONNECTION_FAILED,
+    LG_MSG_INSERT_OR_UPDATE_FAILED, LG_MSG_FIND_OP_FAILED,
+    LG_MSG_CONNECTION_INIT_FAILED, LG_MSG_COMPLETED_TASK, LG_MSG_WAKE_UP_ZOMBIE,
+    LG_MSG_REWAITING_TASK, LG_MSG_SLEEP_ZOMBIE, LG_MSG_SWAPCONTEXT_FAILED,
+    LG_MSG_UNSUPPORTED_PLUGIN, LG_MSG_INVALID_DB_TYPE, LG_MSG_UNDERSIZED_BUF,
+    LG_MSG_DATA_CONVERSION_ERROR, LG_MSG_DICT_ERROR, LG_MSG_IOBUFS_NOT_FOUND,
+    LG_MSG_ENTRIES_NULL, LG_MSG_FD_NOT_FOUND_IN_FDTABLE,
+    LG_MSG_REALLOC_FOR_FD_PTR_FAILED, LG_MSG_DICT_SET_FAILED, LG_MSG_NULL_PTR,
+    LG_MSG_RBTHASH_INIT_BUCKET_FAILED, LG_MSG_ASSERTION_FAILED,
+    LG_MSG_HOSTNAME_NULL, LG_MSG_INVALID_IPV4_FORMAT,
+    LG_MSG_CTX_CLEANUP_STARTED, LG_MSG_TIMER_REGISTER_ERROR,
+    LG_MSG_PTR_HEADER_CORRUPTED, LG_MSG_INVALID_UPLINK, LG_MSG_CLIENT_NULL,
+    LG_MSG_XLATOR_DOES_NOT_IMPLEMENT, LG_MSG_DENTRY_NOT_FOUND,
+    LG_MSG_INODE_NOT_FOUND, LG_MSG_INODE_TABLE_NOT_FOUND,
+    LG_MSG_DENTRY_CREATE_FAILED, LG_MSG_INODE_CONTEXT_FREED,
+    LG_MSG_UNKNOWN_LOCK_TYPE, LG_MSG_UNLOCK_BEFORE_LOCK,
+    LG_MSG_LOCK_OWNER_ERROR, LG_MSG_MEMPOOL_PTR_NULL,
+    LG_MSG_QUOTA_XATTRS_MISSING, LG_MSG_INVALID_STRING, LG_MSG_BIND_REF,
+    LG_MSG_REF_COUNT, LG_MSG_INVALID_ARG, LG_MSG_VOL_OPTION_ADD,
+    LG_MSG_XLATOR_OPTION_INVALID, LG_MSG_GETTIMEOFDAY_FAILED,
+    LG_MSG_GRAPH_INIT_FAILED, LG_MSG_EVENT_NOTIFY_FAILED,
+    LG_MSG_ACTIVE_GRAPH_NULL, LG_MSG_VOLFILE_PARSE_ERROR, LG_MSG_FD_INODE_NULL,
+    LG_MSG_INVALID_VOLFILE_ENTRY, LG_MSG_PER_DENTRY_FAILED,
+    LG_MSG_PARENT_DENTRY_NOT_FOUND, LG_MSG_DENTRY_CYCLIC_LOOP,
+    LG_MSG_INVALID_POLL_IN, LG_MSG_INVALID_POLL_OUT, LG_MSG_EPOLL_FD_ADD_FAILED,
+    LG_MSG_EPOLL_FD_DEL_FAILED, LG_MSG_EPOLL_FD_MODIFY_FAILED,
+    LG_MSG_STARTED_EPOLL_THREAD, LG_MSG_EXITED_EPOLL_THREAD,
+    LG_MSG_START_EPOLL_THREAD_FAILED, LG_MSG_FALLBACK_TO_POLL,
+    LG_MSG_QUOTA_CONF_ERROR, LG_MSG_RBTHASH_GET_ENTRY_FAILED,
+    LG_MSG_RBTHASH_GET_BUCKET_FAILED, LG_MSG_RBTHASH_INSERT_FAILED,
+    LG_MSG_RBTHASH_INIT_ENTRY_FAILED, LG_MSG_TMPFILE_DELETE_FAILED,
+    LG_MSG_MEMPOOL_INVALID_FREE, LG_MSG_LOCK_FAILURE, LG_MSG_SET_LOG_LEVEL,
+    LG_MSG_VERIFY_PLATFORM, LG_MSG_RUNNER_LOG, LG_MSG_LEASEID_BUF_INIT_FAILED,
+    LG_MSG_PTHREAD_ATTR_INIT_FAILED, LG_MSG_INVALID_INODE_LIST,
+    LG_MSG_COMPACT_FAILED, LG_MSG_COMPACT_STATUS, LG_MSG_UTIMENSAT_FAILED,
+    LG_MSG_PTHREAD_NAMING_FAILED, LG_MSG_SYSCALL_RETURNS_WRONG,
+    LG_MSG_XXH64_TO_GFID_FAILED, LG_MSG_ASYNC_WARNING, LG_MSG_ASYNC_FAILURE,
+    LG_MSG_GRAPH_CLEANUP_FAILED, LG_MSG_GRAPH_SETUP_FAILED,
+    LG_MSG_GRAPH_DETACH_STARTED, LG_MSG_GRAPH_ATTACH_FAILED,
+    LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, LG_MSG_DUPLICATE_ENTRY,
+    LG_MSG_THREAD_NAME_TOO_LONG, LG_MSG_SET_THREAD_FAILED,
+    LG_MSG_THREAD_CREATE_FAILED, LG_MSG_FILE_DELETE_FAILED, LG_MSG_WRONG_VALUE,
+    LG_MSG_PATH_OPEN_FAILED, LG_MSG_DISPATCH_HANDLER_FAILED,
+    LG_MSG_READ_FILE_FAILED, LG_MSG_ENTRIES_NOT_PROVIDED,
+    LG_MSG_ENTRIES_PROVIDED, LG_MSG_UNKNOWN_OPTION_TYPE,
+    LG_MSG_OPTION_DEPRECATED, LG_MSG_INVALID_INIT, LG_MSG_OBJECT_NULL,
+    LG_MSG_GRAPH_NOT_SET, LG_MSG_FILENAME_NOT_SPECIFIED, LG_MSG_STRUCT_MISS,
+    LG_MSG_METHOD_MISS, LG_MSG_INPUT_DATA_NULL, LG_MSG_OPEN_LOGFILE_FAILED);
+
+#define LG_MSG_EPOLL_FD_CREATE_FAILED_STR "epoll fd creation failed"
+#define LG_MSG_INVALID_POLL_IN_STR "invalid poll_in value"
+#define LG_MSG_INVALID_POLL_OUT_STR "invalid poll_out value"
+#define LG_MSG_SLOT_NOT_FOUND_STR "could not find slot"
+#define LG_MSG_EPOLL_FD_ADD_FAILED_STR "failed to add fd to epoll"
+#define LG_MSG_EPOLL_FD_DEL_FAILED_STR "fail to delete fd to epoll"
+#define LG_MSG_EPOLL_FD_MODIFY_FAILED_STR "failed to modify fd events"
+#define LG_MSG_STALE_FD_FOUND_STR "stale fd found"
+#define LG_MSG_GENERATION_MISMATCH_STR "generation mismatch"
+#define LG_MSG_STARTED_EPOLL_THREAD_STR "Started thread with index"
+#define LG_MSG_EXITED_EPOLL_THREAD_STR "Exited thread"
+#define LG_MSG_DISPATCH_HANDLER_FAILED_STR "Failed to dispatch handler"
+#define LG_MSG_START_EPOLL_THREAD_FAILED_STR "Failed to start thread"
+#define LG_MSG_PIPE_CREATE_FAILED_STR "pipe creation failed"
+#define LG_MSG_SET_PIPE_FAILED_STR "could not set pipe to non blocking mode"
+#define LG_MSG_REGISTER_PIPE_FAILED_STR                                        \
+    "could not register pipe fd with poll event loop"
+#define LG_MSG_POLL_IGNORE_MULTIPLE_THREADS_STR                                \
+    "Currently poll does not use multiple event processing threads, count "    \
+    "ignored"
+#define LG_MSG_INDEX_NOT_FOUND_STR "index not found"
+#define LG_MSG_READ_FILE_FAILED_STR "read on file returned error"
+#define LG_MSG_RB_TABLE_CREATE_FAILED_STR "Failed to create rb table bucket"
+#define LG_MSG_HASH_FUNC_ERROR_STR "Hash function not given"
+#define LG_MSG_ENTRIES_NOT_PROVIDED_STR                                        \
+    "Both mem-pool and expected entries not provided"
+#define LG_MSG_ENTRIES_PROVIDED_STR                                            \
+    "Both mem-pool and expected entries are provided"
+#define LG_MSG_RBTHASH_INIT_BUCKET_FAILED_STR "failed to init buckets"
+#define LG_MSG_RBTHASH_GET_ENTRY_FAILED_STR "Failed to get entry from mem-pool"
+#define LG_MSG_RBTHASH_GET_BUCKET_FAILED_STR "Failed to get bucket"
+#define LG_MSG_RBTHASH_INSERT_FAILED_STR "Failed to insert entry"
+#define LG_MSG_RBTHASH_INIT_ENTRY_FAILED_STR "Failed to init entry"
+#define LG_MSG_FILE_STAT_FAILED_STR "failed to stat"
+#define LG_MSG_INET_PTON_FAILED_STR "inet_pton() failed"
+#define LG_MSG_INVALID_ENTRY_STR "Invalid arguments"
+#define LG_MSG_NEGATIVE_NUM_PASSED_STR "negative number passed"
+#define LG_MSG_PATH_ERROR_STR "Path manipulation failed"
+#define LG_MSG_FILE_OP_FAILED_STR "could not open/read file, getting ports info"
+#define LG_MSG_RESERVED_PORTS_ERROR_STR                                        \
+    "Not able to get reserved ports, hence there is a possibility that "       \
+    "glusterfs may consume reserved port"
+#define LG_MSG_INVALID_PORT_STR "invalid port"
+#define LG_MSG_GETNAMEINFO_FAILED_STR "Could not lookup hostname"
+#define LG_MSG_GETIFADDRS_FAILED_STR "getifaddrs() failed"
+#define LG_MSG_INVALID_FAMILY_STR "Invalid family"
+#define LG_MSG_CONVERSION_FAILED_STR "String conversion failed"
+#define LG_MSG_GETADDRINFO_FAILED_STR "error in getaddrinfo"
+#define LG_MSG_DUPLICATE_ENTRY_STR "duplicate entry for volfile-server"
+#define LG_MSG_PTHREAD_NAMING_FAILED_STR "Failed to compose thread name"
+#define LG_MSG_THREAD_NAME_TOO_LONG_STR                                        \
+    "Thread name is too long. It has been truncated"
+#define LG_MSG_SET_THREAD_FAILED_STR "Could not set thread name"
+#define LG_MSG_THREAD_CREATE_FAILED_STR "Thread creation failed"
+#define LG_MSG_PTHREAD_ATTR_INIT_FAILED_STR                                    \
+    "Thread attribute initialization failed"
+#define LG_MSG_SKIP_HEADER_FAILED_STR "Failed to skip header section"
+#define LG_MSG_INVALID_LOG_STR "Invalid log-format"
+#define LG_MSG_UTIMENSAT_FAILED_STR "utimenstat failed"
+#define LG_MSG_UTIMES_FAILED_STR "utimes failed"
+#define LG_MSG_FILE_DELETE_FAILED_STR "Unable to delete file"
+#define LG_MSG_BACKTRACE_SAVE_FAILED_STR "Failed to save the backtrace"
+#define LG_MSG_WRONG_VALUE_STR "wrong value"
+#define LG_MSG_DIR_OP_FAILED_STR "Failed to create directory"
+#define LG_MSG_DIR_IS_SYMLINK_STR "dir is symlink"
+#define LG_MSG_RESOLVE_HOSTNAME_FAILED_STR "couldnot resolve hostname"
+#define LG_MSG_PATH_OPEN_FAILED_STR "Unable to open path"
+#define LG_MSG_NO_MEMORY_STR "Error allocating memory"
+#define LG_MSG_EVENT_NOTIFY_FAILED_STR "notification failed"
+#define LG_MSG_PER_DENTRY_FAILED_STR "per dentry fn returned"
+#define LG_MSG_PARENT_DENTRY_NOT_FOUND_STR "parent not found"
+#define LG_MSG_DENTRY_CYCLIC_LOOP_STR                                          \
+    "detected cyclic loop formation during inode linkage"
+#define LG_MSG_CTX_NULL_STR "_ctx not found"
+#define LG_MSG_DENTRY_NOT_FOUND_STR "dentry not found"
+#define LG_MSG_OUT_OF_RANGE_STR "out of range"
+#define LG_MSG_UNKNOWN_OPTION_TYPE_STR "unknown option type"
+#define LG_MSG_VALIDATE_RETURNS_STR "validate of returned"
+#define LG_MSG_OPTION_DEPRECATED_STR                                           \
+    "option is deprecated, continuing with correction"
+#define LG_MSG_VALIDATE_REC_FAILED_STR "validate_rec failed"
+#define LG_MSG_MAPPING_FAILED_STR "mapping failed"
+#define LG_MSG_INIT_IOBUF_FAILED_STR "init failed"
+#define LG_MSG_ARENA_NOT_FOUND_STR "arena not found"
+#define LG_MSG_PAGE_SIZE_EXCEEDED_STR                                          \
+    "page_size of iobufs in arena being added is greater than max available"
+#define LG_MSG_POOL_NOT_FOUND_STR "pool not found"
+#define LG_MSG_IOBUF_NOT_FOUND_STR "iobuf not found"
+#define LG_MSG_DLOPEN_FAILED_STR "DL open failed"
+#define LG_MSG_DLSYM_ERROR_STR "dlsym missing"
+#define LG_MSG_LOAD_FAILED_STR "Failed to load xlator options table"
+#define LG_MSG_INPUT_DATA_NULL_STR                                             \
+    "input data is null. cannot update the lru limit of the inode table. "     \
+    "continuing with older value."
+#define LG_MSG_INIT_FAILED_STR "No init() found"
+#define LG_MSG_VOLUME_ERROR_STR                                                \
+    "Initialization of volume failed. review your volfile again."
+#define LG_MSG_TREE_NOT_FOUND_STR "Translator tree not found"
+#define LG_MSG_SET_LOG_LEVEL_STR "setting log level"
+#define LG_MSG_INVALID_INIT_STR                                                \
+    "Invalid log-level. possible values are DEBUG|WARNING|ERROR|NONE|TRACE"
+#define LG_MSG_OBJECT_NULL_STR "object is null, returning false."
+#define LG_MSG_GRAPH_NOT_SET_STR "Graph is not set for xlator"
+#define LG_MSG_OPEN_LOGFILE_FAILED_STR "failed to open logfile"
+#define LG_MSG_STRDUP_ERROR_STR "failed to create metrics dir"
+#define LG_MSG_FILENAME_NOT_SPECIFIED_STR "no filename specified"
+#define LG_MSG_UNDERSIZED_BUF_STR "data value is smaller than expected"
+#define LG_MSG_DICT_SET_FAILED_STR "unable to set dict"
+#define LG_MSG_COUNT_LESS_THAN_ZERO_STR "count < 0!"
+#define LG_MSG_PAIRS_LESS_THAN_COUNT_STR "less than count data pairs found"
+#define LG_MSG_NULL_PTR_STR "pair->key is null!"
+#define LG_MSG_VALUE_LENGTH_LESS_THAN_ZERO_STR "value->len < 0"
+#define LG_MSG_INVALID_ARG_STR "buf is null"
+#define LG_MSG_KEY_OR_VALUE_NULL_STR "key or value is null"
+#define LG_MSG_NULL_VALUE_IN_DICT_STR "null value found in dict"
+#define LG_MSG_FAILED_TO_LOG_DICT_STR "Failed to log dictionary"
+#define LG_MSG_DICT_ERROR_STR "dict error"
+#define LG_MSG_STRUCT_MISS_STR "struct missing"
+#define LG_MSG_METHOD_MISS_STR "method missing(init)"
+
+#endif /* !_LG_MESSAGES_H_ */
diff --git a/libglusterfs/src/glusterfs/list.h b/libglusterfs/src/glusterfs/list.h
new file mode 100644
index 00000000000..221a710ca30
--- /dev/null
+++ b/libglusterfs/src/glusterfs/list.h
@@ -0,0 +1,273 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _LLIST_H
+#define _LLIST_H
+
+struct list_head {
+    struct list_head *next;
+    struct list_head *prev;
+};
+
+#define INIT_LIST_HEAD(head)                                                   \
+    do {                                                                       \
+        (head)->next = (head)->prev = head;                                    \
+    } while (0)
+
+static inline void
+list_add(struct list_head *new, struct list_head *head)
+{
+    new->prev = head;
+    new->next = head->next;
+
+    new->prev->next = new;
+    new->next->prev = new;
+}
+
+static inline void
+list_add_tail(struct list_head *new, struct list_head *head)
+{
+    new->next = head;
+    new->prev = head->prev;
+
+    new->prev->next = new;
+    new->next->prev = new;
+}
+
+/* This function will insert the element to the list in a order.
+   Order will be based on the compare function provided as a input.
+   If element to be inserted in ascending order compare should return:
+    0: if both the arguments are equal
+   >0: if first argument is greater than second argument
+   <0: if first argument is less than second argument */
+static inline void
+list_add_order(struct list_head *new, struct list_head *head,
+               int (*compare)(struct list_head *, struct list_head *))
+{
+    struct list_head *pos = head->prev;
+
+    while (pos != head) {
+        if (compare(new, pos) >= 0)
+            break;
+
+        /* Iterate the list in the reverse order. This will have
+           better efficiency if the elements are inserted in the
+           ascending order */
+        pos = pos->prev;
+    }
+
+    list_add(new, pos);
+}
+
+static inline void
+list_del(struct list_head *old)
+{
+    old->prev->next = old->next;
+    old->next->prev = old->prev;
+
+    old->next = (void *)0xbabebabe;
+    old->prev = (void *)0xcafecafe;
+}
+
+static inline void
+list_del_init(struct list_head *old)
+{
+    old->prev->next = old->next;
+    old->next->prev = old->prev;
+
+    old->next = old;
+    old->prev = old;
+}
+
+static inline void
+list_move(struct list_head *list, struct list_head *head)
+{
+    list_del(list);
+    list_add(list, head);
+}
+
+static inline void
+list_move_tail(struct list_head *list, struct list_head *head)
+{
+    list_del(list);
+    list_add_tail(list, head);
+}
+
+static inline int
+list_empty(struct list_head *head)
+{
+    return (head->next == head);
+}
+
+static inline void
+__list_splice(struct list_head *list, struct list_head *head)
+{
+    (list->prev)->next = (head->next);
+    (head->next)->prev = (list->prev);
+
+    (head)->next = (list->next);
+    (list->next)->prev = (head);
+}
+
+static inline void
+list_splice(struct list_head *list, struct list_head *head)
+{
+    if (list_empty(list))
+        return;
+
+    __list_splice(list, head);
+}
+
+/* Splice moves @list to the head of the list at @head. */
+static inline void
+list_splice_init(struct list_head *list, struct list_head *head)
+{
+    if (list_empty(list))
+        return;
+
+    __list_splice(list, head);
+    INIT_LIST_HEAD(list);
+}
+
+static inline void
+__list_append(struct list_head *list, struct list_head *head)
+{
+    (head->prev)->next = (list->next);
+    (list->next)->prev = (head->prev);
+    (head->prev) = (list->prev);
+    (list->prev)->next = head;
+}
+
+static inline void
+list_append(struct list_head *list, struct list_head *head)
+{
+    if (list_empty(list))
+        return;
+
+    __list_append(list, head);
+}
+
+/* Append moves @list to the end of @head */
+static inline void
+list_append_init(struct list_head *list, struct list_head *head)
+{
+    if (list_empty(list))
+        return;
+
+    __list_append(list, head);
+    INIT_LIST_HEAD(list);
+}
+
+static inline int
+list_is_last(struct list_head *list, struct list_head *head)
+{
+    return (list->next == head);
+}
+
+static inline int
+list_is_singular(struct list_head *head)
+{
+    return !list_empty(head) && (head->next == head->prev);
+}
+
+/**
+ * list_replace - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ *
+ * If @old was empty, it will be overwritten.
+ */
+static inline void
+list_replace(struct list_head *old, struct list_head *new)
+{
+    new->next = old->next;
+    new->next->prev = new;
+    new->prev = old->prev;
+    new->prev->next = new;
+}
+
+static inline void
+list_replace_init(struct list_head *old, struct list_head *new)
+{
+    list_replace(old, new);
+    INIT_LIST_HEAD(old);
+}
+
+/**
+ * list_rotate_left - rotate the list to the left
+ * @head: the head of the list
+ */
+static inline void
+list_rotate_left(struct list_head *head)
+{
+    struct list_head *first;
+
+    if (!list_empty(head)) {
+        first = head->next;
+        list_move_tail(first, head);
+    }
+}
+
+#define list_entry(ptr, type, member)                                          \
+    ((type *)((char *)(ptr) - (unsigned long)(&((type *)0)->member)))
+
+#define list_first_entry(ptr, type, member)                                    \
+    list_entry((ptr)->next, type, member)
+
+#define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member)
+
+#define list_next_entry(pos, member)                                           \
+    list_entry((pos)->member.next, typeof(*(pos)), member)
+
+#define list_prev_entry(pos, member)                                           \
+    list_entry((pos)->member.prev, typeof(*(pos)), member)
+
+#define list_for_each(pos, head)                                               \
+    for (pos = (head)->next; pos != (head); pos = pos->next)
+
+#define list_for_each_entry(pos, head, member)                                 \
+    for (pos = list_entry((head)->next, typeof(*pos), member);                 \
+         &pos->member != (head);                                               \
+         pos = list_entry(pos->member.next, typeof(*pos), member))
+
+#define list_for_each_entry_safe(pos, n, head, member)                         \
+    for (pos = list_entry((head)->next, typeof(*pos), member),                 \
+        n = list_entry(pos->member.next, typeof(*pos), member);                \
+         &pos->member != (head);                                               \
+         pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+#define list_for_each_entry_reverse(pos, head, member)                         \
+    for (pos = list_entry((head)->prev, typeof(*pos), member);                 \
+         &pos->member != (head);                                               \
+         pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+#define list_for_each_entry_safe_reverse(pos, n, head, member)                 \
+    for (pos = list_entry((head)->prev, typeof(*pos), member),                 \
+        n = list_entry(pos->member.prev, typeof(*pos), member);                \
+         &pos->member != (head);                                               \
+         pos = n, n = list_entry(n->member.prev, typeof(*n), member))
+
+/*
+ * This list implementation has some advantages, but one disadvantage: you
+ * can't use NULL to check whether you're at the head or tail.  Thus, the
+ * address of the head has to be an argument for these macros.
+ */
+
+#define list_next(ptr, head, type, member)                                     \
+    (((ptr)->member.next == head)                                              \
+         ? NULL                                                                \
+         : list_entry((ptr)->member.next, type, member))
+
+#define list_prev(ptr, head, type, member)                                     \
+    (((ptr)->member.prev == head)                                              \
+         ? NULL                                                                \
+         : list_entry((ptr)->member.prev, type, member))
+
+#endif /* _LLIST_H */
diff --git a/libglusterfs/src/glusterfs/lkowner.h b/libglusterfs/src/glusterfs/lkowner.h
new file mode 100644
index 00000000000..692de34bc7a
--- /dev/null
+++ b/libglusterfs/src/glusterfs/lkowner.h
@@ -0,0 +1,93 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _LK_OWNER_H
+#define _LK_OWNER_H
+
+#include "glusterfs/glusterfs-fops.h"
+
+/* LKOWNER to string functions */
+static inline void
+lkowner_unparse(gf_lkowner_t *lkowner, char *buf, int buf_len)
+{
+    int i = 0;
+    int j = 0;
+
+    for (i = 0; i < lkowner->len; i++) {
+        if (i && !(i % 8)) {
+            buf[j] = '-';
+            j++;
+        }
+        sprintf(&buf[j], "%02hhx", lkowner->data[i]);
+        j += 2;
+        if (j == buf_len)
+            break;
+    }
+    if (j < buf_len)
+        buf[j] = '\0';
+}
+
+static inline void
+set_lk_owner_from_ptr(gf_lkowner_t *lkowner, void *data)
+{
+    int i = 0;
+    int j = 0;
+
+    lkowner->len = sizeof(unsigned long);
+    for (i = 0, j = 0; i < lkowner->len; i++, j += 8) {
+        lkowner->data[i] = (char)((((unsigned long)data) >> j) & 0xff);
+    }
+}
+
+static inline void
+set_lk_owner_from_uint64(gf_lkowner_t *lkowner, uint64_t data)
+{
+    int i = 0;
+    int j = 0;
+
+    lkowner->len = 8;
+    for (i = 0, j = 0; i < lkowner->len; i++, j += 8) {
+        lkowner->data[i] = (char)((data >> j) & 0xff);
+    }
+}
+
+/* Return true if the locks have the same owner */
+static inline int
+is_same_lkowner(gf_lkowner_t *l1, gf_lkowner_t *l2)
+{
+    return ((l1->len == l2->len) && !memcmp(l1->data, l2->data, l1->len));
+}
+
+static inline int
+is_lk_owner_null(gf_lkowner_t *lkowner)
+{
+    int is_null = 1;
+    int i = 0;
+
+    if (lkowner == NULL || lkowner->len == 0)
+        goto out;
+
+    for (i = 0; i < lkowner->len; i++) {
+        if (lkowner->data[i] != 0) {
+            is_null = 0;
+            break;
+        }
+    }
+out:
+    return is_null;
+}
+
+static inline void
+lk_owner_copy(gf_lkowner_t *dst, gf_lkowner_t *src)
+{
+    dst->len = src->len;
+    memcpy(dst->data, src->data, src->len);
+}
+#endif /* _LK_OWNER_H */
diff --git a/libglusterfs/src/glusterfs/locking.h b/libglusterfs/src/glusterfs/locking.h
new file mode 100644
index 00000000000..43cc87735d1
--- /dev/null
+++ b/libglusterfs/src/glusterfs/locking.h
@@ -0,0 +1,84 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _LOCKING_H
+#define _LOCKING_H
+
+#include <pthread.h>
+
+#if defined(GF_DARWIN_HOST_OS)
+#include <libkern/OSAtomic.h>
+#define pthread_spinlock_t OSSpinLock
+#define pthread_spin_lock(l) OSSpinLockLock(l)
+#define pthread_spin_unlock(l) OSSpinLockUnlock(l)
+#define pthread_spin_destroy(l) 0
+#define pthread_spin_init(l, v) (*l = v)
+#endif
+
+#if defined(HAVE_SPINLOCK)
+
+typedef union {
+    pthread_spinlock_t spinlock;
+    pthread_mutex_t mutex;
+} gf_lock_t;
+
+#if !defined(LOCKING_IMPL)
+extern int use_spinlocks;
+
+/*
+ * Using a dispatch table would be unpleasant because we're dealing with two
+ * different types.  If the dispatch contains direct pointers to pthread_xx
+ * or mutex_xxx then we have to hope that every possible union alternative
+ * starts at the same address as the union itself.  I'm old enough to remember
+ * compilers where this was not the case (for alignment reasons) so I'm a bit
+ * paranoid about that.  Also, I don't like casting arguments through "void *"
+ * which we'd also have to do to avoid type errors.  The other alternative would
+ * be to define actual functions which pick out the right union member, and put
+ * those in the dispatch tables.  Now we have a pointer dereference through the
+ * dispatch table plus a function call, which is likely to be worse than the
+ * branching here from the ?: construct.  If it were a clear win it might be
+ * worth the extra complexity, but for now this way seems preferable.
+ */
+
+#define LOCK_INIT(x)                                                           \
+    (use_spinlocks ? pthread_spin_init(&((x)->spinlock), 0)                    \
+                   : pthread_mutex_init(&((x)->mutex), 0))
+
+#define LOCK(x)                                                                \
+    (use_spinlocks ? pthread_spin_lock(&((x)->spinlock))                       \
+                   : pthread_mutex_lock(&((x)->mutex)))
+
+#define TRY_LOCK(x)                                                            \
+    (use_spinlocks ? pthread_spin_trylock(&((x)->spinlock))                    \
+                   : pthread_mutex_trylock(&((x)->mutex)))
+
+#define UNLOCK(x)                                                              \
+    (use_spinlocks ? pthread_spin_unlock(&((x)->spinlock))                     \
+                   : pthread_mutex_unlock(&((x)->mutex)))
+
+#define LOCK_DESTROY(x)                                                        \
+    (use_spinlocks ? pthread_spin_destroy(&((x)->spinlock))                    \
+                   : pthread_mutex_destroy(&((x)->mutex)))
+
+#endif
+
+#else
+
+typedef pthread_mutex_t gf_lock_t;
+
+#define LOCK_INIT(x) pthread_mutex_init(x, 0)
+#define LOCK(x) pthread_mutex_lock(x)
+#define TRY_LOCK(x) pthread_mutex_trylock(x)
+#define UNLOCK(x) pthread_mutex_unlock(x)
+#define LOCK_DESTROY(x) pthread_mutex_destroy(x)
+
+#endif /* HAVE_SPINLOCK */
+
+#endif /* _LOCKING_H */
diff --git a/libglusterfs/src/glusterfs/logging.h b/libglusterfs/src/glusterfs/logging.h
new file mode 100644
index 00000000000..b3a6ac191f0
--- /dev/null
+++ b/libglusterfs/src/glusterfs/logging.h
@@ -0,0 +1,383 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __LOGGING_H__
+#define __LOGGING_H__
+
+#include <sys/time.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <pthread.h>
+#include "glusterfs/list.h"
+
+#ifdef GF_DARWIN_HOST_OS
+#define GF_PRI_FSBLK "u"
+#define GF_PRI_DEV PRId32
+#define GF_PRI_INODE PRIu64
+#define GF_PRI_NLINK PRIu32
+#define GF_PRI_SECOND "ld"
+#define GF_PRI_SUSECONDS "06d"
+#define GF_PRI_SNSECONDS "09ld"
+#define GF_PRI_USEC "d"
+#else
+#define GF_PRI_FSBLK PRIu64
+#define GF_PRI_DEV PRIu64
+#define GF_PRI_INODE PRIu64
+#define GF_PRI_NLINK PRIu32
+#define GF_PRI_SECOND "lu"
+#define GF_PRI_SUSECONDS "06ld"
+#define GF_PRI_SNSECONDS "09ld"
+#define GF_PRI_USEC "ld"
+#endif
+#define GF_PRI_BLKSIZE PRId32
+#define GF_PRI_SIZET "zu"
+#define GF_PRI_ATOMIC PRIu64
+
+#ifdef GF_DARWIN_HOST_OS
+#define GF_PRI_TIME "ld"
+#else
+#define GF_PRI_TIME PRIu64
+#endif
+
+#if 0
+/* Syslog definitions :-) */
+#define LOG_EMERG 0   /* system is unusable */
+#define LOG_ALERT 1   /* action must be taken immediately */
+#define LOG_CRIT 2    /* critical conditions */
+#define LOG_ERR 3     /* error conditions */
+#define LOG_WARNING 4 /* warning conditions */
+#define LOG_NOTICE 5  /* normal but significant condition */
+#define LOG_INFO 6    /* informational */
+#define LOG_DEBUG 7   /* debug-level messages */
+#endif
+
+#define GF_LOG_FORMAT_NO_MSG_ID "no-msg-id"
+#define GF_LOG_FORMAT_WITH_MSG_ID "with-msg-id"
+
+#define GF_LOGGER_GLUSTER_LOG "gluster-log"
+#define GF_LOGGER_SYSLOG "syslog"
+
+typedef enum {
+    GF_LOG_NONE,
+    GF_LOG_EMERG,
+    GF_LOG_ALERT,
+    GF_LOG_CRITICAL, /* fatal errors */
+    GF_LOG_ERROR,    /* major failures (not necessarily fatal) */
+    GF_LOG_WARNING,  /* info about normal operation */
+    GF_LOG_NOTICE,
+    GF_LOG_INFO,  /* Normal information */
+    GF_LOG_DEBUG, /* internal errors */
+    GF_LOG_TRACE, /* full trace of operation */
+} gf_loglevel_t;
+
+/* format for the logs */
+typedef enum {
+    gf_logformat_traditional = 0, /* Format as in gluster 3.5 */
+    gf_logformat_withmsgid, /* Format enhanced with MsgID, ident, errstr */
+    gf_logformat_cee        /* log enhanced format in cee */
+} gf_log_format_t;
+
+/* log infrastructure to log to */
+typedef enum {
+    gf_logger_glusterlog = 0, /* locations and files as in gluster 3.5 */
+    gf_logger_syslog          /* log to (r)syslog, based on (r)syslog conf */
+    /* NOTE: In the future journald, lumberjack, next new thing here */
+} gf_log_logger_t;
+
+#define DEFAULT_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
+#define DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY DATADIR "/log/glusterfs/quota_crawl"
+#define DEFAULT_LOG_LEVEL GF_LOG_INFO
+
+typedef struct gf_log_handle_ {
+    pthread_mutex_t logfile_mutex;
+    gf_loglevel_t loglevel;
+    gf_loglevel_t sys_log_level;
+    int gf_log_syslog;
+    char *filename;
+    FILE *logfile;
+    FILE *gf_log_logfile;
+    char *cmd_log_filename;
+    FILE *cmdlogfile;
+    gf_log_logger_t logger;
+    gf_log_format_t logformat;
+    char *ident;
+    int log_control_file_found;
+    struct list_head lru_queue;
+    pthread_mutex_t log_buf_lock;
+    struct _gf_timer *log_flush_timer;
+    int localtime;
+    uint32_t lru_size;
+    uint32_t lru_cur_size;
+    uint32_t timeout;
+    uint8_t logrotate;
+    uint8_t cmd_history_logrotate;
+} gf_log_handle_t;
+
+typedef struct log_buf_ {
+    char *msg;
+    uint64_t msg_id;
+    int errnum;
+    struct timeval oldest;
+    struct timeval latest;
+    char *domain;
+    char *file;
+    char *function;
+    int32_t line;
+    gf_loglevel_t level;
+    int refcount;
+    int graph_id;
+    struct list_head msg_list;
+} log_buf_t;
+
+void
+gf_log_globals_init(void *ctx, gf_loglevel_t level);
+int
+gf_log_init(void *data, const char *filename, const char *ident);
+
+void
+gf_log_logrotate(int signum);
+
+void
+gf_log_cleanup(void);
+
+/* Internal interfaces to log messages with message IDs */
+int
+_gf_msg(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, int errnum, int trace,
+        uint64_t msgid, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 9, 10)));
+
+void
+_gf_msg_backtrace_nomem(gf_loglevel_t level, int stacksize);
+
+int
+_gf_msg_plain(gf_loglevel_t level, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+int
+_gf_msg_plain_nomem(gf_loglevel_t level, const char *msg);
+
+int
+_gf_msg_vplain(gf_loglevel_t level, const char *fmt, va_list ap);
+
+int
+_gf_msg_nomem(const char *domain, const char *file, const char *function,
+              int line, gf_loglevel_t level, size_t size);
+
+int
+_gf_log(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 6, 7)));
+
+int
+_gf_log_callingfn(const char *domain, const char *file, const char *function,
+                  int32_t line, gf_loglevel_t level, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 6, 7)));
+
+int
+_gf_log_eh(const char *function, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+/* treat GF_LOG_TRACE and GF_LOG_NONE as LOG_DEBUG and
+ * other level as is */
+#define SET_LOG_PRIO(level, priority)                                          \
+    do {                                                                       \
+        if (GF_LOG_TRACE == (level) || GF_LOG_NONE == (level)) {               \
+            priority = LOG_DEBUG;                                              \
+        } else {                                                               \
+            priority = (level)-1;                                              \
+        }                                                                      \
+    } while (0)
+
+/* extract just the file name from the path */
+#define GET_FILE_NAME_TO_LOG(file, basename)                                   \
+    do {                                                                       \
+        basename = strrchr((file), '/');                                       \
+        if (basename)                                                          \
+            basename++;                                                        \
+        else                                                                   \
+            basename = (file);                                                 \
+    } while (0)
+
+#define PRINT_SIZE_CHECK(ret, label, strsize)                                  \
+    do {                                                                       \
+        if (ret < 0)                                                           \
+            goto label;                                                        \
+        if ((strsize - ret) > 0) {                                             \
+            strsize -= ret;                                                    \
+        } else {                                                               \
+            ret = 0;                                                           \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define FMT_WARN(fmt...)                                                       \
+    do {                                                                       \
+        if (0)                                                                 \
+            printf(fmt);                                                       \
+    } while (0)
+
+/* Interface to log messages with message IDs */
+#define gf_msg(dom, level, errnum, msgid, fmt...)                              \
+    do {                                                                       \
+        _gf_msg(dom, __FILE__, __FUNCTION__, __LINE__, level, errnum, 0,       \
+                msgid, ##fmt);                                                 \
+    } while (0)
+
+/* no frills, no thrills, just a vanilla message, used to print the graph */
+#define gf_msg_plain(level, fmt...)                                            \
+    do {                                                                       \
+        _gf_msg_plain(level, ##fmt);                                           \
+    } while (0)
+
+#define gf_msg_plain_nomem(level, msg)                                         \
+    do {                                                                       \
+        _gf_msg_plain_nomem(level, msg);                                       \
+    } while (0)
+
+#define gf_msg_vplain(level, fmt, va)                                          \
+    do {                                                                       \
+        _gf_msg_vplain(level, fmt, va);                                        \
+    } while (0)
+
+#define gf_msg_backtrace_nomem(level, stacksize)                               \
+    do {                                                                       \
+        _gf_msg_backtrace_nomem(level, stacksize);                             \
+    } while (0)
+
+#define gf_msg_callingfn(dom, level, errnum, msgid, fmt...)                    \
+    do {                                                                       \
+        _gf_msg(dom, __FILE__, __FUNCTION__, __LINE__, level, errnum, 1,       \
+                msgid, ##fmt);                                                 \
+    } while (0)
+
+/* No malloc or calloc should be called in this function */
+#define gf_msg_nomem(dom, level, size)                                         \
+    do {                                                                       \
+        _gf_msg_nomem(dom, __FILE__, __FUNCTION__, __LINE__, level, size);     \
+    } while (0)
+
+/* Debug or trace messages do not need message IDs as these are more developer
+ * related. Hence, the following abstractions are provided for the same */
+#define gf_msg_debug(dom, errnum, fmt...)                                      \
+    do {                                                                       \
+        _gf_msg(dom, __FILE__, __FUNCTION__, __LINE__, GF_LOG_DEBUG, errnum,   \
+                0, 0, ##fmt);                                                  \
+    } while (0)
+
+#define gf_msg_trace(dom, errnum, fmt...)                                      \
+    do {                                                                       \
+        _gf_msg(dom, __FILE__, __FUNCTION__, __LINE__, GF_LOG_TRACE, errnum,   \
+                0, 0, ##fmt);                                                  \
+    } while (0)
+
+#define gf_log(dom, level, fmt...)                                             \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+        _gf_log(dom, __FILE__, __FUNCTION__, __LINE__, level, ##fmt);          \
+    } while (0)
+
+#define gf_log_eh(fmt...)                                                      \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+        _gf_log_eh(__FUNCTION__, ##fmt);                                       \
+    } while (0)
+
+#define gf_log_callingfn(dom, level, fmt...)                                   \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+        _gf_log_callingfn(dom, __FILE__, __FUNCTION__, __LINE__, level,        \
+                          ##fmt);                                              \
+    } while (0)
+
+/* Log once in GF_UNIVERSAL_ANSWER times */
+#define GF_LOG_OCCASIONALLY(var, args...)                                      \
+    if (var++ == 0 || !((var - 1) % GF_UNIVERSAL_ANSWER)) {                    \
+        gf_log(args);                                                          \
+    }
+
+struct _glusterfs_ctx;
+
+void
+gf_log_disable_syslog(void);
+void
+gf_log_enable_syslog(void);
+gf_loglevel_t
+gf_log_get_loglevel(void);
+void
+gf_log_set_loglevel(struct _glusterfs_ctx *ctx, gf_loglevel_t level);
+int
+gf_log_get_localtime(void);
+void
+gf_log_set_localtime(int);
+void
+gf_log_flush(void);
+gf_loglevel_t
+gf_log_get_xl_loglevel(void *xl);
+void
+gf_log_set_xl_loglevel(void *xl, gf_loglevel_t level);
+
+int
+gf_cmd_log(const char *domain, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+int
+gf_cmd_log_init(const char *filename);
+
+void
+set_sys_log_level(gf_loglevel_t level);
+
+int
+gf_log_fini(void *data);
+
+void
+gf_log_set_logger(gf_log_logger_t logger);
+
+void
+gf_log_set_logformat(gf_log_format_t format);
+
+void
+gf_log_set_log_buf_size(uint32_t buf_size);
+
+void
+gf_log_set_log_flush_timeout(uint32_t timeout);
+
+void
+gf_log_flush_msgs(struct _glusterfs_ctx *ctx);
+
+int
+gf_log_inject_timer_event(struct _glusterfs_ctx *ctx);
+
+void
+gf_log_disable_suppression_before_exit(struct _glusterfs_ctx *ctx);
+
+#define GF_DEBUG(xl, format, args...)                                          \
+    gf_log((xl)->name, GF_LOG_DEBUG, format, ##args)
+#define GF_INFO(xl, format, args...)                                           \
+    gf_log((xl)->name, GF_LOG_INFO, format, ##args)
+#define GF_WARNING(xl, format, args...)                                        \
+    gf_log((xl)->name, GF_LOG_WARNING, format, ##args)
+#define GF_ERROR(xl, format, args...)                                          \
+    gf_log((xl)->name, GF_LOG_ERROR, format, ##args)
+
+int
+_gf_smsg(const char *domain, const char *file, const char *function,
+         int32_t line, gf_loglevel_t level, int errnum, int trace,
+         uint64_t msgid, const char *event, ...);
+
+/* Interface to log messages with message IDs */
+#define gf_smsg(dom, level, errnum, msgid, event...)                           \
+    do {                                                                       \
+        _gf_smsg(dom, __FILE__, __FUNCTION__, __LINE__, level, errnum, 0,      \
+                 msgid, msgid##_STR, ##event);                                 \
+    } while (0)
+
+#endif /* __LOGGING_H__ */
diff --git a/libglusterfs/src/glusterfs/lvm-defaults.h b/libglusterfs/src/glusterfs/lvm-defaults.h
new file mode 100644
index 00000000000..32feebf3f6e
--- /dev/null
+++ b/libglusterfs/src/glusterfs/lvm-defaults.h
@@ -0,0 +1,20 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _LVM_DEFAULTS_H
+#define _LVM_DEFAULTS_H
+
+#define LVM_RESIZE "/sbin/lvresize"
+#define LVM_CREATE "/sbin/lvcreate"
+#define LVM_CONVERT "/sbin/lvconvert"
+#define LVM_REMOVE "/sbin/lvremove"
+#define LVS "/sbin/lvs"
+
+#endif /* _LVM_DEFAULTS_H */
diff --git a/libglusterfs/src/glusterfs/mem-pool.h b/libglusterfs/src/glusterfs/mem-pool.h
new file mode 100644
index 00000000000..e5b3276d047
--- /dev/null
+++ b/libglusterfs/src/glusterfs/mem-pool.h
@@ -0,0 +1,336 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _MEM_POOL_H_
+#define _MEM_POOL_H_
+
+#include "glusterfs/list.h"
+#include "glusterfs/atomic.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/mem-types.h"
+#include "glusterfs/glusterfs.h" /* for glusterfs_ctx_t */
+#include <stdlib.h>
+#include <inttypes.h>
+#include <string.h>
+#include <stdarg.h>
+
+/*
+ * Need this for unit tests since inline functions
+ * access memory allocation and need to use the
+ * unit test versions
+ */
+#ifdef UNIT_TESTING
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+#endif
+
+#define GF_MEM_TRAILER_SIZE 8
+#define GF_MEM_HEADER_MAGIC 0xCAFEBABE
+#define GF_MEM_TRAILER_MAGIC 0xBAADF00D
+#define GF_MEM_INVALID_MAGIC 0xDEADC0DE
+
+#define POOL_SMALLEST 7 /* i.e. 128 */
+#define POOL_LARGEST 20 /* i.e. 1048576 */
+#define NPOOLS (POOL_LARGEST - POOL_SMALLEST + 1)
+
+struct mem_acct_rec {
+    const char *typestr;
+    uint64_t size;
+    uint64_t max_size;
+    uint64_t total_allocs;
+    uint32_t num_allocs;
+    uint32_t max_num_allocs;
+    gf_lock_t lock;
+#ifdef DEBUG
+    struct list_head obj_list;
+#endif
+};
+
+struct mem_acct {
+    uint32_t num_types;
+    gf_atomic_t refcnt;
+    struct mem_acct_rec rec[0];
+};
+
+struct mem_header {
+    uint32_t type;
+    size_t size;
+    struct mem_acct *mem_acct;
+    uint32_t magic;
+#ifdef DEBUG
+    struct list_head acct_list;
+#endif
+    int padding[8];
+};
+
+#define GF_MEM_HEADER_SIZE (sizeof(struct mem_header))
+
+#ifdef DEBUG
+struct mem_invalid {
+    uint32_t magic;
+    void *mem_acct;
+    uint32_t type;
+    size_t size;
+    void *baseaddr;
+};
+#endif
+
+void *
+__gf_calloc(size_t cnt, size_t size, uint32_t type, const char *typestr);
+
+void *
+__gf_malloc(size_t size, uint32_t type, const char *typestr);
+
+void *
+__gf_realloc(void *ptr, size_t size);
+
+int
+gf_vasprintf(char **string_ptr, const char *format, va_list arg);
+
+int
+gf_asprintf(char **string_ptr, const char *format, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+void
+__gf_free(void *ptr);
+
+static inline void *
+__gf_default_malloc(size_t size)
+{
+    void *ptr = NULL;
+
+    ptr = malloc(size);
+    if (!ptr)
+        gf_msg_nomem("", GF_LOG_ALERT, size);
+
+    return ptr;
+}
+
+static inline void *
+__gf_default_calloc(int cnt, size_t size)
+{
+    void *ptr = NULL;
+
+    ptr = calloc(cnt, size);
+    if (!ptr)
+        gf_msg_nomem("", GF_LOG_ALERT, (cnt * size));
+
+    return ptr;
+}
+
+static inline void *
+__gf_default_realloc(void *oldptr, size_t size)
+{
+    void *ptr = NULL;
+
+    ptr = realloc(oldptr, size);
+    if (!ptr)
+        gf_msg_nomem("", GF_LOG_ALERT, size);
+
+    return ptr;
+}
+
+#define MALLOC(size) __gf_default_malloc(size)
+#define CALLOC(cnt, size) __gf_default_calloc(cnt, size)
+#define REALLOC(ptr, size) __gf_default_realloc(ptr, size)
+
+#define FREE(ptr)                                                              \
+    do {                                                                       \
+        if (ptr != NULL) {                                                     \
+            free((void *)ptr);                                                 \
+            ptr = (void *)0xeeeeeeee;                                          \
+        }                                                                      \
+    } while (0)
+
+#define GF_CALLOC(nmemb, size, type) __gf_calloc(nmemb, size, type, #type)
+
+#define GF_MALLOC(size, type) __gf_malloc(size, type, #type)
+
+#define GF_REALLOC(ptr, size) __gf_realloc(ptr, size)
+
+#define GF_FREE(free_ptr) __gf_free(free_ptr)
+
+static inline char *
+gf_strndup(const char *src, size_t len)
+{
+    char *dup_str = NULL;
+
+    if (!src) {
+        goto out;
+    }
+
+    dup_str = GF_MALLOC(len + 1, gf_common_mt_strdup);
+    if (!dup_str) {
+        goto out;
+    }
+
+    memcpy(dup_str, src, len);
+    dup_str[len] = '\0';
+out:
+    return dup_str;
+}
+
+static inline char *
+gf_strdup(const char *src)
+{
+    if (!src)
+        return NULL;
+
+    return gf_strndup(src, strlen(src));
+}
+
+static inline void *
+gf_memdup(const void *src, size_t size)
+{
+    void *dup_mem = NULL;
+
+    dup_mem = GF_MALLOC(size, gf_common_mt_memdup);
+    if (!dup_mem)
+        goto out;
+
+    memcpy(dup_mem, src, size);
+
+out:
+    return dup_mem;
+}
+
+#ifdef GF_DISABLE_MEMPOOL
+
+/* No-op memory pool enough to fit current API without massive redesign. */
+
+struct mem_pool {
+    unsigned long sizeof_type;
+};
+
+#define mem_pools_init()                                                       \
+    do {                                                                       \
+    } while (0)
+#define mem_pools_fini()                                                       \
+    do {                                                                       \
+    } while (0)
+#define mem_pool_thread_destructor(pool_list) (void)pool_list
+
+#else /* !GF_DISABLE_MEMPOOL */
+
+/* kind of 'header' for the actual mem_pool_shared structure, this might make
+ * it possible to dump some more details in a statedump */
+struct mem_pool {
+    /* object size, without pooled_obj_hdr_t */
+    unsigned long sizeof_type;
+    unsigned long count; /* requested pool size (unused) */
+    char *name;
+    char *xl_name;
+    gf_atomic_t active;     /* current allocations */
+#ifdef DEBUG
+    gf_atomic_t hit;        /* number of allocations served from pt_pool */
+    gf_atomic_t miss;       /* number of std allocs due to miss */
+#endif
+    struct list_head owner; /* glusterfs_ctx_t->mempool_list */
+    glusterfs_ctx_t *ctx;   /* take ctx->lock when updating owner */
+
+    struct mem_pool_shared *pool; /* the initial pool that was returned */
+};
+
+typedef struct pooled_obj_hdr {
+    unsigned long magic;
+    struct pooled_obj_hdr *next;
+    struct per_thread_pool_list *pool_list;
+    unsigned int power_of_two;
+
+    /* track the pool that was used to request this object */
+    struct mem_pool *pool;
+} pooled_obj_hdr_t;
+
+/* Each memory block inside a pool has a fixed size that is a power of two.
+ * However each object will have a header that will reduce the available
+ * space. */
+#define AVAILABLE_SIZE(p2) ((1UL << (p2)) - sizeof(pooled_obj_hdr_t))
+
+typedef struct per_thread_pool {
+    /* the pool that was used to request this allocation */
+    struct mem_pool_shared *parent;
+    /* Everything else is protected by our own lock. */
+    pooled_obj_hdr_t *hot_list;
+    pooled_obj_hdr_t *cold_list;
+} per_thread_pool_t;
+
+typedef struct per_thread_pool_list {
+    /* thr_list is used to place the TLS pool_list into the active global list
+     * (pool_threads) or the inactive global list (pool_free_threads). It's
+     * protected by the global pool_lock. */
+    struct list_head thr_list;
+
+    /* This lock is used to update poison and the hot/cold lists of members
+     * of 'pools' array. */
+    pthread_spinlock_t lock;
+
+    /* This field is used to mark a pool_list as not being owned by any thread.
+     * This means that the sweeper thread won't be cleaning objects stored in
+     * its pools. mem_put() uses it to decide if the object being released is
+     * placed into its original pool_list or directly destroyed. */
+    bool poison;
+
+    /*
+     * There's really more than one pool, but the actual number is hidden
+     * in the implementation code so we just make it a single-element array
+     * here.
+     */
+    per_thread_pool_t pools[1];
+} per_thread_pool_list_t;
+
+/* actual pool structure, shared between different mem_pools */
+struct mem_pool_shared {
+    unsigned int power_of_two;
+    /*
+     * Updates to these are *not* protected by a global lock, so races
+     * could occur and the numbers might be slightly off.  Don't expect
+     * them to line up exactly.  It's the general trends that matter, and
+     * it's not worth the locked-bus-cycle overhead to make these precise.
+     */
+    gf_atomic_t allocs_hot;
+    gf_atomic_t allocs_cold;
+    gf_atomic_t allocs_stdc;
+    gf_atomic_t frees_to_list;
+};
+
+void
+mem_pools_init(void); /* start the pool_sweeper thread */
+void
+mem_pools_fini(void); /* cleanup memory pools */
+void
+mem_pool_thread_destructor(per_thread_pool_list_t *pool_list);
+
+#endif /* GF_DISABLE_MEMPOOL */
+
+struct mem_pool *
+mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type,
+                unsigned long count, char *name);
+
+#define mem_pool_new(type, count)                                              \
+    mem_pool_new_fn(THIS->ctx, sizeof(type), count, #type)
+
+#define mem_pool_new_ctx(ctx, type, count)                                     \
+    mem_pool_new_fn(ctx, sizeof(type), count, #type)
+
+void
+mem_put(void *ptr);
+void *
+mem_get(struct mem_pool *pool);
+void *
+mem_get0(struct mem_pool *pool);
+
+void
+mem_pool_destroy(struct mem_pool *pool);
+
+void
+gf_mem_acct_enable_set(void *ctx);
+
+#endif /* _MEM_POOL_H */
diff --git a/libglusterfs/src/glusterfs/mem-types.h b/libglusterfs/src/glusterfs/mem-types.h
new file mode 100644
index 00000000000..d45d5b68c91
--- /dev/null
+++ b/libglusterfs/src/glusterfs/mem-types.h
@@ -0,0 +1,139 @@
+/*
+  Copyright (c) 2008-2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __MEM_TYPES_H__
+#define __MEM_TYPES_H__
+
+enum gf_common_mem_types_ {
+    gf_common_mt_dnscache6, /* used only in one location */
+    gf_common_mt_event_pool,
+    gf_common_mt_reg,
+    gf_common_mt_pollfd,    /* used only in one location */
+    gf_common_mt_fdentry_t, /* used only in one location */
+    gf_common_mt_fdtable_t, /* used only in one location */
+    gf_common_mt_fd_ctx,    /* used only in one location */
+    gf_common_mt_gf_dirent_t,
+    gf_common_mt_inode_t,   /* used only in one location */
+    gf_common_mt_inode_ctx, /* used only in one location */
+    gf_common_mt_list_head,
+    gf_common_mt_inode_table_t, /* used only in one location */
+    gf_common_mt_xlator_t,
+    gf_common_mt_xlator_list_t, /* used only in one location */
+    gf_common_mt_volume_opt_list_t,
+    gf_common_mt_gf_timer_t,          /* used only in one location */
+    gf_common_mt_gf_timer_registry_t, /* used only in one location */
+    gf_common_mt_auth_handle_t,       /* used only in one location */
+    gf_common_mt_iobuf,               /* used only in one location */
+    gf_common_mt_iobuf_arena,         /* used only in one location */
+    gf_common_mt_iobref,              /* used only in one location */
+    gf_common_mt_iobuf_pool,          /* used only in one location */
+    gf_common_mt_iovec,
+    gf_common_mt_memdup,   /* used only in one location */
+    gf_common_mt_asprintf, /* used only in one location */
+    gf_common_mt_strdup,
+    gf_common_mt_socket_private_t, /* used only in one location */
+    gf_common_mt_ioq,              /* used only in one location */
+    gf_common_mt_char,
+    gf_common_mt_rbthash_table_t,      /* used only in one location */
+    gf_common_mt_rbthash_bucket,       /* used only in one location */
+    gf_common_mt_mem_pool,             /* used only in one location */
+    gf_common_mt_rpcsvc_auth_list,     /* used only in one location */
+    gf_common_mt_rpcsvc_t,             /* used only in one location */
+    gf_common_mt_rpcsvc_program_t,     /* used only in one location */
+    gf_common_mt_rpcsvc_listener_t,    /* used only in one location */
+    gf_common_mt_rpcsvc_wrapper_t,     /* used only in one location */
+    gf_common_mt_rpcclnt_t,            /* used only in one location */
+    gf_common_mt_rpcclnt_savedframe_t, /* used only in one location */
+    gf_common_mt_rpc_trans_t,
+    gf_common_mt_rpc_trans_pollin_t,  /* used only in one location */
+    gf_common_mt_rpc_trans_reqinfo_t, /* used only in one location */
+    gf_common_mt_glusterfs_graph_t,
+    gf_common_mt_rdma_private_t,       /* used only in one location */
+    gf_common_mt_rpc_transport_t,      /* used only in one location */
+    gf_common_mt_rdma_post_t,          /* used only in one location */
+    gf_common_mt_qpent,                /* used only in one location */
+    gf_common_mt_rdma_device_t,        /* used only in one location */
+    gf_common_mt_rdma_arena_mr,        /* used only in one location */
+    gf_common_mt_sge,                  /* used only in one location */
+    gf_common_mt_rpcclnt_cb_program_t, /* used only in one location */
+    gf_common_mt_libxl_marker_local,   /* used only in one location */
+    gf_common_mt_graph_buf,            /* used only in one location */
+    gf_common_mt_trie_trie,            /* used only in one location */
+    gf_common_mt_trie_data,            /* used only in one location */
+    gf_common_mt_trie_node,            /* used only in one location */
+    gf_common_mt_trie_buf,             /* used only in one location */
+    gf_common_mt_run_argv,             /* used only in one location */
+    gf_common_mt_run_logbuf,           /* used only in one location */
+    gf_common_mt_fd_lk_ctx_t,          /* used only in one location */
+    gf_common_mt_fd_lk_ctx_node_t,     /* used only in one location */
+    gf_common_mt_buffer_t,             /* used only in one location */
+    gf_common_mt_circular_buffer_t,    /* used only in one location */
+    gf_common_mt_eh_t,
+    gf_common_mt_store_handle_t, /* used only in one location */
+    gf_common_mt_store_iter_t,   /* used only in one location */
+    gf_common_mt_drc_client_t,   /* used only in one location */
+    gf_common_mt_drc_globals_t,  /* used only in one location */
+    gf_common_mt_groups_t,
+    gf_common_mt_cliententry_t, /* used only in one location */
+    gf_common_mt_clienttable_t, /* used only in one location */
+    gf_common_mt_client_t,      /* used only in one location */
+    gf_common_mt_client_ctx,    /* used only in one location */
+    gf_common_mt_auxgids,       /* used only in one location */
+    gf_common_mt_syncopctx,     /* used only in one location */
+    gf_common_mt_iobrefs,       /* used only in one location */
+    gf_common_mt_gsync_status_t,
+    gf_common_mt_uuid_t,
+    gf_common_mt_mgmt_v3_lock_obj_t, /* used only in one location */
+    gf_common_mt_txn_opinfo_obj_t,   /* used only in one location */
+    gf_common_mt_strfd_t,            /* used only in one location */
+    gf_common_mt_strfd_data_t,       /* used only in one location */
+    gf_common_mt_regex_t,            /* used only in one location */
+    gf_common_mt_ereg,               /* used only in one location */
+    gf_common_mt_wr,                 /* used only in one location */
+    gf_common_mt_dnscache,           /* used only in one location */
+    gf_common_mt_dnscache_entry,     /* used only in one location */
+    gf_common_mt_parser_t,           /* used only in one location */
+    gf_common_quota_meta_t,
+    gf_common_mt_rbuf_t,  /* used only in one location */
+    gf_common_mt_rlist_t, /* used only in one location */
+    gf_common_mt_rvec_t,  /* used only in one location */
+    /* glusterd can load the nfs-xlator dynamically and needs these two */
+    gf_common_mt_nfs_netgroups,   /* used only in one location */
+    gf_common_mt_nfs_exports,     /* used only in one location */
+    gf_common_mt_gf_brick_spec_t, /* used only in one location */
+    gf_common_mt_int,
+    gf_common_mt_pointer,
+    gf_common_mt_synctask,  /* used only in one location */
+    gf_common_mt_syncstack, /* used only in one location */
+    gf_common_mt_syncenv,   /* used only in one location */
+    gf_common_mt_scan_data, /* used only in one location */
+    gf_common_list_node,
+    gf_mt_default_args_t,     /* used only in one location */
+    gf_mt_default_args_cbk_t, /* used only in one location */
+    /*used for compound fops*/
+    gf_mt_compound_req_t, /* used only in one location */
+    gf_mt_compound_rsp_t, /* used only in one location */
+    gf_common_mt_tw_ctx,  /* used only in one location */
+    gf_common_mt_tw_timer_list,
+    /*lock migration*/
+    gf_common_mt_lock_mig,
+    /* throttle */
+    gf_common_mt_tbf_t,          /* used only in one location */
+    gf_common_mt_tbf_bucket_t,   /* used only in one location */
+    gf_common_mt_tbf_throttle_t, /* used only in one location */
+    gf_common_mt_pthread_t,      /* used only in one location */
+    gf_common_ping_local_t,      /* used only in one location */
+    gf_common_volfile_t,
+    gf_common_mt_mgmt_v3_lock_timer_t, /* used only in one location */
+    gf_common_mt_server_cmdline_t,     /* used only in one location */
+    gf_common_mt_latency_t,
+    gf_common_mt_end
+};
+#endif
diff --git a/libglusterfs/src/glusterfs/monitoring.h b/libglusterfs/src/glusterfs/monitoring.h
new file mode 100644
index 00000000000..09d9f54e734
--- /dev/null
+++ b/libglusterfs/src/glusterfs/monitoring.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __MONITORING_H__
+#define __MONITORING_H__
+
+#include "glusterfs/glusterfs.h"
+
+#define GLUSTER_METRICS_DIR "/var/run/gluster/metrics"
+
+char *
+gf_monitor_metrics(glusterfs_ctx_t *ctx);
+
+#endif /* __MONITORING_H__ */
diff --git a/libglusterfs/src/glusterfs/options.h b/libglusterfs/src/glusterfs/options.h
new file mode 100644
index 00000000000..747b13ba375
--- /dev/null
+++ b/libglusterfs/src/glusterfs/options.h
@@ -0,0 +1,327 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _OPTIONS_H
+#define _OPTIONS_H
+
+#include <stdio.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/libglusterfs-messages.h"
+/* Add possible new type of option you may need */
+typedef enum {
+    GF_OPTION_TYPE_ANY = 0,
+    GF_OPTION_TYPE_STR,
+    GF_OPTION_TYPE_INT,
+    GF_OPTION_TYPE_SIZET,
+    GF_OPTION_TYPE_PERCENT,
+    GF_OPTION_TYPE_PERCENT_OR_SIZET,
+    GF_OPTION_TYPE_BOOL,
+    GF_OPTION_TYPE_XLATOR,
+    GF_OPTION_TYPE_PATH,
+    GF_OPTION_TYPE_TIME,
+    GF_OPTION_TYPE_DOUBLE,
+    GF_OPTION_TYPE_INTERNET_ADDRESS,
+    GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+    GF_OPTION_TYPE_PRIORITY_LIST,
+    GF_OPTION_TYPE_SIZE_LIST,
+    GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
+    GF_OPTION_TYPE_MAX,
+} volume_option_type_t;
+
+typedef enum {
+    GF_OPT_VALIDATE_BOTH = 0,
+    GF_OPT_VALIDATE_MIN,
+    GF_OPT_VALIDATE_MAX,
+} opt_validate_type_t;
+
+typedef enum {
+    OPT_FLAG_NONE = 0,
+    OPT_FLAG_SETTABLE = 1 << 0,   /* can be set using volume set */
+    OPT_FLAG_CLIENT_OPT = 1 << 1, /* affects clients */
+    OPT_FLAG_GLOBAL = 1
+                      << 2, /* affects all instances of the particular xlator */
+    OPT_FLAG_FORCE = 1 << 3,       /* needs force to be reset */
+    OPT_FLAG_NEVER_RESET = 1 << 4, /* which should not be reset */
+    OPT_FLAG_DOC = 1 << 5,         /* can be shown in volume set help */
+} opt_flags_t;
+
+typedef enum {
+    OPT_STATUS_ADVANCED = 0,
+    OPT_STATUS_BASIC = 1,
+    OPT_STATUS_EXPERIMENTAL = 2,
+    OPT_STATUS_DEPRECATED = 3,
+} opt_level_t;
+
+#define ZR_VOLUME_MAX_NUM_KEY 4
+#define ZR_OPTION_MAX_ARRAY_SIZE 64
+/* The maximum number of releases that an option could be backported to
+ * based on the release schedule as in August 2017 (3), plus one more
+ * Refer comment on volume_options.op_version for more information.
+ */
+#define GF_MAX_RELEASES 4
+
+/* Custom validation functoins for options
+ * TODO: Need to check what sorts of validation is being done, and decide if
+ * passing the volinfo is actually required. If it is, then we should possibly
+ * try a solution in GD2 for this.
+ */
+/* typedef int (*option_validation_fn) (glusterd_volinfo_t *volinfo, dict_t
+ *dict, char *key, char *value, char **op_errstr);
+ */
+
+/* Each translator should define this structure */
+/* XXX: This structure is in use by GD2, and SHOULD NOT be modified.
+ * If there is a need to add new members, add them to the end of the structure.
+ * If the struct must be modified, GD2 MUST be updated as well
+ */
+typedef struct volume_options {
+    char *key[ZR_VOLUME_MAX_NUM_KEY];
+    /* different key, same meaning */
+    volume_option_type_t type;
+    double min; /* 0 means no range */
+    double max; /* 0 means no range */
+    char *value[ZR_OPTION_MAX_ARRAY_SIZE];
+    /* If specified, will check for one of
+       the value from this array */
+    char *default_value;
+    char *description; /* about the key */
+    /* Required for int options where only the min value
+     * is given and is 0. This will cause validation not to
+     * happen
+     */
+    opt_validate_type_t validate;
+
+    /* The op-version at which this option was introduced.
+     * This is an array to support options that get backported to supported
+     * releases.
+     * Normally, an option introduced for a major release just has a single
+     * entry in the array, with op-version of the major release
+     * For an option that is backported, the op-versions of the all the
+     * releases it was ported to should be added, starting from the newest,
+     * to the oldest.
+     */
+    uint32_t op_version[GF_MAX_RELEASES];
+    /* The op-version at which this option was deprecated.
+     * Follows the same rules as above.
+     */
+    uint32_t deprecated[GF_MAX_RELEASES];
+    /* Additional flags for an option
+     * Check the OPT_FLAG_* enums for available flags
+     */
+    uint32_t flags;
+    /* Tags applicable to this option, which can be used to group similar
+     * options
+     */
+    char *tags[ZR_OPTION_MAX_ARRAY_SIZE];
+    /* A custom validation function if required
+     * TODO: See todo above for option_validation_fn
+     */
+    /* option_validation_fn validate_fn; */
+    /* This is actual key that should be set in the options dict. Can
+     * contain varstrings
+     */
+    char *setkey;
+
+    /* A 'level' is about the technical depth / understanding one
+       needs to handle the option. 'category' is based on
+       quality (ie, tests, people behind it, documentation available) */
+
+    /* The level at which the option is classified */
+    opt_level_t level;
+
+    /* Flag to understand how this option is categorized */
+    gf_category_t category;
+} volume_option_t;
+
+typedef struct vol_opt_list {
+    struct list_head list;
+    volume_option_t *given_opt;
+} volume_opt_list_t;
+
+int
+xlator_tree_reconfigure(xlator_t *old_xl, xlator_t *new_xl);
+int
+xlator_validate_rec(xlator_t *xlator, char **op_errstr);
+int
+graph_reconf_validateopt(glusterfs_graph_t *graph, char **op_errstr);
+int
+xlator_option_info_list(volume_opt_list_t *list, char *key, char **def_val,
+                        char **descr);
+/*
+int validate_xlator_volume_options (xlator_t *xl, dict_t *options,
+                                    volume_option_t *opt, char **op_errstr);
+*/
+int
+xlator_options_validate_list(xlator_t *xl, dict_t *options,
+                             volume_opt_list_t *list, char **op_errstr);
+int
+xlator_option_validate(xlator_t *xl, char *key, char *value,
+                       volume_option_t *opt, char **op_errstr);
+int
+xlator_options_validate(xlator_t *xl, dict_t *options, char **errstr);
+
+int
+xlator_option_validate_addr_list(xlator_t *xl, const char *key,
+                                 const char *value, volume_option_t *opt,
+                                 char **op_errstr);
+
+volume_option_t *
+xlator_volume_option_get(xlator_t *xl, const char *key);
+
+volume_option_t *
+xlator_volume_option_get_list(volume_opt_list_t *vol_list, const char *key);
+
+#define DECLARE_INIT_OPT(type_t, type)                                         \
+    int xlator_option_init_##type(xlator_t *this, dict_t *options, char *key,  \
+                                  type_t *val_p);
+
+DECLARE_INIT_OPT(char *, str);
+DECLARE_INIT_OPT(uint64_t, uint64);
+DECLARE_INIT_OPT(int64_t, int64);
+DECLARE_INIT_OPT(uint32_t, uint32);
+DECLARE_INIT_OPT(int32_t, int32);
+DECLARE_INIT_OPT(uint64_t, size);
+DECLARE_INIT_OPT(uint64_t, size_uint64);
+DECLARE_INIT_OPT(double, percent);
+DECLARE_INIT_OPT(double, percent_or_size);
+DECLARE_INIT_OPT(gf_boolean_t, bool);
+DECLARE_INIT_OPT(xlator_t *, xlator);
+DECLARE_INIT_OPT(char *, path);
+DECLARE_INIT_OPT(double, double);
+DECLARE_INIT_OPT(uint32_t, time);
+
+#define DEFINE_INIT_OPT(type_t, type, conv)                                    \
+    int xlator_option_init_##type(xlator_t *this, dict_t *options, char *key,  \
+                                  type_t *val_p)                               \
+    {                                                                          \
+        int ret = 0;                                                           \
+        volume_option_t *opt = NULL;                                           \
+        char *def_value = NULL;                                                \
+        char *set_value = NULL;                                                \
+        char *value = NULL;                                                    \
+        xlator_t *old_THIS = NULL;                                             \
+                                                                               \
+        opt = xlator_volume_option_get(this, key);                             \
+        if (!opt) {                                                            \
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ENTRY,   \
+                   "unknown option: %s", key);                                 \
+            ret = -1;                                                          \
+            return ret;                                                        \
+        }                                                                      \
+        def_value = opt->default_value;                                        \
+        ret = dict_get_str(options, key, &set_value);                          \
+                                                                               \
+        if (def_value)                                                         \
+            value = def_value;                                                 \
+        if (set_value)                                                         \
+            value = set_value;                                                 \
+        if (!value) {                                                          \
+            gf_msg_trace(this->name, 0, "option %s not set", key);             \
+            *val_p = (type_t)0;                                                \
+            return 0;                                                          \
+        }                                                                      \
+        if (value == def_value) {                                              \
+            gf_msg_trace(this->name, 0,                                        \
+                         "option %s using default"                             \
+                         " value %s",                                          \
+                         key, value);                                          \
+        } else {                                                               \
+            gf_msg_debug(this->name, 0,                                        \
+                         "option %s using set"                                 \
+                         " value %s",                                          \
+                         key, value);                                          \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = this;                                                           \
+        ret = conv(value, val_p);                                              \
+        THIS = old_THIS;                                                       \
+        if (ret) {                                                             \
+            gf_msg(this->name, GF_LOG_INFO, 0, LG_MSG_CONVERSION_FAILED,       \
+                   "option %s conversion failed value %s", key, value);        \
+            return ret;                                                        \
+        }                                                                      \
+        ret = xlator_option_validate(this, key, value, opt, NULL);             \
+        return ret;                                                            \
+    }
+
+#define GF_OPTION_INIT(key, val, type, err_label)                              \
+    do {                                                                       \
+        int val_ret = 0;                                                       \
+        val_ret = xlator_option_init_##type(THIS, THIS->options, key, &(val)); \
+        if (val_ret)                                                           \
+            goto err_label;                                                    \
+    } while (0)
+
+#define DECLARE_RECONF_OPT(type_t, type)                                       \
+    int xlator_option_reconf_##type(xlator_t *this, dict_t *options,           \
+                                    char *key, int keylen, type_t *val_p);
+
+DECLARE_RECONF_OPT(char *, str);
+DECLARE_RECONF_OPT(uint64_t, uint64);
+DECLARE_RECONF_OPT(int64_t, int64);
+DECLARE_RECONF_OPT(uint32_t, uint32);
+DECLARE_RECONF_OPT(int32_t, int32);
+DECLARE_RECONF_OPT(uint64_t, size);
+DECLARE_RECONF_OPT(uint64_t, size_uint64);
+DECLARE_RECONF_OPT(double, percent);
+DECLARE_RECONF_OPT(double, percent_or_size);
+DECLARE_RECONF_OPT(gf_boolean_t, bool);
+DECLARE_RECONF_OPT(xlator_t *, xlator);
+DECLARE_RECONF_OPT(char *, path);
+DECLARE_RECONF_OPT(double, double);
+DECLARE_RECONF_OPT(uint32_t, time);
+
+#define DEFINE_RECONF_OPT(type_t, type, conv)                                  \
+    int xlator_option_reconf_##type(xlator_t *this, dict_t *options,           \
+                                    char *key, int keylen, type_t *val_p)      \
+    {                                                                          \
+        int ret = 0;                                                           \
+        char *value = NULL;                                                    \
+        xlator_t *old_THIS = NULL;                                             \
+                                                                               \
+        volume_option_t *opt = xlator_volume_option_get(this, key);            \
+        if (!opt) {                                                            \
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ENTRY,   \
+                   "unknown option: %s", key);                                 \
+            return -1;                                                         \
+        }                                                                      \
+        ret = dict_get_strn(options, key, keylen, &value);                     \
+        if (ret == 0 && value) {                                               \
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,                              \
+                   "option %s using set value %s", key, value);                \
+        } else if (opt->default_value) {                                       \
+            value = opt->default_value;                                        \
+            gf_msg_trace(this->name, 0, "option %s using default value %s",    \
+                         key, value);                                          \
+        } else {                                                               \
+            gf_msg_trace(this->name, 0, "option %s not set", key);             \
+            *val_p = (type_t)0;                                                \
+            return 0;                                                          \
+        }                                                                      \
+                                                                               \
+        old_THIS = THIS;                                                       \
+        THIS = this;                                                           \
+        ret = conv(value, val_p);                                              \
+        THIS = old_THIS;                                                       \
+        if (ret)                                                               \
+            return ret;                                                        \
+        return xlator_option_validate(this, key, value, opt, NULL);            \
+    }
+
+#define GF_OPTION_RECONF(key, val, opt, type, err_label)                       \
+    do {                                                                       \
+        if (xlator_option_reconf_##type(THIS, opt, key, SLEN(key), &(val)))    \
+            goto err_label;                                                    \
+    } while (0)
+
+#endif /* !_OPTIONS_H */
diff --git a/libglusterfs/src/glusterfs/parse-utils.h b/libglusterfs/src/glusterfs/parse-utils.h
new file mode 100644
index 00000000000..8653b9dd180
--- /dev/null
+++ b/libglusterfs/src/glusterfs/parse-utils.h
@@ -0,0 +1,50 @@
+/*
+   Copyright 2014-present Facebook. All Rights Reserved
+
+   This file is part of GlusterFS.
+
+   Author :
+   Shreyas Siravara <shreyas.siravara@gmail.com>
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _PARSE_UTILS_H
+#define _PARSE_UTILS_H
+
+#include <regex.h>
+
+#define GF_PARSE "parse-utils"
+
+struct parser {
+    regex_t preg;         /* Compiled regex */
+    regmatch_t pmatch[1]; /* The match */
+    char *complete_str;   /* The string we are parsing */
+    char *regex;          /* Regex used to parse the string */
+    char *_rstr;          /* Temp string to hold offsets */
+};
+
+/* Initializes some of the parsers variables */
+struct parser *
+parser_init(const char *regex);
+
+/* Sets the string to parse */
+int
+parser_set_string(struct parser *parser, const char *complete_str);
+
+/* Frees memory used by the string after all matches are found */
+int
+parser_unset_string(struct parser *parser);
+
+/* Free memory used by the parser */
+void
+parser_deinit(struct parser *ptr);
+
+/* Get the next matching string */
+char *
+parser_get_next_match(struct parser *parser);
+
+#endif /* _PARSE_UTILS_H */
diff --git a/libglusterfs/src/glusterfs/quota-common-utils.h b/libglusterfs/src/glusterfs/quota-common-utils.h
new file mode 100644
index 00000000000..0096e340756
--- /dev/null
+++ b/libglusterfs/src/glusterfs/quota-common-utils.h
@@ -0,0 +1,68 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _QUOTA_COMMON_UTILS_H
+#define _QUOTA_COMMON_UTILS_H
+
+#include "glusterfs/iatt.h"
+
+#define GF_QUOTA_CONF_VERSION 1.2
+#define QUOTA_CONF_HEADER "GlusterFS Quota conf | version: v1.2\n"
+#define QUOTA_CONF_HEADER_1_1 "GlusterFS Quota conf | version: v1.1\n"
+
+typedef enum {
+    GF_QUOTA_CONF_TYPE_USAGE = 1,
+    GF_QUOTA_CONF_TYPE_OBJECTS
+} gf_quota_conf_type_t;
+
+struct _quota_limits {
+    int64_t hl;
+    int64_t sl;
+} __attribute__((__packed__));
+typedef struct _quota_limits quota_limits_t;
+
+struct _quota_meta {
+    int64_t size;
+    int64_t file_count;
+    int64_t dir_count;
+} __attribute__((__packed__));
+typedef struct _quota_meta quota_meta_t;
+
+gf_boolean_t
+quota_meta_is_null(const quota_meta_t *meta);
+
+int32_t
+quota_data_to_meta(data_t *data, quota_meta_t *meta);
+
+int32_t
+quota_dict_get_inode_meta(dict_t *dict, char *key, const int keylen,
+                          quota_meta_t *meta);
+
+int32_t
+quota_dict_get_meta(dict_t *dict, char *key, const int keylen,
+                    quota_meta_t *meta);
+
+int32_t
+quota_dict_set_meta(dict_t *dict, char *key, const quota_meta_t *meta,
+                    ia_type_t ia_type);
+
+int32_t
+quota_conf_read_header(int fd, char *buf);
+
+int32_t
+quota_conf_read_version(int fd, float *version);
+
+int32_t
+quota_conf_read_gfid(int fd, void *buf, char *type, float version);
+
+int32_t
+quota_conf_skip_header(int fd);
+
+#endif /* _QUOTA_COMMON_UTILS_H */
diff --git a/libglusterfs/src/glusterfs/rbthash.h b/libglusterfs/src/glusterfs/rbthash.h
new file mode 100644
index 00000000000..4c731de69c2
--- /dev/null
+++ b/libglusterfs/src/glusterfs/rbthash.h
@@ -0,0 +1,75 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __RBTHASH_TABLE_H_
+#define __RBTHASH_TABLE_H_
+
+#include <stdint.h>               // for uint32_t
+#include "glusterfs/glusterfs.h"  // for gf_boolean_t, glusterfs_ctx_t
+#include "glusterfs/list.h"       // for list_head
+#include "glusterfs/locking.h"    // for gf_lock_t
+struct mem_pool;
+
+#define GF_RBTHASH_MEMPOOL 16384  // 1048576
+#define GF_RBTHASH "rbthash"
+
+struct rbthash_bucket {
+    struct rb_table *bucket;
+    gf_lock_t bucketlock;
+};
+
+typedef struct rbthash_entry {
+    void *data;
+    void *key;
+    int keylen;
+    uint32_t keyhash;
+    struct list_head list;
+} rbthash_entry_t;
+
+typedef uint32_t (*rbt_hasher_t)(void *data, int len);
+typedef void (*rbt_data_destroyer_t)(void *data);
+typedef void (*rbt_traverse_t)(void *data, void *mydata);
+
+typedef struct rbthash_table {
+    int size;
+    int numbuckets;
+    struct mem_pool *entrypool;
+    gf_lock_t tablelock;
+    struct rbthash_bucket *buckets;
+    rbt_hasher_t hashfunc;
+    rbt_data_destroyer_t dfunc;
+    gf_boolean_t pool_alloced;
+    struct list_head list;
+} rbthash_table_t;
+
+extern rbthash_table_t *
+rbthash_table_init(glusterfs_ctx_t *ctx, int buckets, rbt_hasher_t hfunc,
+                   rbt_data_destroyer_t dfunc, unsigned long expected_entries,
+                   struct mem_pool *entrypool);
+
+extern int
+rbthash_insert(rbthash_table_t *tbl, void *data, void *key, int keylen);
+
+extern void *
+rbthash_get(rbthash_table_t *tbl, void *key, int keylen);
+
+extern void *
+rbthash_remove(rbthash_table_t *tbl, void *key, int keylen);
+
+extern void *
+rbthash_replace(rbthash_table_t *tbl, void *key, int keylen, void *newdata);
+
+extern void
+rbthash_table_destroy(rbthash_table_t *tbl);
+
+extern void
+rbthash_table_traverse(rbthash_table_t *tbl, rbt_traverse_t traverse,
+                       void *mydata);
+#endif
diff --git a/libglusterfs/src/glusterfs/refcount.h b/libglusterfs/src/glusterfs/refcount.h
new file mode 100644
index 00000000000..cf922dabb05
--- /dev/null
+++ b/libglusterfs/src/glusterfs/refcount.h
@@ -0,0 +1,101 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _REFCOUNT_H
+#define _REFCOUNT_H
+
+/* check for compiler support for __sync_*_and_fetch()
+ *
+ * a more comprehensive feature test is shown at
+ * http://lists.iptel.org/pipermail/semsdev/2010-October/005075.html
+ * this is sufficient for RHEL5 i386 builds
+ */
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) &&                \
+    !defined(__i386__)
+#undef REFCOUNT_NEEDS_LOCK
+#else
+#define REFCOUNT_NEEDS_LOCK
+#include "glusterfs/locking.h"
+#endif /* compiler support for __sync_*_and_fetch() */
+
+typedef void (*gf_ref_release_t)(void *data);
+
+struct _gf_ref {
+#ifdef REFCOUNT_NEEDS_LOCK
+    gf_lock_t lk; /* lock for atomically adjust cnt */
+#endif
+    unsigned int cnt; /* number of users, free on 0 */
+
+    gf_ref_release_t release; /* cleanup when cnt == 0 */
+    void *data;               /* parameter passed to release() */
+};
+typedef struct _gf_ref gf_ref_t;
+
+/* _gf_ref_get -- increase the refcount
+ *
+ * @return: greater then 0 when a reference was taken, 0 when not
+ */
+void *
+_gf_ref_get(gf_ref_t *ref);
+
+/* _gf_ref_put -- decrease the refcount
+ *
+ * @return: greater then 0 when there are still references, 0 when cleanup
+ *          should be done, gf_ref_release_t is called on cleanup
+ */
+unsigned int
+_gf_ref_put(gf_ref_t *ref);
+
+/* _gf_ref_init -- initialize an embedded refcount object
+ *
+ * @release: function to call when the refcount == 0
+ * @data: parameter to be passed to @release
+ */
+void
+_gf_ref_init(gf_ref_t *ref, gf_ref_release_t release, void *data);
+
+/*
+ * Strong suggestion to use the simplified GF_REF_* API.
+ */
+
+/* GF_REF_DECL -- declaration to put inside your structure
+ *
+ * Example:
+ *   struct my_struct {
+ *       GF_REF_DECL;
+ *
+ *       ... // additional members
+ *   };
+ */
+#define GF_REF_DECL gf_ref_t _ref
+
+/* GF_REF_INIT -- initialize a GF_REF_DECL structure
+ *
+ * @p: allocated structure with GF_REF_DECL
+ * @d: destructor to call once refcounting reaches 0
+ *
+ * Sets the refcount to 1.
+ */
+#define GF_REF_INIT(p, d) _gf_ref_init(&(p)->_ref, (gf_ref_release_t)d, p)
+
+/* GF_REF_GET -- increase the refcount of a GF_REF_DECL structure
+ *
+ * @return: greater then 0 when a reference was taken, 0 when not
+ */
+#define GF_REF_GET(p) _gf_ref_get(&(p)->_ref)
+
+/* GF_REF_PUT -- decrease the refcount of a GF_REF_DECL structure
+ *
+ * @return: greater then 0 when there are still references, 0 when cleanup
+ *          should be done, gf_ref_release_t is called on cleanup
+ */
+#define GF_REF_PUT(p) _gf_ref_put(&(p)->_ref)
+
+#endif /* _REFCOUNT_H */
diff --git a/libglusterfs/src/glusterfs/revision.h b/libglusterfs/src/glusterfs/revision.h
new file mode 100644
index 00000000000..3c404d30e78
--- /dev/null
+++ b/libglusterfs/src/glusterfs/revision.h
@@ -0,0 +1 @@
+#define GLUSTERFS_REPOSITORY_REVISION "git://git.gluster.org/glusterfs.git"
diff --git a/libglusterfs/src/glusterfs/rot-buffs.h b/libglusterfs/src/glusterfs/rot-buffs.h
new file mode 100644
index 00000000000..9dc227d58b8
--- /dev/null
+++ b/libglusterfs/src/glusterfs/rot-buffs.h
@@ -0,0 +1,125 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ROT_BUFFS_H
+#define __ROT_BUFFS_H
+
+#include "glusterfs/list.h"
+#include "glusterfs/locking.h"
+#include "glusterfs/common-utils.h"
+
+typedef struct rbuf_iovec {
+    struct iovec iov;
+
+    struct list_head list;
+} rbuf_iovec_t;
+
+#define RBUF_IOVEC_SIZE (sizeof(rbuf_iovec_t))
+
+typedef struct rbuf_list {
+    gf_lock_t c_lock;
+
+    pthread_mutex_t b_lock; /* protects this structure */
+    pthread_cond_t b_cond;  /* signal for writer completion */
+
+    gf_boolean_t awaiting;
+
+    unsigned long long pending;   /* pending writers */
+    unsigned long long completed; /* completed writers */
+
+    rbuf_iovec_t *rvec; /* currently used IO vector */
+
+    struct list_head veclist; /* list of attached rbuf_iov */
+
+    unsigned long long used;  /* consumable entries
+                                  attached in ->veclist */
+    unsigned long long total; /* total entries in ->veclist (used
+                                 during deallocation) */
+
+    unsigned long seq[2]; /* if interested, this whould store
+                             the start sequence number and the
+                             range */
+
+    struct list_head list; /* attachment to rbuf_t */
+} rbuf_list_t;
+
+struct rlist_iter {
+    struct list_head veclist;
+
+    unsigned long long iter;
+};
+
+#define RLIST_ENTRY_COUNT(rlist) rlist->used
+
+#define rlist_iter_init(riter, rlist)                                          \
+    do {                                                                       \
+        (riter)->iter = rlist->used;                                           \
+        (riter)->veclist = rlist->veclist;                                     \
+    } while (0)
+
+#define rvec_for_each_entry(pos, riter)                                        \
+    for (pos = list_entry((riter)->veclist.next, typeof(*pos), list);          \
+         (riter)->iter > 0;                                                    \
+         pos = list_entry(pos->list.next, typeof(*pos), list),                 \
+        --((riter)->iter))
+
+/**
+ * Sequence number assignment routine is called during buffer
+ * switch under rbuff ->lock.
+ */
+typedef void(sequence_fn)(rbuf_list_t *, void *);
+
+#define RLIST_STORE_SEQ(rlist, start, range)                                   \
+    do {                                                                       \
+        rlist->seq[0] = start;                                                 \
+        rlist->seq[1] = range;                                                 \
+    } while (0)
+
+#define RLIST_GET_SEQ(rlist, start, range)                                     \
+    do {                                                                       \
+        start = rlist->seq[0];                                                 \
+        range = rlist->seq[1];                                                 \
+    } while (0)
+
+typedef struct rbuf {
+    gf_lock_t lock; /* protects "current" rlist */
+
+    rbuf_list_t *current; /* cached pointer to first free rlist */
+
+    struct list_head freelist;
+} rbuf_t;
+
+typedef enum {
+    RBUF_CONSUMABLE = 1,
+    RBUF_BUSY,
+    RBUF_EMPTY,
+    RBUF_WOULD_STARVE,
+} rlist_retval_t;
+
+/* Initialization/Destruction */
+rbuf_t *
+rbuf_init(int);
+void
+rbuf_dtor(rbuf_t *);
+
+/* Producer API */
+char *
+rbuf_reserve_write_area(rbuf_t *, size_t, void **);
+int
+rbuf_write_complete(void *);
+
+/* Consumer API */
+int
+rbuf_get_buffer(rbuf_t *, void **, sequence_fn *, void *);
+int
+rbuf_wait_for_completion(rbuf_t *, void *, void (*)(rbuf_list_t *, void *),
+                         void *);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/run.h b/libglusterfs/src/glusterfs/run.h
new file mode 100644
index 00000000000..76af95fd27f
--- /dev/null
+++ b/libglusterfs/src/glusterfs/run.h
@@ -0,0 +1,207 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __RUN_H__
+#define __RUN_H__
+
+#define RUN_PIPE -1
+
+struct runner {
+    char **argv;
+    unsigned argvlen;
+    int runerr;
+    pid_t chpid;
+    int chfd[3];
+    FILE *chio[3];
+};
+
+typedef struct runner runner_t;
+
+/**
+ * initialize runner_t instance.
+ *
+ * @param runner pointer to runner_t instance
+ */
+void
+runinit(runner_t *runner);
+
+/**
+ * get FILE pointer to which child's stdio is redirected.
+ *
+ * @param runner pointer to runner_t instance
+ * @param fd specifies which standard file descriptor is
+ *        is asked for
+ *
+ * @see runner_redir()
+ */
+FILE *
+runner_chio(runner_t *runner, int fd);
+
+/**
+ * add an argument.
+ *
+ * 'arg' is duplicated.
+ *
+ * Errors are deferred, no error handling is necessary.
+ *
+ * @param runner pointer to runner_t instance
+ * @param arg    command line argument
+ */
+void
+runner_add_arg(runner_t *runner, const char *arg);
+
+/**
+ * add a sequence of arguments.
+ *
+ * Variadic function, calls runner_add_arg() on each vararg.
+ * Argument sequence MUST be NULL terminated.
+ *
+ * Errors are deferred, no error handling is necessary.
+ *
+ * @param runner pointer to runner_t instance
+ *
+ * @see runner_add_arg()
+ */
+void
+runner_add_args(runner_t *runner, ...);
+
+/**
+ * add an argument with printf style formatting.
+ *
+ * Errors are deferred, no error handling is necessary.
+ *
+ * @param runner pointer to runner_t instance
+ * @param format printf style format specifier
+ */
+void
+runner_argprintf(runner_t *runner, const char *format, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+/**
+ * log a message about the command to be run.
+ *
+ * @param runner  pointer to runner_t instance
+ *
+ * @param dom  log domain
+ * @param lvl  log level
+ * @param msg  message with which the command is prefixed in log
+ *
+ * @see gf_log()
+ */
+void
+runner_log(runner_t *runner, const char *dom, gf_loglevel_t lvl,
+           const char *msg);
+
+/**
+ * set up redirection for child.
+ *
+ * @param runner  pointer to runner_t instance
+ *
+ * @param fd      fd of child to redirect (0, 1, or 2)
+ * @param tgt_fd  fd on parent side to redirect to.
+ *                Note that runner_end() will close tgt_fd,
+ *                if user needs it in another context it should
+ *                be dup'd beforehand.
+ *                RUN_PIPE can be used for requiring a
+ *                pipe from child to parent. The FILE
+ *                created for this purpose will be
+ *                accessible via runner_chio() (after
+ *                runner_start() has been invoked).
+ *
+ * @see  runner_start(), dup(2), runner_chio(), runner_start()
+ */
+void
+runner_redir(runner_t *runner, int fd, int tgt_fd);
+
+/**
+ * spawn child with accumulated arg list.
+ *
+ * @param runner  pointer to runner_t instance
+ *
+ * @return  0 on successful spawn
+ *          -1 on failure (either due to earlier errors or execve(2) failing)
+ *
+ * @see runner_cout()
+ */
+int
+runner_start(runner_t *runner);
+
+/**
+ * complete operation and free resources.
+ *
+ * If child exists, waits for it. Redirections will be closed.
+ * Dynamically allocated memory shall be freed.
+ *
+ * @param runner  pointer to runner_t instance
+ *
+ * @return  0 if child terminated successfully
+ *          -1 if there is no running child
+ *          n > 0 if child failed; value to be interpreted as status
+ *                in waitpid(2)
+ *
+ * @see waitpid(2)
+ */
+int
+runner_end(runner_t *runner);
+
+/**
+ * variant of runner_end() which does not free internal data
+ * so that the runner instance can be run again.
+ *
+ * @see runner_end()
+ */
+int
+runner_end_reuse(runner_t *runner);
+
+/**
+ * spawn and child, take it to completion and free resources.
+ *
+ * Essentially it's a concatenation of runner_start() and runner_end()
+ * with simplified return semantics.
+ *
+ * @param runner  pointer to runner_t instance
+ *
+ * @return  0 on success
+ *          -1 on failuire
+ *
+ * @see runner_start(), runner_end()
+ */
+int
+runner_run(runner_t *runner);
+
+/**
+ * variant for runner_run() which does not wait for acknowledgement
+ * from child, and always assumes it succeeds.
+ */
+int
+runner_run_nowait(runner_t *runner);
+
+/**
+ * variant of runner_run() which does not free internal data
+ * so that the runner instance can be run again.
+ *
+ * @see runner_run()
+ */
+int
+runner_run_reuse(runner_t *runner);
+
+/**
+ * run a command with args.
+ *
+ * Variadic function, child process is spawned with
+ * the given sequence of args and waited for.
+ * Argument sequence MUST be NULL terminated.
+ *
+ * @return 0 on success
+ *         -1 on failure
+ */
+int
+runcmd(const char *arg, ...);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/stack.h b/libglusterfs/src/glusterfs/stack.h
new file mode 100644
index 00000000000..536a330d38b
--- /dev/null
+++ b/libglusterfs/src/glusterfs/stack.h
@@ -0,0 +1,555 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+  This file defines MACROS and static inlines used to emulate a function
+  call over asynchronous communication with remote server
+*/
+
+#ifndef _STACK_H
+#define _STACK_H
+
+struct _call_stack;
+typedef struct _call_stack call_stack_t;
+struct _call_frame;
+typedef struct _call_frame call_frame_t;
+struct call_pool;
+typedef struct call_pool call_pool_t;
+
+#include <sys/time.h>
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/dict.h"
+#include "glusterfs/list.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/lkowner.h"
+#include "glusterfs/client_t.h"
+#include "glusterfs/libglusterfs-messages.h"
+#include "glusterfs/timespec.h"
+
+#define NFS_PID 1
+#define LOW_PRIO_PROC_PID -1
+
+#define STACK_ERR_XL_NAME(stack) (stack->err_xl ? stack->err_xl->name : "-")
+#define STACK_CLIENT_NAME(stack)                                               \
+    (stack->client ? stack->client->client_uid : "-")
+
+typedef int32_t (*ret_fn_t)(call_frame_t *frame, call_frame_t *prev_frame,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            ...);
+
+void
+gf_frame_latency_update(call_frame_t *frame);
+
+struct call_pool {
+    union {
+        struct list_head all_frames;
+        struct {
+            call_stack_t *next_call;
+            call_stack_t *prev_call;
+        } all_stacks;
+    };
+    int64_t cnt;
+    gf_atomic_t total_count;
+    gf_lock_t lock;
+    struct mem_pool *frame_mem_pool;
+    struct mem_pool *stack_mem_pool;
+};
+
+struct _call_frame {
+    call_stack_t *root;   /* stack root */
+    call_frame_t *parent; /* previous BP */
+    struct list_head frames;
+    void *local;    /* local variables */
+    xlator_t *this; /* implicit object */
+    ret_fn_t ret;   /* op_return address */
+    int32_t ref_count;
+    gf_lock_t lock;
+    void *cookie; /* unique cookie */
+    gf_boolean_t complete;
+
+    glusterfs_fop_t op;
+    struct timespec begin; /* when this frame was created */
+    struct timespec end;   /* when this frame completed */
+    const char *wind_from;
+    const char *wind_to;
+    const char *unwind_from;
+    const char *unwind_to;
+};
+
+struct _ns_info {
+    uint32_t hash;      /* Hash of the namespace from SuperFastHash */
+    gf_boolean_t found; /* Set to true if we found a namespace */
+};
+
+typedef struct _ns_info ns_info_t;
+
+#define SMALL_GROUP_COUNT 128
+
+struct _call_stack {
+    union {
+        struct list_head all_frames;
+        struct {
+            call_stack_t *next_call;
+            call_stack_t *prev_call;
+        };
+    };
+    call_pool_t *pool;
+    gf_lock_t stack_lock;
+    client_t *client;
+    uint64_t unique;
+    void *state; /* pointer to request state */
+    uid_t uid;
+    gid_t gid;
+    pid_t pid;
+    char identifier[UNIX_PATH_MAX];
+    uint16_t ngrps;
+    uint32_t groups_small[SMALL_GROUP_COUNT];
+    uint32_t *groups_large;
+    uint32_t *groups;
+    gf_lkowner_t lk_owner;
+    glusterfs_ctx_t *ctx;
+
+    struct list_head myframes; /* List of call_frame_t that go
+                                  to make the call stack */
+
+    int32_t op;
+    int8_t type;
+    struct timespec tv;
+    xlator_t *err_xl;
+    int32_t error;
+
+    uint32_t flags;        /* use it wisely, think of it as a mechanism to
+                              send information over the wire too */
+    struct timespec ctime; /* timestamp, most probably set at
+                              creation of stack. */
+
+    ns_info_t ns_info;
+};
+
+/* call_stack flags field users */
+#define MDATA_CTIME (1 << 0)
+#define MDATA_MTIME (1 << 1)
+#define MDATA_ATIME (1 << 2)
+#define MDATA_PAR_CTIME (1 << 3)
+#define MDATA_PAR_MTIME (1 << 4)
+#define MDATA_PAR_ATIME (1 << 5)
+
+#define frame_set_uid_gid(frm, u, g)                                           \
+    do {                                                                       \
+        if (frm) {                                                             \
+            (frm)->root->uid = u;                                              \
+            (frm)->root->gid = g;                                              \
+            (frm)->root->ngrps = 0;                                            \
+        }                                                                      \
+    } while (0);
+
+struct xlator_fops;
+
+static inline void
+FRAME_DESTROY(call_frame_t *frame)
+{
+    void *local = NULL;
+
+    if (frame->root->ctx->measure_latency)
+        gf_frame_latency_update(frame);
+
+    list_del_init(&frame->frames);
+    if (frame->local) {
+        local = frame->local;
+        frame->local = NULL;
+    }
+
+    LOCK_DESTROY(&frame->lock);
+    mem_put(frame);
+
+    if (local)
+        mem_put(local);
+}
+
+static inline void
+STACK_DESTROY(call_stack_t *stack)
+{
+    call_frame_t *frame = NULL;
+    call_frame_t *tmp = NULL;
+
+    LOCK(&stack->pool->lock);
+    {
+        list_del_init(&stack->all_frames);
+        stack->pool->cnt--;
+    }
+    UNLOCK(&stack->pool->lock);
+
+    LOCK_DESTROY(&stack->stack_lock);
+
+    list_for_each_entry_safe(frame, tmp, &stack->myframes, frames)
+    {
+        FRAME_DESTROY(frame);
+    }
+
+    GF_FREE(stack->groups_large);
+
+    mem_put(stack);
+}
+
+static inline void
+STACK_RESET(call_stack_t *stack)
+{
+    call_frame_t *frame = NULL;
+    call_frame_t *tmp = NULL;
+    call_frame_t *last = NULL;
+    struct list_head toreset = {0};
+
+    INIT_LIST_HEAD(&toreset);
+
+    /* We acquire call_pool->lock only to remove the frames from this stack
+     * to preserve atomicity. This synchronizes across concurrent requests
+     * like statedump, STACK_DESTROY etc. */
+
+    LOCK(&stack->pool->lock);
+    {
+        last = list_last_entry(&stack->myframes, call_frame_t, frames);
+        list_del_init(&last->frames);
+        list_splice_init(&stack->myframes, &toreset);
+        list_add(&last->frames, &stack->myframes);
+    }
+    UNLOCK(&stack->pool->lock);
+
+    list_for_each_entry_safe(frame, tmp, &toreset, frames)
+    {
+        FRAME_DESTROY(frame);
+    }
+}
+
+#define FRAME_SU_DO(frm, local_type)                                           \
+    do {                                                                       \
+        local_type *__local = (frm)->local;                                    \
+        __local->uid = frm->root->uid;                                         \
+        __local->gid = frm->root->gid;                                         \
+        __local->pid = frm->root->pid;                                         \
+        frm->root->uid = 0;                                                    \
+        frm->root->gid = 0;                                                    \
+        frm->root->pid = GF_CLIENT_PID_NO_ROOT_SQUASH;                         \
+    } while (0);
+
+#define FRAME_SU_UNDO(frm, local_type)                                         \
+    do {                                                                       \
+        local_type *__local = (frm)->local;                                    \
+        frm->root->uid = __local->uid;                                         \
+        frm->root->gid = __local->gid;                                         \
+        frm->root->pid = __local->pid;                                         \
+    } while (0);
+
+/* NOTE: make sure to keep this as an macro, mainly because, we need 'fn'
+   field here to be the proper fn ptr, so its address is valid entry in
+   'xlator_fops' struct.
+   To understand this, check the `xlator.h:struct xlator_fops`, and then
+   see a STACK_WIND call, which generally calls `subvol->fops->fop`, so
+   the address offset should give the index */
+
+/* +1 is required as 0 means NULL fop, and we don't have a variable for it */
+#define get_fop_index_from_fn(xl, fn)                                          \
+    (1 + (((long)&(fn) - (long)&((xl)->fops->stat)) / sizeof(void *)))
+
+/* NOTE: the above reason holds good here too. But notice that we are getting
+ the base address of the 'stat' fop, which is the first entry in the fop
+ structure. All we need to do is move as much as 'idx' fields, and get the
+ actual pointer from that field. */
+
+static inline void *
+get_the_pt_fop(void *base_fop, int fop_idx)
+{
+    void *target_addr = (base_fop + ((fop_idx - 1) * sizeof(void *)));
+    /* all below type casting is for not getting warning. */
+    return (void *)*(unsigned long *)target_addr;
+}
+
+/* make a call without switching frames */
+#define STACK_WIND_TAIL(frame, obj, fn, params...)                             \
+    do {                                                                       \
+        xlator_t *old_THIS = NULL;                                             \
+        xlator_t *next_xl = obj;                                               \
+        typeof(fn) next_xl_fn = fn;                                            \
+        int opn = get_fop_index_from_fn((next_xl), (fn));                      \
+                                                                               \
+        frame->this = next_xl;                                                 \
+        frame->wind_to = #fn;                                                  \
+        old_THIS = THIS;                                                       \
+        THIS = next_xl;                                                        \
+        gf_msg_trace("stack-trace", 0,                                         \
+                     "stack-address: %p, "                                     \
+                     "winding from %s to %s",                                  \
+                     frame->root, old_THIS->name, THIS->name);                 \
+        /* Need to capture counts at leaf node */                              \
+        if (!next_xl->pass_through && !next_xl->children) {                    \
+            GF_ATOMIC_INC(next_xl->stats.total.metrics[opn].fop);              \
+            GF_ATOMIC_INC(next_xl->stats.interval.metrics[opn].fop);           \
+            GF_ATOMIC_INC(next_xl->stats.total.count);                         \
+            GF_ATOMIC_INC(next_xl->stats.interval.count);                      \
+        }                                                                      \
+                                                                               \
+        if (next_xl->pass_through) {                                           \
+            next_xl_fn = get_the_pt_fop(&next_xl->pass_through_fops->stat,     \
+                                        opn);                                  \
+        }                                                                      \
+        next_xl_fn(frame, next_xl, params);                                    \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+/* make a call */
+#define STACK_WIND(frame, rfn, obj, fn, params...)                             \
+    STACK_WIND_COMMON(frame, rfn, 0, NULL, obj, fn, params)
+
+/* make a call with a cookie */
+#define STACK_WIND_COOKIE(frame, rfn, cky, obj, fn, params...)                 \
+    STACK_WIND_COMMON(frame, rfn, 1, cky, obj, fn, params)
+
+/* Cookie passed as the argument can be NULL (ptr) or 0 (int). Hence we
+   have to have a mechanism to separate out the two STACK_WIND formats.
+   Needed a common macro, as other than for cookie, all the other code
+   is common across.
+ */
+#define STACK_WIND_COMMON(frame, rfn, has_cookie, cky, obj, fn, params...)     \
+    do {                                                                       \
+        call_frame_t *_new = NULL;                                             \
+        xlator_t *old_THIS = NULL;                                             \
+        typeof(fn) next_xl_fn = fn;                                            \
+                                                                               \
+        _new = mem_get0(frame->root->pool->frame_mem_pool);                    \
+        if (!_new) {                                                           \
+            break;                                                             \
+        }                                                                      \
+        typeof(fn##_cbk) tmp_cbk = rfn;                                        \
+        _new->root = frame->root;                                              \
+        _new->this = obj;                                                      \
+        _new->ret = (ret_fn_t)tmp_cbk;                                         \
+        _new->parent = frame;                                                  \
+        /* (void *) is required for avoiding gcc warning */                    \
+        _new->cookie = ((has_cookie == 1) ? (void *)(cky) : (void *)_new);     \
+        _new->wind_from = __FUNCTION__;                                        \
+        _new->wind_to = #fn;                                                   \
+        _new->unwind_to = #rfn;                                                \
+        LOCK_INIT(&_new->lock);                                                \
+        LOCK(&frame->root->stack_lock);                                        \
+        {                                                                      \
+            list_add(&_new->frames, &frame->root->myframes);                   \
+            frame->ref_count++;                                                \
+        }                                                                      \
+        UNLOCK(&frame->root->stack_lock);                                      \
+        fn##_cbk = rfn;                                                        \
+        old_THIS = THIS;                                                       \
+        THIS = obj;                                                            \
+        gf_msg_trace("stack-trace", 0,                                         \
+                     "stack-address: %p, "                                     \
+                     "winding from %s to %s",                                  \
+                     frame->root, old_THIS->name, THIS->name);                 \
+        if (obj->ctx->measure_latency)                                         \
+            timespec_now(&_new->begin);                                        \
+        _new->op = get_fop_index_from_fn((_new->this), (fn));                  \
+        if (!obj->pass_through) {                                              \
+            GF_ATOMIC_INC(obj->stats.total.metrics[_new->op].fop);             \
+            GF_ATOMIC_INC(obj->stats.interval.metrics[_new->op].fop);          \
+            GF_ATOMIC_INC(obj->stats.total.count);                             \
+            GF_ATOMIC_INC(obj->stats.interval.count);                          \
+        } else {                                                               \
+            /* we want to get to the actual fop to call */                     \
+            next_xl_fn = get_the_pt_fop(&obj->pass_through_fops->stat,         \
+                                        _new->op);                             \
+        }                                                                      \
+        next_xl_fn(_new, obj, params);                                         \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define STACK_UNWIND STACK_UNWIND_STRICT
+
+/* return from function in type-safe way */
+#define STACK_UNWIND_STRICT(fop, frame, op_ret, op_errno, params...)           \
+    do {                                                                       \
+        fop_##fop##_cbk_t fn = NULL;                                           \
+        call_frame_t *_parent = NULL;                                          \
+        xlator_t *old_THIS = NULL;                                             \
+                                                                               \
+        if (!frame) {                                                          \
+            gf_msg("stack", GF_LOG_CRITICAL, 0, LG_MSG_FRAME_ERROR, "!frame"); \
+            break;                                                             \
+        }                                                                      \
+        if ((op_ret) < 0) {                                                    \
+            gf_msg_debug("stack-trace", op_errno,                              \
+                         "stack-address: %p, "                                 \
+                         "%s returned %d error: %s",                           \
+                         frame->root, THIS->name, (int32_t)(op_ret),           \
+                         strerror(op_errno));                                  \
+        } else {                                                               \
+            gf_msg_trace("stack-trace", 0,                                     \
+                         "stack-address: %p, "                                 \
+                         "%s returned %d",                                     \
+                         frame->root, THIS->name, (int32_t)(op_ret));          \
+        }                                                                      \
+        fn = (fop_##fop##_cbk_t)frame->ret;                                    \
+        _parent = frame->parent;                                               \
+        LOCK(&frame->root->stack_lock);                                        \
+        {                                                                      \
+            _parent->ref_count--;                                              \
+            if ((op_ret) < 0 && (op_errno) != frame->root->error) {            \
+                frame->root->err_xl = frame->this;                             \
+                frame->root->error = (op_errno);                               \
+            } else if ((op_ret) == 0) {                                        \
+                frame->root->err_xl = NULL;                                    \
+                frame->root->error = 0;                                        \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&frame->root->stack_lock);                                      \
+        old_THIS = THIS;                                                       \
+        THIS = _parent->this;                                                  \
+        frame->complete = _gf_true;                                            \
+        frame->unwind_from = __FUNCTION__;                                     \
+        if (frame->this->ctx->measure_latency) {                               \
+            timespec_now(&frame->end);                                         \
+            /* required for top most xlator */                                 \
+            if (_parent->ret == NULL)                                          \
+                timespec_now(&_parent->end);                                   \
+        }                                                                      \
+        if (op_ret < 0) {                                                      \
+            GF_ATOMIC_INC(THIS->stats.total.metrics[frame->op].cbk);           \
+            GF_ATOMIC_INC(THIS->stats.interval.metrics[frame->op].cbk);        \
+        }                                                                      \
+        fn(_parent, frame->cookie, _parent->this, op_ret, op_errno, params);   \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+static inline int
+call_stack_alloc_groups(call_stack_t *stack, int ngrps)
+{
+    if (ngrps <= SMALL_GROUP_COUNT) {
+        stack->groups = stack->groups_small;
+    } else {
+        GF_FREE(stack->groups_large);
+        stack->groups_large = GF_CALLOC(ngrps, sizeof(gid_t),
+                                        gf_common_mt_groups_t);
+        if (!stack->groups_large)
+            return -1;
+        stack->groups = stack->groups_large;
+    }
+
+    stack->ngrps = ngrps;
+
+    return 0;
+}
+
+static inline int
+call_stack_groups_capacity(call_stack_t *stack)
+{
+    return max(stack->ngrps, SMALL_GROUP_COUNT);
+}
+
+static inline int
+call_frames_count(call_stack_t *call_stack)
+{
+    call_frame_t *pos;
+    int32_t count = 0;
+
+    if (!call_stack)
+        return count;
+
+    list_for_each_entry(pos, &call_stack->myframes, frames) count++;
+
+    return count;
+}
+
+static inline call_frame_t *
+copy_frame(call_frame_t *frame)
+{
+    call_stack_t *newstack = NULL;
+    call_stack_t *oldstack = NULL;
+    call_frame_t *newframe = NULL;
+
+    if (!frame) {
+        return NULL;
+    }
+
+    newstack = mem_get0(frame->root->pool->stack_mem_pool);
+    if (newstack == NULL) {
+        return NULL;
+    }
+
+    INIT_LIST_HEAD(&newstack->myframes);
+
+    newframe = mem_get0(frame->root->pool->frame_mem_pool);
+    if (!newframe) {
+        mem_put(newstack);
+        return NULL;
+    }
+
+    newframe->this = frame->this;
+    newframe->root = newstack;
+    INIT_LIST_HEAD(&newframe->frames);
+    list_add(&newframe->frames, &newstack->myframes);
+
+    oldstack = frame->root;
+
+    newstack->uid = oldstack->uid;
+    newstack->gid = oldstack->gid;
+    newstack->pid = oldstack->pid;
+    newstack->op = oldstack->op;
+    newstack->type = oldstack->type;
+    newstack->ctime = oldstack->ctime;
+    newstack->flags = oldstack->flags;
+    if (call_stack_alloc_groups(newstack, oldstack->ngrps) != 0) {
+        mem_put(newstack);
+        return NULL;
+    }
+    if (!oldstack->groups) {
+        gf_msg_debug("stack", EINVAL, "groups is null (ngrps: %d)",
+                     oldstack->ngrps);
+        /* Considering 'groups' is NULL, set ngrps to 0 */
+        oldstack->ngrps = 0;
+
+        if (oldstack->groups_large)
+            oldstack->groups = oldstack->groups_large;
+        else
+            oldstack->groups = oldstack->groups_small;
+    }
+    newstack->ngrps = oldstack->ngrps;
+    memcpy(newstack->groups, oldstack->groups, sizeof(gid_t) * oldstack->ngrps);
+    newstack->unique = oldstack->unique;
+    newstack->pool = oldstack->pool;
+    newstack->lk_owner = oldstack->lk_owner;
+    newstack->ctx = oldstack->ctx;
+
+    if (newstack->ctx->measure_latency) {
+        timespec_now(&newstack->tv);
+        memcpy(&newframe->begin, &newstack->tv, sizeof(newstack->tv));
+    }
+
+    LOCK_INIT(&newframe->lock);
+    LOCK_INIT(&newstack->stack_lock);
+
+    LOCK(&oldstack->pool->lock);
+    {
+        list_add(&newstack->all_frames, &oldstack->all_frames);
+        newstack->pool->cnt++;
+    }
+    UNLOCK(&oldstack->pool->lock);
+    GF_ATOMIC_INC(newstack->pool->total_count);
+
+    return newframe;
+}
+
+void
+call_stack_set_groups(call_stack_t *stack, int ngrps, gid_t **groupbuf_p);
+void
+gf_proc_dump_pending_frames(call_pool_t *call_pool);
+void
+gf_proc_dump_pending_frames_to_dict(call_pool_t *call_pool, dict_t *dict);
+call_frame_t *
+create_frame(xlator_t *xl, call_pool_t *pool);
+gf_boolean_t
+__is_fuse_call(call_frame_t *frame);
+#endif /* _STACK_H */
diff --git a/libglusterfs/src/glusterfs/statedump.h b/libglusterfs/src/glusterfs/statedump.h
new file mode 100644
index 00000000000..ce082706bdf
--- /dev/null
+++ b/libglusterfs/src/glusterfs/statedump.h
@@ -0,0 +1,132 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef STATEDUMP_H
+#define STATEDUMP_H
+
+#include <stdarg.h>
+#include "glusterfs/inode.h"
+#include "glusterfs/strfd.h"
+
+#define GF_DUMP_MAX_BUF_LEN 4096
+
+typedef struct gf_dump_xl_options_ {
+    gf_boolean_t dump_priv;
+    gf_boolean_t dump_inode;
+    gf_boolean_t dump_fd;
+    gf_boolean_t dump_inodectx;
+    gf_boolean_t dump_fdctx;
+    gf_boolean_t dump_history;
+} gf_dump_xl_options_t;
+
+typedef struct gf_dump_options_ {
+    gf_boolean_t dump_mem;
+    gf_boolean_t dump_iobuf;
+    gf_boolean_t dump_callpool;
+    gf_dump_xl_options_t xl_options;  // options for all xlators
+    char *dump_path;
+} gf_dump_options_t;
+
+extern gf_dump_options_t dump_options;
+
+__attribute__((__format__(__printf__, 3, 4))) static inline void
+_gf_proc_dump_build_key(char *key, const char *prefix, const char *fmt, ...)
+{
+    va_list ap;
+    int32_t len;
+
+    len = snprintf(key, GF_DUMP_MAX_BUF_LEN, "%s.", prefix);
+    if (len >= 0) {
+        va_start(ap, fmt);
+        len = vsnprintf(key + len, GF_DUMP_MAX_BUF_LEN - len, fmt, ap);
+        va_end(ap);
+    }
+    if (len < 0) {
+        *key = 0;
+    }
+}
+
+#define gf_proc_dump_build_key(key, key_prefix, fmt...)                        \
+    {                                                                          \
+        _gf_proc_dump_build_key(key, key_prefix, ##fmt);                       \
+    }
+
+#define GF_PROC_DUMP_SET_OPTION(opt, val) opt = val
+
+#define GF_CHECK_DUMP_OPTION_ENABLED(option_dump, var, label)                  \
+    do {                                                                       \
+        if (option_dump == _gf_true) {                                         \
+            var = _gf_false;                                                   \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+void
+gf_proc_dump_init();
+
+void
+gf_proc_dump_fini(void);
+
+void
+gf_proc_dump_cleanup(void);
+
+void
+gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx);
+
+int
+gf_proc_dump_add_section(char *key, ...)
+    __attribute__((__format__(__printf__, 1, 2)));
+
+int
+gf_proc_dump_write(char *key, char *value, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+void
+inode_table_dump(inode_table_t *itable, char *prefix);
+
+void
+inode_table_dump_to_dict(inode_table_t *itable, char *prefix, dict_t *dict);
+
+void
+fdtable_dump(fdtable_t *fdtable, char *prefix);
+
+void
+fdtable_dump_to_dict(fdtable_t *fdtable, char *prefix, dict_t *dict);
+
+void
+inode_dump(inode_t *inode, char *prefix);
+
+void
+gf_proc_dump_mem_info_to_dict(dict_t *dict);
+
+void
+gf_proc_dump_mempool_info_to_dict(glusterfs_ctx_t *ctx, dict_t *dict);
+
+void
+glusterd_init(int sig);
+
+void
+gf_proc_dump_xlator_private(xlator_t *this, strfd_t *strfd);
+
+void
+gf_proc_dump_mallinfo(strfd_t *strfd);
+
+void
+gf_proc_dump_xlator_history(xlator_t *this, strfd_t *strfd);
+
+void
+gf_proc_dump_xlator_meminfo(xlator_t *this, strfd_t *strfd);
+
+void
+gf_proc_dump_xlator_profile(xlator_t *this, strfd_t *strfd);
+
+void
+gf_latency_statedump_and_reset(char *key, gf_latency_t *lat);
+#endif /* STATEDUMP_H */
diff --git a/libglusterfs/src/glusterfs/store.h b/libglusterfs/src/glusterfs/store.h
new file mode 100644
index 00000000000..a1f70c7b840
--- /dev/null
+++ b/libglusterfs/src/glusterfs/store.h
@@ -0,0 +1,112 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_STORE_H_
+#define _GLUSTERD_STORE_H_
+
+#include "glusterfs/compat.h"
+#include "glusterfs/glusterfs.h"
+
+struct gf_store_handle_ {
+    char *path;
+    int fd;
+    int tmp_fd;
+    FILE *read;
+    int locked; /* state of lockf() */
+};
+
+typedef struct gf_store_handle_ gf_store_handle_t;
+
+struct gf_store_iter_ {
+    FILE *file;
+    char filepath[PATH_MAX];
+};
+
+typedef struct gf_store_iter_ gf_store_iter_t;
+
+typedef enum {
+    GD_STORE_SUCCESS,
+    GD_STORE_KEY_NULL,
+    GD_STORE_VALUE_NULL,
+    GD_STORE_KEY_VALUE_NULL,
+    GD_STORE_EOF,
+    GD_STORE_ENOMEM,
+    GD_STORE_STAT_FAILED
+} gf_store_op_errno_t;
+
+int32_t
+gf_store_mkdir(char *path);
+
+int32_t
+gf_store_handle_create_on_absence(gf_store_handle_t **shandle, char *path);
+
+int32_t
+gf_store_mkstemp(gf_store_handle_t *shandle);
+
+int
+gf_store_sync_direntry(char *path);
+
+int32_t
+gf_store_rename_tmppath(gf_store_handle_t *shandle);
+
+int32_t
+gf_store_unlink_tmppath(gf_store_handle_t *shandle);
+
+int
+gf_store_read_and_tokenize(FILE *file, char **iter_key, char **iter_val,
+                           gf_store_op_errno_t *store_errno);
+
+int32_t
+gf_store_retrieve_value(gf_store_handle_t *handle, char *key, char **value);
+
+int32_t
+gf_store_save_value(int fd, char *key, char *value);
+
+int32_t
+gf_store_save_items(int fd, char *items);
+
+int32_t
+gf_store_handle_new(const char *path, gf_store_handle_t **handle);
+
+int
+gf_store_handle_retrieve(char *path, gf_store_handle_t **handle);
+
+int32_t
+gf_store_handle_destroy(gf_store_handle_t *handle);
+
+int32_t
+gf_store_iter_new(gf_store_handle_t *shandle, gf_store_iter_t **iter);
+
+int32_t
+gf_store_validate_key_value(char *storepath, char *key, char *val,
+                            gf_store_op_errno_t *op_errno);
+
+int32_t
+gf_store_iter_get_next(gf_store_iter_t *iter, char **key, char **value,
+                       gf_store_op_errno_t *op_errno);
+
+int32_t
+gf_store_iter_get_matching(gf_store_iter_t *iter, char *key, char **value);
+
+int32_t
+gf_store_iter_destroy(gf_store_iter_t **iter);
+
+char *
+gf_store_strerror(gf_store_op_errno_t op_errno);
+
+int
+gf_store_lock(gf_store_handle_t *sh);
+
+void
+gf_store_unlock(gf_store_handle_t *sh);
+
+int
+gf_store_locked_local(gf_store_handle_t *sh);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/strfd.h b/libglusterfs/src/glusterfs/strfd.h
new file mode 100644
index 00000000000..861cd02e005
--- /dev/null
+++ b/libglusterfs/src/glusterfs/strfd.h
@@ -0,0 +1,34 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _STRFD_H
+#define _STRFD_H
+
+typedef struct {
+    void *data;
+    size_t alloc_size;
+    size_t size;
+    off_t pos;
+} strfd_t;
+
+strfd_t *
+strfd_open();
+
+int
+strprintf(strfd_t *strfd, const char *fmt, ...)
+    __attribute__((__format__(__printf__, 2, 3)));
+
+int
+strvprintf(strfd_t *strfd, const char *fmt, va_list ap);
+
+int
+strfd_close(strfd_t *strfd);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/syncop-utils.h b/libglusterfs/src/glusterfs/syncop-utils.h
new file mode 100644
index 00000000000..1f3ee403edc
--- /dev/null
+++ b/libglusterfs/src/glusterfs/syncop-utils.h
@@ -0,0 +1,54 @@
+/*
+  Copyright (c) 2015, Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _SYNCOP_UTILS_H
+#define _SYNCOP_UTILS_H
+
+typedef int (*syncop_dir_scan_fn_t)(xlator_t *subvol, gf_dirent_t *entry,
+                                    loc_t *parent, void *data);
+int
+syncop_ftw(xlator_t *subvol, loc_t *loc, int pid, void *data,
+           int (*fn)(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                     void *data));
+
+int
+syncop_mt_dir_scan(call_frame_t *frame, xlator_t *subvol, loc_t *loc, int pid,
+                   void *data, syncop_dir_scan_fn_t fn, dict_t *xdata,
+                   uint32_t max_jobs, uint32_t max_qlen);
+
+int
+syncop_dir_scan(xlator_t *subvol, loc_t *loc, int pid, void *data,
+                int (*fn)(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                          void *data));
+
+int
+syncop_dirfd(xlator_t *subvol, loc_t *loc, fd_t **fd, int pid);
+
+int
+syncop_is_subvol_local(xlator_t *this, loc_t *loc, gf_boolean_t *is_local);
+
+int
+syncop_gfid_to_path(inode_table_t *itable, xlator_t *subvol, uuid_t gfid,
+                    char **path_p);
+
+int
+syncop_ftw_throttle(xlator_t *subvol, loc_t *loc, int pid, void *data,
+                    int (*fn)(xlator_t *subvol, gf_dirent_t *entry,
+                              loc_t *parent, void *data),
+                    int count, int sleep_time);
+int
+syncop_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid,
+                  inode_t **inode, dict_t *xdata, dict_t **rsp_dict);
+
+int
+syncop_gfid_to_path_hard(inode_table_t *itable, xlator_t *subvol, uuid_t gfid,
+                         inode_t *inode, char **path_p,
+                         gf_boolean_t hard_resolve);
+#endif /* _SYNCOP_H */
diff --git a/libglusterfs/src/glusterfs/syncop.h b/libglusterfs/src/glusterfs/syncop.h
new file mode 100644
index 00000000000..4e9241a32fc
--- /dev/null
+++ b/libglusterfs/src/glusterfs/syncop.h
@@ -0,0 +1,718 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _SYNCOP_H
+#define _SYNCOP_H
+
+#include <sys/time.h>
+#include <pthread.h>
+#include <ucontext.h>
+#include "glusterfs/dict.h"   // for dict_t
+#include "glusterfs/stack.h"  // for call_frame_t, STACK_DESTROY, STACK_...
+#include "glusterfs/timer.h"
+
+#define SYNCENV_PROC_MAX 16
+#define SYNCENV_PROC_MIN 2
+#define SYNCPROC_IDLE_TIME 600
+
+/*
+ * Flags for syncopctx valid elements
+ */
+#define SYNCOPCTX_UID 0x00000001
+#define SYNCOPCTX_GID 0x00000002
+#define SYNCOPCTX_GROUPS 0x00000004
+#define SYNCOPCTX_PID 0x00000008
+#define SYNCOPCTX_LKOWNER 0x00000010
+
+#ifdef HAVE_TSAN_API
+/* Currently hardcoded within thread context maintained by the sanitizer. */
+#define TSAN_THREAD_NAMELEN 64
+#endif
+
+struct synctask;
+struct syncproc;
+struct syncenv;
+struct synccond;
+
+typedef int (*synctask_cbk_t)(int ret, call_frame_t *frame, void *opaque);
+
+typedef int (*synctask_fn_t)(void *opaque);
+
+typedef enum {
+    SYNCTASK_INIT = 0,
+    SYNCTASK_RUN,
+    SYNCTASK_SUSPEND,
+    SYNCTASK_WAIT,
+    SYNCTASK_DONE,
+    SYNCTASK_ZOMBIE,
+} synctask_state_t;
+
+/* for one sequential execution of @syncfn */
+struct synctask {
+    struct list_head all_tasks;
+    struct syncenv *env;
+    xlator_t *xl;
+    call_frame_t *frame;
+    call_frame_t *opframe;
+    synctask_cbk_t synccbk;
+    synctask_fn_t syncfn;
+    struct timespec *delta;
+    gf_timer_t *timer;
+    struct synccond *synccond;
+    void *opaque;
+    void *stack;
+    synctask_state_t state;
+    int woken;
+    int slept;
+    int ret;
+
+    uid_t uid;
+    gid_t gid;
+
+#ifdef HAVE_TSAN_API
+    struct {
+        void *fiber;
+        char name[TSAN_THREAD_NAMELEN];
+    } tsan;
+#endif
+
+    ucontext_t ctx;
+    struct syncproc *proc;
+
+    pthread_mutex_t mutex; /* for synchronous spawning of synctask */
+    pthread_cond_t cond;
+    int done;
+
+    struct list_head waitq; /* can wait only "once" at a time */
+};
+
+struct syncproc {
+    pthread_t processor;
+
+#ifdef HAVE_TSAN_API
+    struct {
+        void *fiber;
+        char name[TSAN_THREAD_NAMELEN];
+    } tsan;
+#endif
+
+    ucontext_t sched;
+    struct syncenv *env;
+    struct synctask *current;
+};
+
+/* hosts the scheduler thread and framework for executing synctasks */
+struct syncenv {
+    struct syncproc proc[SYNCENV_PROC_MAX];
+
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+
+    struct list_head runq;
+    struct list_head waitq;
+
+    int procs;
+    int procs_idle;
+
+    int runcount;
+
+    int procmin;
+    int procmax;
+
+    size_t stacksize;
+
+    int destroy; /* FLAG to mark syncenv is in destroy mode
+                    so that no more synctasks are accepted*/
+};
+
+typedef enum { LOCK_NULL = 0, LOCK_TASK, LOCK_THREAD } lock_type_t;
+
+typedef enum {
+    SYNC_LOCK_DEFAULT = 0,
+    SYNC_LOCK_RECURSIVE, /*it allows recursive locking*/
+} lock_attr_t;
+
+struct synclock {
+    pthread_mutex_t guard;  /* guard the remaining members, pair @cond */
+    pthread_cond_t cond;    /* waiting non-synctasks */
+    struct list_head waitq; /* waiting synctasks */
+    volatile int lock;      /* true(non zero) or false(zero), lock status */
+    lock_attr_t attr;
+    struct synctask *owner; /* NULL if current owner is not a synctask */
+    pthread_t owner_tid;
+    lock_type_t type;
+};
+typedef struct synclock synclock_t;
+
+struct synccond {
+    pthread_mutex_t pmutex;
+    pthread_cond_t pcond;
+    struct list_head waitq;
+};
+typedef struct synccond synccond_t;
+
+struct syncbarrier {
+    gf_boolean_t initialized; /*Set on successful initialization*/
+    pthread_mutex_t guard;    /* guard the remaining members, pair @cond */
+    pthread_cond_t cond;      /* waiting non-synctasks */
+    struct list_head waitq;   /* waiting synctasks */
+    int count;                /* count the number of wakes */
+    int waitfor;              /* no. of wakes until which task can be in
+                                 waitq before being woken up. */
+};
+typedef struct syncbarrier syncbarrier_t;
+
+struct syncargs {
+    int op_ret;
+    int op_errno;
+
+    /*
+     * The below 3 iatt structures are used in the fops
+     * whose callbacks get struct iatt as one of the
+     * a return arguments. Currently, the maximum number
+     * of iatt structures returned is 3 for some fops
+     * such as mknod, copy_file_range, mkdir etc. So
+     * all the following 3 iatt structures would be used
+     * for those fops.
+     */
+    struct iatt iatt1;
+    struct iatt iatt2;
+    struct iatt iatt3;
+    dict_t *xattr;
+    struct statvfs statvfs_buf;
+    struct iovec *vector;
+    int count;
+    struct iobref *iobref;
+    char *buffer;
+    dict_t *xdata;
+    struct gf_flock flock;
+    struct gf_lease lease;
+    dict_t *dict_out;
+
+    /* some more _cbk needs */
+    uuid_t uuid;
+    char *errstr;
+    dict_t *dict;
+    pthread_mutex_t lock_dict;
+
+    syncbarrier_t barrier;
+
+    /* do not touch */
+    struct synctask *task;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    int done;
+
+    gf_dirent_t entries;
+    off_t offset;
+
+    lock_migration_info_t locklist;
+};
+
+struct syncopctx {
+    unsigned int valid; /* valid flags for elements that are set */
+    uid_t uid;
+    gid_t gid;
+    int grpsize;
+    int ngrps;
+    gid_t *groups;
+    pid_t pid;
+    gf_lkowner_t lk_owner;
+};
+
+#define __yawn(args)                                                           \
+    do {                                                                       \
+        args->task = synctask_get();                                           \
+        if (args->task)                                                        \
+            break;                                                             \
+        pthread_mutex_init(&args->mutex, NULL);                                \
+        pthread_cond_init(&args->cond, NULL);                                  \
+        args->done = 0;                                                        \
+    } while (0)
+
+#define __wake(args)                                                           \
+    do {                                                                       \
+        if (args->task) {                                                      \
+            synctask_wake(args->task);                                         \
+        } else {                                                               \
+            pthread_mutex_lock(&args->mutex);                                  \
+            {                                                                  \
+                args->done = 1;                                                \
+                pthread_cond_signal(&args->cond);                              \
+            }                                                                  \
+            pthread_mutex_unlock(&args->mutex);                                \
+        }                                                                      \
+    } while (0)
+
+#define __yield(args)                                                          \
+    do {                                                                       \
+        if (args->task) {                                                      \
+            synctask_yield(args->task, NULL);                                  \
+        } else {                                                               \
+            pthread_mutex_lock(&args->mutex);                                  \
+            {                                                                  \
+                while (!args->done)                                            \
+                    pthread_cond_wait(&args->cond, &args->mutex);              \
+            }                                                                  \
+            pthread_mutex_unlock(&args->mutex);                                \
+            pthread_mutex_destroy(&args->mutex);                               \
+            pthread_cond_destroy(&args->cond);                                 \
+        }                                                                      \
+    } while (0)
+
+#define SYNCOP(subvol, stb, cbk, fn_op, params...)                             \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+        call_frame_t *frame = NULL;                                            \
+                                                                               \
+        task = synctask_get();                                                 \
+        stb->task = task;                                                      \
+        if (task)                                                              \
+            frame = copy_frame(task->opframe);                                 \
+        else                                                                   \
+            frame = syncop_create_frame(THIS);                                 \
+                                                                               \
+        if (!frame) {                                                          \
+            stb->op_ret = -1;                                                  \
+            stb->op_errno = errno;                                             \
+            break;                                                             \
+        }                                                                      \
+                                                                               \
+        if (task) {                                                            \
+            frame->root->uid = task->uid;                                      \
+            frame->root->gid = task->gid;                                      \
+        }                                                                      \
+                                                                               \
+        __yawn(stb);                                                           \
+                                                                               \
+        frame->op = get_fop_index_from_fn(subvol, fn_op);                      \
+        STACK_WIND_COOKIE(frame, cbk, (void *)stb, subvol, fn_op, params);     \
+                                                                               \
+        __yield(stb);                                                          \
+        STACK_DESTROY(frame->root);                                            \
+    } while (0)
+
+/*
+ * syncop_xxx() calls are executed in two ways, one is inside a synctask where
+ * the executing function will do 'swapcontext' and the other is without
+ * synctask where the executing thread is made to wait using pthread_cond_wait.
+ * Executing thread may change when syncop_xxx() is executed inside a synctask.
+ * This leads to errno_location change i.e. errno may give errno of
+ * non-executing thread. So errno is not touched inside a synctask execution.
+ * All gfapi calls are executed using the second way of executing syncop_xxx()
+ * where the executing thread waits using pthread_cond_wait so it is ok to set
+ * errno in these cases. The following macro makes syncop_xxx() behave just
+ * like a system call, where -1 is returned and errno is set when a failure
+ * occurs.
+ */
+#define DECODE_SYNCOP_ERR(ret)                                                 \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            errno = -ret;                                                      \
+            ret = -1;                                                          \
+        } else {                                                               \
+            errno = 0;                                                         \
+        }                                                                      \
+    } while (0)
+
+#define SYNCENV_DEFAULT_STACKSIZE (2 * 1024 * 1024)
+
+struct syncenv *
+syncenv_new(size_t stacksize, int procmin, int procmax);
+void
+syncenv_destroy(struct syncenv *);
+void
+syncenv_scale(struct syncenv *env);
+
+int
+synctask_new1(struct syncenv *, size_t stacksize, synctask_fn_t, synctask_cbk_t,
+              call_frame_t *frame, void *);
+int
+synctask_new(struct syncenv *, synctask_fn_t, synctask_cbk_t,
+             call_frame_t *frame, void *);
+struct synctask *
+synctask_create(struct syncenv *, size_t stacksize, synctask_fn_t,
+                synctask_cbk_t, call_frame_t *, void *);
+int
+synctask_join(struct synctask *task);
+void
+synctask_wake(struct synctask *task);
+void
+synctask_yield(struct synctask *task, struct timespec *delta);
+void
+synctask_sleep(int32_t secs);
+void
+synctask_waitfor(struct synctask *task, int count);
+
+#define synctask_barrier_init(args) syncbarrier_init(&args->barrier)
+#define synctask_barrier_wait(args, n) syncbarrier_wait(&args->barrier, n)
+#define synctask_barrier_wake(args) syncbarrier_wake(&args->barrier)
+
+int
+synctask_setid(struct synctask *task, uid_t uid, gid_t gid);
+#define SYNCTASK_SETID(uid, gid) synctask_setid(synctask_get(), uid, gid);
+
+int
+syncopctx_setfsuid(void *uid);
+int
+syncopctx_setfsgid(void *gid);
+int
+syncopctx_setfsgroups(int count, const void *groups);
+int
+syncopctx_setfspid(void *pid);
+int
+syncopctx_setfslkowner(gf_lkowner_t *lk_owner);
+
+static inline call_frame_t *
+syncop_create_frame(xlator_t *this)
+{
+    call_frame_t *frame = NULL;
+    int ngrps = -1;
+    struct syncopctx *opctx = NULL;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+
+    frame->root->type = GF_OP_TYPE_FOP;
+    opctx = syncopctx_getctx();
+
+    if (opctx && (opctx->valid & SYNCOPCTX_PID))
+        frame->root->pid = opctx->pid;
+    else
+        frame->root->pid = getpid();
+
+    if (opctx && (opctx->valid & SYNCOPCTX_UID))
+        frame->root->uid = opctx->uid;
+    else
+        frame->root->uid = geteuid();
+
+    if (opctx && (opctx->valid & SYNCOPCTX_GID))
+        frame->root->gid = opctx->gid;
+    else
+        frame->root->gid = getegid();
+
+    if (opctx && (opctx->valid & SYNCOPCTX_GROUPS)) {
+        ngrps = opctx->ngrps;
+
+        if (ngrps != 0 && opctx->groups != NULL) {
+            if (call_stack_alloc_groups(frame->root, ngrps) != 0) {
+                STACK_DESTROY(frame->root);
+                return NULL;
+            }
+
+            memcpy(frame->root->groups, opctx->groups, (sizeof(gid_t) * ngrps));
+        }
+    } else {
+        ngrps = getgroups(0, 0);
+        if (ngrps < 0) {
+            STACK_DESTROY(frame->root);
+            return NULL;
+        }
+
+        if (call_stack_alloc_groups(frame->root, ngrps) != 0) {
+            STACK_DESTROY(frame->root);
+            return NULL;
+        }
+
+        if (getgroups(ngrps, frame->root->groups) < 0) {
+            STACK_DESTROY(frame->root);
+            return NULL;
+        }
+    }
+
+    if (opctx && (opctx->valid & SYNCOPCTX_LKOWNER))
+        frame->root->lk_owner = opctx->lk_owner;
+
+    return frame;
+}
+
+int
+synclock_init(synclock_t *lock, lock_attr_t attr);
+int
+synclock_destroy(synclock_t *lock);
+int
+synclock_lock(synclock_t *lock);
+int
+synclock_trylock(synclock_t *lock);
+int
+synclock_unlock(synclock_t *lock);
+
+int32_t
+synccond_init(synccond_t *cond);
+
+void
+synccond_destroy(synccond_t *cond);
+
+int
+synccond_wait(synccond_t *cond, synclock_t *lock);
+
+int
+synccond_timedwait(synccond_t *cond, synclock_t *lock, struct timespec *delta);
+
+void
+synccond_signal(synccond_t *cond);
+
+void
+synccond_broadcast(synccond_t *cond);
+
+int
+syncbarrier_init(syncbarrier_t *barrier);
+int
+syncbarrier_wait(syncbarrier_t *barrier, int waitfor);
+int
+syncbarrier_wake(syncbarrier_t *barrier);
+int
+syncbarrier_destroy(syncbarrier_t *barrier);
+
+int
+syncop_lookup(xlator_t *subvol, loc_t *loc,
+              /* out */
+              struct iatt *iatt, struct iatt *parent,
+              /* xdata */
+              dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_readdirp(xlator_t *subvol, fd_t *fd, size_t size, off_t off,
+                /* out */
+                gf_dirent_t *entries, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_readdir(xlator_t *subvol, fd_t *fd, size_t size, off_t off,
+               gf_dirent_t *entries, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_opendir(xlator_t *subvol, loc_t *loc, fd_t *fd, dict_t *xdata_in,
+               dict_t **xdata_out);
+
+int
+syncop_setattr(xlator_t *subvol, loc_t *loc, struct iatt *iatt, int valid,
+               /* out */
+               struct iatt *preop, struct iatt *postop, dict_t *xdata_in,
+               dict_t **xdata_out);
+
+int
+syncop_fsetattr(xlator_t *subvol, fd_t *fd, struct iatt *iatt, int valid,
+                /* out */
+                struct iatt *preop, struct iatt *postop, dict_t *xdata_in,
+                dict_t **xdata_out);
+
+int
+syncop_statfs(xlator_t *subvol, loc_t *loc,
+              /* out */
+              struct statvfs *buf, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_setxattr(xlator_t *subvol, loc_t *loc, dict_t *dict, int32_t flags,
+                dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_fsetxattr(xlator_t *subvol, fd_t *fd, dict_t *dict, int32_t flags,
+                 dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_listxattr(xlator_t *subvol, loc_t *loc, dict_t **dict, dict_t *xdata_in,
+                 dict_t **xdata_out);
+
+int
+syncop_getxattr(xlator_t *xl, loc_t *loc, dict_t **dict, const char *key,
+                dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_fgetxattr(xlator_t *xl, fd_t *fd, dict_t **dict, const char *key,
+                 dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_removexattr(xlator_t *subvol, loc_t *loc, const char *name,
+                   dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_fremovexattr(xlator_t *subvol, fd_t *fd, const char *name,
+                    dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_create(xlator_t *subvol, loc_t *loc, int32_t flags, mode_t mode,
+              fd_t *fd, struct iatt *iatt, dict_t *xdata_in,
+              dict_t **xdata_out);
+
+int
+syncop_open(xlator_t *subvol, loc_t *loc, int32_t flags, fd_t *fd,
+            dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_close(fd_t *fd);
+
+int
+syncop_write(xlator_t *subvol, fd_t *fd, const char *buf, int size,
+             off_t offset, struct iobref *iobref, uint32_t flags,
+             dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_writev(xlator_t *subvol, fd_t *fd, const struct iovec *vector,
+              int32_t count, off_t offset, struct iobref *iobref,
+              uint32_t flags, struct iatt *preiatt, struct iatt *postiatt,
+              dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_readv(xlator_t *subvol, fd_t *fd, size_t size, off_t off, uint32_t flags,
+             /* out */
+             struct iovec **vector, int *count, struct iobref **iobref,
+             struct iatt *iatt, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_ftruncate(xlator_t *subvol, fd_t *fd, off_t offset, struct iatt *preiatt,
+                 struct iatt *postiatt, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_truncate(xlator_t *subvol, loc_t *loc, off_t offset, dict_t *xdata_in,
+                dict_t **xdata_out);
+
+int
+syncop_unlink(xlator_t *subvol, loc_t *loc, dict_t *xdata_in,
+              dict_t **xdata_out);
+
+int
+syncop_rmdir(xlator_t *subvol, loc_t *loc, int flags, dict_t *xdata_in,
+             dict_t **xdata_out);
+
+int
+syncop_fsync(xlator_t *subvol, fd_t *fd, int dataonly, struct iatt *preiatt,
+             struct iatt *postiatt, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_flush(xlator_t *subvol, fd_t *fd, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_fstat(xlator_t *subvol, fd_t *fd, struct iatt *stbuf, dict_t *xdata_in,
+             dict_t **xdata_out);
+
+int
+syncop_stat(xlator_t *subvol, loc_t *loc, struct iatt *stbuf, dict_t *xdata_in,
+            dict_t **xdata_out);
+
+int
+syncop_symlink(xlator_t *subvol, loc_t *loc, const char *newpath,
+               struct iatt *iatt, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_readlink(xlator_t *subvol, loc_t *loc, char **buffer, size_t size,
+                dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_mknod(xlator_t *subvol, loc_t *loc, mode_t mode, dev_t rdev,
+             struct iatt *iatt, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_mkdir(xlator_t *subvol, loc_t *loc, mode_t mode, struct iatt *iatt,
+             dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_link(xlator_t *subvol, loc_t *oldloc, loc_t *newloc, struct iatt *iatt,
+            dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_fsyncdir(xlator_t *subvol, fd_t *fd, int datasync, dict_t *xdata_in,
+                dict_t **xdata_out);
+
+int
+syncop_access(xlator_t *subvol, loc_t *loc, int32_t mask, dict_t *xdata_in,
+              dict_t **xdata_out);
+
+int
+syncop_fallocate(xlator_t *subvol, fd_t *fd, int32_t keep_size, off_t offset,
+                 size_t len, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_discard(xlator_t *subvol, fd_t *fd, off_t offset, size_t len,
+               dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_zerofill(xlator_t *subvol, fd_t *fd, off_t offset, off_t len,
+                dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_rename(xlator_t *subvol, loc_t *oldloc, loc_t *newloc, dict_t *xdata_in,
+              dict_t **xdata_out);
+
+int
+syncop_lk(xlator_t *subvol, fd_t *fd, int cmd, struct gf_flock *flock,
+          dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_inodelk(xlator_t *subvol, const char *volume, loc_t *loc, int32_t cmd,
+               struct gf_flock *lock, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_lease(xlator_t *subvol, loc_t *loc, struct gf_lease *lease,
+             dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_ipc(xlator_t *subvol, int op, dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_xattrop(xlator_t *subvol, loc_t *loc, gf_xattrop_flags_t flags,
+               dict_t *dict, dict_t *xdata_in, dict_t **dict_out,
+               dict_t **xdata_out);
+
+int
+syncop_fxattrop(xlator_t *subvol, fd_t *fd, gf_xattrop_flags_t flags,
+                dict_t *dict, dict_t *xdata_in, dict_t **dict_out,
+                dict_t **xdata_out);
+
+int
+syncop_seek(xlator_t *subvol, fd_t *fd, off_t offset, gf_seek_what_t what,
+            dict_t *xdata_in, off_t *off);
+
+int
+syncop_getactivelk(xlator_t *subvol, loc_t *loc,
+                   lock_migration_info_t *locklist, dict_t *xdata_in,
+                   dict_t **xdata_out);
+
+int
+syncop_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+syncop_setactivelk(xlator_t *subvol, loc_t *loc,
+                   lock_migration_info_t *locklist, dict_t *xdata_in,
+                   dict_t **xdata_out);
+
+int
+syncop_put(xlator_t *subvol, loc_t *loc, mode_t mode, mode_t umask,
+           uint32_t flags, struct iovec *vector, int32_t count, off_t offset,
+           struct iobref *iobref, dict_t *xattr, struct iatt *iatt,
+           dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+syncop_icreate(xlator_t *subvol, loc_t *loc, mode_t mode, dict_t *xdata_out);
+
+int
+syncop_entrylk(xlator_t *subvol, const char *volume, loc_t *loc,
+               const char *basename, entrylk_cmd cmd, entrylk_type type,
+               dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_copy_file_range(xlator_t *subvol, fd_t *fd_in, off64_t off_in,
+                       fd_t *fd_out, off64_t off_out, size_t len,
+                       uint32_t flags, struct iatt *stbuf,
+                       struct iatt *preiatt_dst, struct iatt *postiatt_dst,
+                       dict_t *xdata_in, dict_t **xdata_out);
+
+int
+syncop_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, struct iatt *stbuf,
+                           struct iatt *prebuf_dst, struct iatt *postbuf_dst,
+                           dict_t *xdata);
+
+#endif /* _SYNCOP_H */
diff --git a/libglusterfs/src/glusterfs/syscall.h b/libglusterfs/src/glusterfs/syscall.h
new file mode 100644
index 00000000000..b6d3ab4f2ad
--- /dev/null
+++ b/libglusterfs/src/glusterfs/syscall.h
@@ -0,0 +1,278 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __SYSCALL_H__
+#define __SYSCALL_H__
+
+#include <dirent.h>
+#include <sys/uio.h>
+#include <sys/statvfs.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <stdio.h>
+
+/* GF follows the Linux XATTR definition, which differs in Darwin. */
+#define GF_XATTR_CREATE 0x1  /* set value, fail if attr already exists */
+#define GF_XATTR_REPLACE 0x2 /* set value, fail if attr does not exist */
+
+/* Linux kernel version 2.6.x don't have these defined
+   define if not defined */
+
+#ifndef XATTR_SECURITY_PREFIX
+#define XATTR_SECURITY_PREFIX "security."
+#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
+#endif
+
+#ifndef XATTR_SYSTEM_PREFIX
+#define XATTR_SYSTEM_PREFIX "system."
+#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
+#endif
+
+#ifndef XATTR_TRUSTED_PREFIX
+#define XATTR_TRUSTED_PREFIX "trusted."
+#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
+#endif
+
+#ifndef XATTR_USER_PREFIX
+#define XATTR_USER_PREFIX "user."
+#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
+#endif
+
+#if defined(GF_DARWIN_HOST_OS)
+#include <sys/xattr.h>
+#define XATTR_DARWIN_NOSECURITY XATTR_NOSECURITY
+#define XATTR_DARWIN_NODEFAULT XATTR_NODEFAULT
+#define XATTR_DARWIN_SHOWCOMPRESSION XATTR_SHOWCOMPRESSION
+#endif
+
+int
+sys_lstat(const char *path, struct stat *buf);
+
+int
+sys_stat(const char *path, struct stat *buf);
+
+int
+sys_fstat(int fd, struct stat *buf);
+
+int
+sys_fstatat(int dirfd, const char *pathname, struct stat *buf, int flags);
+
+int
+sys_open(const char *pathname, int flags, int mode);
+
+int
+sys_openat(int dirfd, const char *pathname, int flags, int mode);
+
+DIR *
+sys_opendir(const char *name);
+
+struct dirent *
+sys_readdir(DIR *dir, struct dirent *de);
+
+ssize_t
+sys_readlink(const char *path, char *buf, size_t bufsiz);
+
+int
+sys_closedir(DIR *dir);
+
+int
+sys_mknod(const char *pathname, mode_t mode, dev_t dev);
+
+int
+sys_mkdir(const char *pathname, mode_t mode);
+
+int
+sys_mkdirat(int dirfd, const char *pathname, mode_t mode);
+
+int
+sys_unlink(const char *pathname);
+
+int
+sys_unlinkat(int dfd, const char *pathname);
+
+int
+sys_rmdir(const char *pathname);
+
+int
+sys_symlink(const char *oldpath, const char *newpath);
+
+int
+sys_symlinkat(const char *oldpath, int dirfd, const char *newpath);
+
+int
+sys_rename(const char *oldpath, const char *newpath);
+
+int
+sys_link(const char *oldpath, const char *newpath);
+
+int
+sys_linkat(int oldfd, const char *oldpath, int newfd, const char *newpath);
+
+int
+sys_chmod(const char *path, mode_t mode);
+
+int
+sys_fchmod(int fd, mode_t mode);
+
+int
+sys_chown(const char *path, uid_t owner, gid_t group);
+
+int
+sys_fchown(int fd, uid_t owner, gid_t group);
+
+int
+sys_lchown(const char *path, uid_t owner, gid_t group);
+
+int
+sys_truncate(const char *path, off_t length);
+
+int
+sys_ftruncate(int fd, off_t length);
+
+int
+sys_utimes(const char *filename, const struct timeval times[2]);
+
+#if defined(HAVE_UTIMENSAT)
+int
+sys_utimensat(int dirfd, const char *filename, const struct timespec times[2],
+              int flags);
+#endif
+
+int
+sys_futimes(int fd, const struct timeval times[2]);
+
+int
+sys_creat(const char *pathname, mode_t mode);
+
+ssize_t
+sys_readv(int fd, const struct iovec *iov, int iovcnt);
+
+ssize_t
+sys_writev(int fd, const struct iovec *iov, int iovcnt);
+
+ssize_t
+sys_read(int fd, void *buf, size_t count);
+
+ssize_t
+sys_write(int fd, const void *buf, size_t count);
+
+off_t
+sys_lseek(int fd, off_t offset, int whence);
+
+int
+sys_statvfs(const char *path, struct statvfs *buf);
+
+int
+sys_fstatvfs(int fd, struct statvfs *buf);
+
+int
+sys_close(int fd);
+
+int
+sys_fsync(int fd);
+
+int
+sys_fdatasync(int fd);
+
+void
+gf_add_prefix(const char *ns, const char *key, char **newkey);
+
+void
+gf_remove_prefix(const char *ns, const char *key, char **newkey);
+
+int
+sys_lsetxattr(const char *path, const char *name, const void *value,
+              size_t size, int flags);
+
+ssize_t
+sys_llistxattr(const char *path, char *list, size_t size);
+
+ssize_t
+sys_lgetxattr(const char *path, const char *name, void *value, size_t size);
+
+ssize_t
+sys_fgetxattr(int filedes, const char *name, void *value, size_t size);
+
+int
+sys_fsetxattr(int filedes, const char *name, const void *value, size_t size,
+              int flags);
+
+ssize_t
+sys_flistxattr(int filedes, char *list, size_t size);
+
+int
+sys_lremovexattr(const char *path, const char *name);
+
+int
+sys_fremovexattr(int filedes, const char *name);
+
+int
+sys_access(const char *pathname, int mode);
+
+int
+sys_fallocate(int fd, int mode, off_t offset, off_t len);
+
+ssize_t
+sys_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset);
+
+ssize_t
+sys_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset);
+
+ssize_t
+sys_pread(int fd, void *buf, size_t count, off_t offset);
+
+ssize_t
+sys_pwrite(int fd, const void *buf, size_t count, off_t offset);
+
+int
+sys_socket(int domain, int type, int protocol);
+
+int
+sys_accept(int sock, struct sockaddr *sockaddr, socklen_t *socklen, int flags);
+
+#ifdef GF_BSD_HOST_OS
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
+
+/*
+ * According to the man page of copy_file_range, both off_in and off_out are
+ * pointers to the data type loff_t (i.e. loff_t *). But, freebsd does not
+ * have (and recognize) loff_t. Since loff_t is 64 bits, use off64_t
+ * instead.  Since it's a pointer type it should be okay. It just needs
+ * to be a pointer-to-64-bit pointer for both 32- and 64-bit platforms.
+ * off64_t is recognized by freebsd.
+ * TODO: In future, when freebsd can recognize loff_t, probably revisit this
+ *       and change the off_in and off_out to (loff_t *).
+ */
+ssize_t
+sys_copy_file_range(int fd_in, off64_t *off_in, int fd_out, off64_t *off_out,
+                    size_t len, unsigned int flags);
+
+int
+sys_kill(pid_t pid, int sig);
+
+#ifdef __FreeBSD__
+int
+sys_sysctl(const int *name, u_int namelen, void *oldp, size_t *oldlenp,
+           const void *newp, size_t newlen);
+#endif
+
+#endif /* __SYSCALL_H__ */
diff --git a/libglusterfs/src/glusterfs/template-component-messages.h b/libglusterfs/src/glusterfs/template-component-messages.h
new file mode 100644
index 00000000000..aa7ad3d1baa
--- /dev/null
+++ b/libglusterfs/src/glusterfs/template-component-messages.h
@@ -0,0 +1,28 @@
+/*
+ Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _component_MESSAGES_H_
+#define _component_MESSAGES_H_
+
+#include "glusterfs/glfs-message-id.h"
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(component, message id, message id, ...);
+
+#endif /* !_component_MESSAGES_H_ */
diff --git a/libglusterfs/src/glusterfs/throttle-tbf.h b/libglusterfs/src/glusterfs/throttle-tbf.h
new file mode 100644
index 00000000000..cccb13c83d9
--- /dev/null
+++ b/libglusterfs/src/glusterfs/throttle-tbf.h
@@ -0,0 +1,74 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs/list.h"
+#include "glusterfs/xlator.h"
+#include "glusterfs/locking.h"
+
+#ifndef THROTTLE_TBF_H__
+#define THROTTLE_TBF_H__
+
+typedef enum tbf_ops {
+    TBF_OP_MIN = -1,
+    TBF_OP_HASH = 0,    /* checksum calculation  */
+    TBF_OP_READ = 1,    /* inode read(s)         */
+    TBF_OP_READDIR = 2, /* dentry read(s)        */
+    TBF_OP_MAX = 3,
+} tbf_ops_t;
+
+/**
+ * Operation rate specification
+ */
+typedef struct tbf_opspec {
+    tbf_ops_t op;
+
+    unsigned long rate;
+
+    unsigned long maxlimit;
+
+    unsigned long token_gen_interval; /* Token generation interval in usec */
+} tbf_opspec_t;
+
+/**
+ * Token bucket for each operation type
+ */
+typedef struct tbf_bucket {
+    gf_lock_t lock;
+
+    pthread_t tokener; /* token generator thread          */
+
+    unsigned long tokenrate; /* token generation rate           */
+
+    unsigned long tokens; /* number of current tokens        */
+
+    unsigned long maxtokens; /* maximum token in the bucket     */
+
+    struct list_head queued; /* list of non-conformant requests */
+
+    unsigned long token_gen_interval; /* Token generation interval in usec */
+} tbf_bucket_t;
+
+typedef struct tbf {
+    tbf_bucket_t **bucket;
+} tbf_t;
+
+tbf_t *
+tbf_init(tbf_opspec_t *, unsigned int);
+
+int
+tbf_mod(tbf_t *, tbf_opspec_t *);
+
+void
+tbf_throttle(tbf_t *, tbf_ops_t, unsigned long);
+
+#define TBF_THROTTLE_BEGIN(tbf, op, tokens) (tbf_throttle(tbf, op, tokens))
+#define TBF_THROTTLE_END(tbf, op, tokens)
+
+#endif /** THROTTLE_TBF_H__ */
diff --git a/libglusterfs/src/glusterfs/timer.h b/libglusterfs/src/glusterfs/timer.h
new file mode 100644
index 00000000000..ae5b2edf451
--- /dev/null
+++ b/libglusterfs/src/glusterfs/timer.h
@@ -0,0 +1,56 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _TIMER_H
+#define _TIMER_H
+
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/xlator.h"
+#include <sys/time.h>
+#include <pthread.h>
+
+typedef void (*gf_timer_cbk_t)(void *);
+
+struct _gf_timer {
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_timer *next;
+            struct _gf_timer *prev;
+        };
+    };
+    struct timespec at;
+    gf_timer_cbk_t callbk;
+    void *data;
+    xlator_t *xl;
+    gf_boolean_t fired;
+};
+
+struct _gf_timer_registry {
+    struct list_head active;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    pthread_t th;
+    char fin;
+};
+
+typedef struct _gf_timer gf_timer_t;
+typedef struct _gf_timer_registry gf_timer_registry_t;
+
+gf_timer_t *
+gf_timer_call_after(glusterfs_ctx_t *ctx, struct timespec delta,
+                    gf_timer_cbk_t cbk, void *data);
+
+int32_t
+gf_timer_call_cancel(glusterfs_ctx_t *ctx, gf_timer_t *event);
+
+void
+gf_timer_registry_destroy(glusterfs_ctx_t *ctx);
+#endif /* _TIMER_H */
diff --git a/libglusterfs/src/glusterfs/timespec.h b/libglusterfs/src/glusterfs/timespec.h
new file mode 100644
index 00000000000..bb9ab446a5f
--- /dev/null
+++ b/libglusterfs/src/glusterfs/timespec.h
@@ -0,0 +1,33 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __INCLUDE_TIMESPEC_H__
+#define __INCLUDE_TIMESPEC_H__
+
+#include <stdint.h>
+#include <sys/time.h>
+
+#define TS(ts) ((ts.tv_sec * 1000000000LL) + ts.tv_nsec)
+#define NANO (+1.0E-9)
+#define GIGA UINT64_C(1000000000)
+
+void
+timespec_now(struct timespec *ts);
+void
+timespec_now_realtime(struct timespec *ts);
+void
+timespec_adjust_delta(struct timespec *ts, struct timespec delta);
+void
+timespec_sub(const struct timespec *begin, const struct timespec *end,
+             struct timespec *res);
+int
+timespec_cmp(const struct timespec *lhs_ts, const struct timespec *rhs_ts);
+
+#endif /*  __INCLUDE_TIMESPEC_H__ */
diff --git a/libglusterfs/src/glusterfs/trie.h b/libglusterfs/src/glusterfs/trie.h
new file mode 100644
index 00000000000..6d2d8015964
--- /dev/null
+++ b/libglusterfs/src/glusterfs/trie.h
@@ -0,0 +1,52 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _TRIE_H_
+#define _TRIE_H_
+
+struct trienode;
+typedef struct trienode trienode_t;
+
+struct trie;
+typedef struct trie trie_t;
+
+struct trienodevec {
+    trienode_t **nodes;
+    unsigned cnt;
+};
+
+trie_t *
+trie_new();
+
+int
+trie_add(trie_t *trie, const char *word);
+
+void
+trie_destroy(trie_t *trie);
+
+void
+trie_destroy_bynode(trienode_t *node);
+
+int
+trie_measure(trie_t *trie, const char *word, trienode_t **nodes, int nodecnt);
+
+int
+trie_measure_vec(trie_t *trie, const char *word, struct trienodevec *nodevec);
+
+void
+trie_reset_search(trie_t *trie);
+
+int
+trienode_get_dist(trienode_t *node);
+
+int
+trienode_get_word(trienode_t *node, char **buf);
+
+#endif
diff --git a/libglusterfs/src/glusterfs/upcall-utils.h b/libglusterfs/src/glusterfs/upcall-utils.h
new file mode 100644
index 00000000000..0de8428c5fc
--- /dev/null
+++ b/libglusterfs/src/glusterfs/upcall-utils.h
@@ -0,0 +1,110 @@
+/*
+  Copyright (c) 2015, Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _UPCALL_UTILS_H
+#define _UPCALL_UTILS_H
+
+#include "glusterfs/iatt.h"
+#include "glusterfs/compat-uuid.h"
+#include "glusterfs/compat.h"
+
+/* Flags sent for cache_invalidation */
+#define UP_NLINK 0x00000001 /* update nlink */
+#define UP_MODE 0x00000002  /* update mode and ctime */
+#define UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define UP_SIZE 0x00000008  /* update fsize */
+#define UP_TIMES 0x00000010 /* update all times */
+#define UP_ATIME 0x00000020 /* update atime only */
+#define UP_PERM                                                                \
+    0x00000040 /* update fields needed for permission                          \
+                  checking */
+#define UP_RENAME                                                              \
+    0x00000080 /* this is a rename op - delete the cache                       \
+                  entry */
+#define UP_FORGET                                                              \
+    0x00000100                     /* inode_forget on server side -            \
+                                      invalidate the cache entry */
+#define UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define UP_XATTR 0x00000400    /* update the xattrs and ctime */
+#define UP_XATTR_RM 0x00000800 /* Remove the xattrs and update ctime */
+
+#define UP_EXPLICIT_LOOKUP 0x00001000 /* Request an explicit lookup */
+
+#define UP_INVAL_ATTR 0x00002000 /* Request to invalidate iatt and xatt */
+
+/* for fops - open, read, lk, */
+#define UP_UPDATE_CLIENT (UP_ATIME)
+
+/* for fop - write, truncate */
+#define UP_WRITE_FLAGS (UP_SIZE | UP_TIMES)
+
+/* for fop - setattr */
+#define UP_ATTR_FLAGS (UP_SIZE | UP_TIMES | UP_OWN | UP_MODE | UP_PERM)
+/* for fop - rename */
+#define UP_RENAME_FLAGS (UP_RENAME)
+
+/* to invalidate parent directory entries for fops -rename, unlink, rmdir,
+ * mkdir, create */
+#define UP_PARENT_DENTRY_FLAGS (UP_PARENT_TIMES)
+
+/* for fop - unlink, link, rmdir, mkdir */
+#define UP_NLINK_FLAGS (UP_NLINK | UP_TIMES)
+
+#define IATT_UPDATE_FLAGS                                                      \
+    (UP_NLINK | UP_MODE | UP_OWN | UP_SIZE | UP_TIMES | UP_ATIME | UP_PERM)
+
+typedef enum {
+    GF_UPCALL_EVENT_NULL,
+    GF_UPCALL_CACHE_INVALIDATION,
+    GF_UPCALL_RECALL_LEASE,
+    GF_UPCALL_INODELK_CONTENTION,
+    GF_UPCALL_ENTRYLK_CONTENTION,
+} gf_upcall_event_t;
+
+struct gf_upcall {
+    char *client_uid;
+    uuid_t gfid;
+    uint32_t event_type;
+    void *data;
+};
+
+struct gf_upcall_cache_invalidation {
+    uint32_t flags;
+    uint32_t expire_time_attr;
+    struct iatt stat;
+    struct iatt p_stat;    /* parent dir stat */
+    struct iatt oldp_stat; /* oldparent dir stat */
+    dict_t *dict;          /* For xattrs */
+};
+
+struct gf_upcall_recall_lease {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    uuid_t tid;          /* transaction id of the fop that caused
+                            the recall */
+    dict_t *dict;
+};
+
+struct gf_upcall_inodelk_contention {
+    struct gf_flock flock;
+    pid_t pid;
+    const char *domain;
+    dict_t *xdata;
+};
+
+struct gf_upcall_entrylk_contention {
+    uint32_t type;
+    pid_t pid;
+    const char *name;
+    const char *domain;
+    dict_t *xdata;
+};
+
+#endif /* _UPCALL_UTILS_H */
diff --git a/libglusterfs/src/glusterfs/xlator.h b/libglusterfs/src/glusterfs/xlator.h
new file mode 100644
index 00000000000..4fd3abdaeff
--- /dev/null
+++ b/libglusterfs/src/glusterfs/xlator.h
@@ -0,0 +1,1106 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _XLATOR_H
+#define _XLATOR_H
+
+#include <stdint.h>                    // for int32_t
+#include <sys/types.h>                 // for off_t, mode_t, off64_t, dev_t
+#include "glusterfs/glusterfs-fops.h"  // for GF_FOP_MAXVALUE, entrylk_cmd
+#include "glusterfs/atomic.h"          // for gf_atomic_t
+#include "glusterfs/glusterfs.h"       // for gf_boolean_t, glusterfs_ctx_t
+#include "glusterfs/compat-uuid.h"     // for uuid_t
+#include "glusterfs/compat.h"
+#include "glusterfs/event-history.h"
+#include "glusterfs/dict.h"
+#include "glusterfs/latency.h"
+
+#define FIRST_CHILD(xl) (xl->children->xlator)
+#define SECOND_CHILD(xl) (xl->children->next->xlator)
+
+#define GF_SET_ATTR_MODE 0x1
+#define GF_SET_ATTR_UID 0x2
+#define GF_SET_ATTR_GID 0x4
+#define GF_SET_ATTR_SIZE 0x8
+#define GF_SET_ATTR_ATIME 0x10
+#define GF_SET_ATTR_MTIME 0x20
+#define GF_SET_ATTR_CTIME 0x40
+#define GF_ATTR_ATIME_NOW 0x80
+#define GF_ATTR_MTIME_NOW 0x100
+
+#define gf_attr_mode_set(mode) ((mode)&GF_SET_ATTR_MODE)
+#define gf_attr_uid_set(mode) ((mode)&GF_SET_ATTR_UID)
+#define gf_attr_gid_set(mode) ((mode)&GF_SET_ATTR_GID)
+#define gf_attr_size_set(mode) ((mode)&GF_SET_ATTR_SIZE)
+#define gf_attr_atime_set(mode) ((mode)&GF_SET_ATTR_ATIME)
+#define gf_attr_mtime_set(mode) ((mode)&GF_SET_ATTR_MTIME)
+
+struct _xlator;
+typedef struct _xlator xlator_t;
+struct _dir_entry;
+typedef struct _dir_entry dir_entry_t;
+struct _gf_dirent;
+typedef struct _gf_dirent gf_dirent_t;
+struct _loc;
+typedef struct _loc loc_t;
+
+typedef int32_t (*event_notify_fn_t)(xlator_t *this, int32_t event, void *data,
+                                     ...);
+
+#include "glusterfs/list.h"
+#include "glusterfs/gf-dirent.h"
+#include "glusterfs/stack.h"
+#include "glusterfs/iobuf.h"
+#include "glusterfs/globals.h"
+#include "glusterfs/iatt.h"
+#include "glusterfs/options.h"
+#include "glusterfs/client_t.h"
+
+struct _loc {
+    const char *path;
+    const char *name;
+    inode_t *inode;
+    inode_t *parent;
+    /* Currently all location based operations are through 'gfid' of inode.
+     * But the 'inode->gfid' only gets set in higher most layer (as in,
+     * 'fuse', 'protocol/server', or 'nfs/server'). So if translators want
+     * to send fops on a inode before the 'inode->gfid' is set, they have to
+     * make use of below 'gfid' fields
+     */
+    uuid_t gfid;
+    uuid_t pargfid;
+};
+
+typedef int32_t (*fop_getspec_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, char *spec_data);
+
+typedef int32_t (*fop_rchecksum_cbk_t)(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, uint32_t weak_checksum,
+                                       uint8_t *strong_checksum, dict_t *xdata);
+
+typedef int32_t (*fop_getspec_t)(call_frame_t *frame, xlator_t *this,
+                                 const char *key, int32_t flag);
+
+typedef int32_t (*fop_rchecksum_t)(call_frame_t *frame, xlator_t *this,
+                                   fd_t *fd, off_t offset, int32_t len,
+                                   dict_t *xdata);
+
+typedef int32_t (*fop_lookup_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, inode_t *inode,
+                                    struct iatt *buf, dict_t *xdata,
+                                    struct iatt *postparent);
+
+typedef int32_t (*fop_stat_cbk_t)(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, struct iatt *buf,
+                                  dict_t *xdata);
+
+typedef int32_t (*fop_fstat_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct iatt *buf,
+                                   dict_t *xdata);
+
+typedef int32_t (*fop_truncate_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, struct iatt *prebuf,
+                                      struct iatt *postbuf, dict_t *xdata);
+
+typedef int32_t (*fop_ftruncate_cbk_t)(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, struct iatt *prebuf,
+                                       struct iatt *postbuf, dict_t *xdata);
+
+typedef int32_t (*fop_access_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_readlink_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, const char *path,
+                                      struct iatt *buf, dict_t *xdata);
+
+typedef int32_t (*fop_mknod_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, inode_t *inode,
+                                   struct iatt *buf, struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_mkdir_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, inode_t *inode,
+                                   struct iatt *buf, struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_unlink_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *preparent,
+                                    struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_rmdir_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_symlink_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, inode_t *inode,
+                                     struct iatt *buf, struct iatt *preparent,
+                                     struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_rename_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *buf,
+                                    struct iatt *preoldparent,
+                                    struct iatt *postoldparent,
+                                    struct iatt *prenewparent,
+                                    struct iatt *postnewparent, dict_t *xdata);
+
+typedef int32_t (*fop_link_cbk_t)(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, inode_t *inode,
+                                  struct iatt *buf, struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_create_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, fd_t *fd, inode_t *inode,
+                                    struct iatt *buf, struct iatt *preparent,
+                                    struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_open_cbk_t)(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+typedef int32_t (*fop_readv_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct iovec *vector,
+                                   int32_t count, struct iatt *stbuf,
+                                   struct iobref *iobref, dict_t *xdata);
+
+typedef int32_t (*fop_writev_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *prebuf,
+                                    struct iatt *postbuf, dict_t *xdata);
+
+typedef int32_t (*fop_flush_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_fsync_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct iatt *prebuf,
+                                   struct iatt *postbuf, dict_t *xdata);
+
+typedef int32_t (*fop_opendir_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, fd_t *fd, dict_t *xdata);
+
+typedef int32_t (*fop_fsyncdir_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_statfs_cbk_t)(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, struct statvfs *buf,
+                                    dict_t *xdata);
+
+typedef int32_t (*fop_setxattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_getxattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *dict,
+                                      dict_t *xdata);
+
+typedef int32_t (*fop_fsetxattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_fgetxattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *dict,
+                                       dict_t *xdata);
+
+typedef int32_t (*fop_removexattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_fremovexattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                          xlator_t *this, int32_t op_ret,
+                                          int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_lk_cbk_t)(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, struct gf_flock *flock,
+                                dict_t *xdata);
+
+typedef int32_t (*fop_inodelk_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_finodelk_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_entrylk_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_fentrylk_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_readdir_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, gf_dirent_t *entries,
+                                     dict_t *xdata);
+
+typedef int32_t (*fop_readdirp_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, gf_dirent_t *entries,
+                                      dict_t *xdata);
+
+typedef int32_t (*fop_xattrop_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, dict_t *xattr,
+                                     dict_t *xdata);
+
+typedef int32_t (*fop_fxattrop_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xattr,
+                                      dict_t *xdata);
+
+typedef int32_t (*fop_setattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, struct iatt *preop_stbuf,
+                                     struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_fsetattr_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno,
+                                      struct iatt *preop_stbuf,
+                                      struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_fallocate_cbk_t)(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno,
+                                       struct iatt *preop_stbuf,
+                                       struct iatt *postop_stbuf,
+                                       dict_t *xdata);
+
+typedef int32_t (*fop_discard_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, struct iatt *preop_stbuf,
+                                     struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_zerofill_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno,
+                                      struct iatt *preop_stbuf,
+                                      struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_ipc_cbk_t)(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_seek_cbk_t)(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, off_t offset,
+                                  dict_t *xdata);
+
+typedef int32_t (*fop_lease_cbk_t)(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, struct gf_lease *lease,
+                                   dict_t *xdata);
+typedef int32_t (*fop_compound_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, void *data,
+                                      dict_t *xdata);
+
+typedef int32_t (*fop_getactivelk_cbk_t)(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno,
+                                         lock_migration_info_t *locklist,
+                                         dict_t *xdata);
+
+typedef int32_t (*fop_setactivelk_cbk_t)(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata);
+
+typedef int32_t (*fop_put_cbk_t)(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, inode_t *inode,
+                                 struct iatt *buf, struct iatt *preparent,
+                                 struct iatt *postparent, dict_t *xdata);
+
+typedef int32_t (*fop_icreate_cbk_t)(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, inode_t *inode,
+                                     struct iatt *buf, dict_t *xdata);
+
+typedef int32_t (*fop_namelink_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, struct iatt *prebuf,
+                                      struct iatt *postbuf, dict_t *xdata);
+
+typedef int32_t (*fop_copy_file_range_cbk_t)(
+    call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+    int32_t op_errno, struct iatt *stbuf, struct iatt *prebuf_dst,
+    struct iatt *postbuf_dst, dict_t *xdata);
+
+typedef int32_t (*fop_lookup_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                dict_t *xdata);
+
+typedef int32_t (*fop_stat_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                              dict_t *xdata);
+
+typedef int32_t (*fop_fstat_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               dict_t *xdata);
+
+typedef int32_t (*fop_truncate_t)(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, off_t offset, dict_t *xdata);
+
+typedef int32_t (*fop_ftruncate_t)(call_frame_t *frame, xlator_t *this,
+                                   fd_t *fd, off_t offset, dict_t *xdata);
+
+typedef int32_t (*fop_access_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                int32_t mask, dict_t *xdata);
+
+typedef int32_t (*fop_readlink_t)(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, size_t size, dict_t *xdata);
+
+typedef int32_t (*fop_mknod_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               mode_t mode, dev_t rdev, mode_t umask,
+                               dict_t *xdata);
+
+typedef int32_t (*fop_mkdir_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               mode_t mode, mode_t umask, dict_t *xdata);
+
+typedef int32_t (*fop_unlink_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                int xflags, dict_t *xdata);
+
+typedef int32_t (*fop_rmdir_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               int xflags, dict_t *xdata);
+
+typedef int32_t (*fop_symlink_t)(call_frame_t *frame, xlator_t *this,
+                                 const char *linkname, loc_t *loc, mode_t umask,
+                                 dict_t *xdata);
+
+typedef int32_t (*fop_rename_t)(call_frame_t *frame, xlator_t *this,
+                                loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+typedef int32_t (*fop_link_t)(call_frame_t *frame, xlator_t *this,
+                              loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+typedef int32_t (*fop_create_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                int32_t flags, mode_t mode, mode_t umask,
+                                fd_t *fd, dict_t *xdata);
+
+/* Tell subsequent writes on the fd_t to fsync after every writev fop without
+ * requiring a fsync fop.
+ */
+#define GF_OPEN_FSYNC 0x01
+
+/* Tell write-behind to disable writing behind despite O_SYNC not being set.
+ */
+#define GF_OPEN_NOWB 0x02
+
+typedef int32_t (*fop_open_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                              int32_t flags, fd_t *fd, dict_t *xdata);
+
+typedef int32_t (*fop_readv_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               size_t size, off_t offset, uint32_t flags,
+                               dict_t *xdata);
+
+typedef int32_t (*fop_writev_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                struct iovec *vector, int32_t count,
+                                off_t offset, uint32_t flags,
+                                struct iobref *iobref, dict_t *xdata);
+
+typedef int32_t (*fop_flush_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               dict_t *xdata);
+
+typedef int32_t (*fop_fsync_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int32_t datasync, dict_t *xdata);
+
+typedef int32_t (*fop_opendir_t)(call_frame_t *frame, xlator_t *this,
+                                 loc_t *loc, fd_t *fd, dict_t *xdata);
+
+typedef int32_t (*fop_fsyncdir_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                  int32_t datasync, dict_t *xdata);
+
+typedef int32_t (*fop_statfs_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                dict_t *xdata);
+
+typedef int32_t (*fop_setxattr_t)(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, dict_t *dict, int32_t flags,
+                                  dict_t *xdata);
+
+typedef int32_t (*fop_getxattr_t)(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, const char *name, dict_t *xdata);
+
+typedef int32_t (*fop_fsetxattr_t)(call_frame_t *frame, xlator_t *this,
+                                   fd_t *fd, dict_t *dict, int32_t flags,
+                                   dict_t *xdata);
+
+typedef int32_t (*fop_fgetxattr_t)(call_frame_t *frame, xlator_t *this,
+                                   fd_t *fd, const char *name, dict_t *xdata);
+
+typedef int32_t (*fop_removexattr_t)(call_frame_t *frame, xlator_t *this,
+                                     loc_t *loc, const char *name,
+                                     dict_t *xdata);
+
+typedef int32_t (*fop_fremovexattr_t)(call_frame_t *frame, xlator_t *this,
+                                      fd_t *fd, const char *name,
+                                      dict_t *xdata);
+
+typedef int32_t (*fop_lk_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+typedef int32_t (*fop_inodelk_t)(call_frame_t *frame, xlator_t *this,
+                                 const char *volume, loc_t *loc, int32_t cmd,
+                                 struct gf_flock *flock, dict_t *xdata);
+
+typedef int32_t (*fop_finodelk_t)(call_frame_t *frame, xlator_t *this,
+                                  const char *volume, fd_t *fd, int32_t cmd,
+                                  struct gf_flock *flock, dict_t *xdata);
+
+typedef int32_t (*fop_entrylk_t)(call_frame_t *frame, xlator_t *this,
+                                 const char *volume, loc_t *loc,
+                                 const char *basename, entrylk_cmd cmd,
+                                 entrylk_type type, dict_t *xdata);
+
+typedef int32_t (*fop_fentrylk_t)(call_frame_t *frame, xlator_t *this,
+                                  const char *volume, fd_t *fd,
+                                  const char *basename, entrylk_cmd cmd,
+                                  entrylk_type type, dict_t *xdata);
+
+typedef int32_t (*fop_readdir_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                 size_t size, off_t offset, dict_t *xdata);
+
+typedef int32_t (*fop_readdirp_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                  size_t size, off_t offset, dict_t *xdata);
+
+typedef int32_t (*fop_xattrop_t)(call_frame_t *frame, xlator_t *this,
+                                 loc_t *loc, gf_xattrop_flags_t optype,
+                                 dict_t *xattr, dict_t *xdata);
+
+typedef int32_t (*fop_fxattrop_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                  gf_xattrop_flags_t optype, dict_t *xattr,
+                                  dict_t *xdata);
+
+typedef int32_t (*fop_setattr_t)(call_frame_t *frame, xlator_t *this,
+                                 loc_t *loc, struct iatt *stbuf, int32_t valid,
+                                 dict_t *xdata);
+
+typedef int32_t (*fop_fsetattr_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                  struct iatt *stbuf, int32_t valid,
+                                  dict_t *xdata);
+
+typedef int32_t (*fop_fallocate_t)(call_frame_t *frame, xlator_t *this,
+                                   fd_t *fd, int32_t keep_size, off_t offset,
+                                   size_t len, dict_t *xdata);
+
+typedef int32_t (*fop_discard_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                 off_t offset, size_t len, dict_t *xdata);
+
+typedef int32_t (*fop_zerofill_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                  off_t offset, off_t len, dict_t *xdata);
+
+typedef int32_t (*fop_ipc_t)(call_frame_t *frame, xlator_t *this, int32_t op,
+                             dict_t *xdata);
+
+typedef int32_t (*fop_seek_t)(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              off_t offset, gf_seek_what_t what, dict_t *xdata);
+
+typedef int32_t (*fop_lease_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               struct gf_lease *lease, dict_t *xdata);
+
+typedef int32_t (*fop_compound_t)(call_frame_t *frame, xlator_t *this,
+                                  void *args, dict_t *xdata);
+
+typedef int32_t (*fop_getactivelk_t)(call_frame_t *frame, xlator_t *this,
+                                     loc_t *loc, dict_t *xdata);
+
+typedef int32_t (*fop_setactivelk_t)(call_frame_t *frame, xlator_t *this,
+                                     loc_t *loc,
+                                     lock_migration_info_t *locklist,
+                                     dict_t *xdata);
+
+typedef int32_t (*fop_put_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                             mode_t mode, mode_t umask, uint32_t flags,
+                             struct iovec *vector, int32_t count, off_t offset,
+                             struct iobref *iobref, dict_t *xattr,
+                             dict_t *xdata);
+
+typedef int32_t (*fop_icreate_t)(call_frame_t *frame, xlator_t *this,
+                                 loc_t *loc, mode_t mode, dict_t *xdata);
+
+typedef int32_t (*fop_namelink_t)(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, dict_t *xdata);
+typedef int32_t (*fop_copy_file_range_t)(call_frame_t *frame, xlator_t *this,
+                                         fd_t *fd_in, off64_t off_in,
+                                         fd_t *fd_out, off64_t off_out,
+                                         size_t len, uint32_t flags,
+                                         dict_t *xdata);
+
+/* WARNING: make sure the list is in order with FOP definition in
+   `rpc/xdr/src/glusterfs-fops.x`.
+   If it is not in order, mainly the metrics related feature would be broken */
+struct xlator_fops {
+    fop_stat_t stat;
+    fop_readlink_t readlink;
+    fop_mknod_t mknod;
+    fop_mkdir_t mkdir;
+    fop_unlink_t unlink;
+    fop_rmdir_t rmdir;
+    fop_symlink_t symlink;
+    fop_rename_t rename;
+    fop_link_t link;
+    fop_truncate_t truncate;
+    fop_open_t open;
+    fop_readv_t readv;
+    fop_writev_t writev;
+    fop_statfs_t statfs;
+    fop_flush_t flush;
+    fop_fsync_t fsync;
+    fop_setxattr_t setxattr;
+    fop_getxattr_t getxattr;
+    fop_removexattr_t removexattr;
+    fop_opendir_t opendir;
+    fop_fsyncdir_t fsyncdir;
+    fop_access_t access;
+    fop_create_t create;
+    fop_ftruncate_t ftruncate;
+    fop_fstat_t fstat;
+    fop_lk_t lk;
+    fop_lookup_t lookup;
+    fop_readdir_t readdir;
+    fop_inodelk_t inodelk;
+    fop_finodelk_t finodelk;
+    fop_entrylk_t entrylk;
+    fop_fentrylk_t fentrylk;
+    fop_xattrop_t xattrop;
+    fop_fxattrop_t fxattrop;
+    fop_fgetxattr_t fgetxattr;
+    fop_fsetxattr_t fsetxattr;
+    fop_rchecksum_t rchecksum;
+    fop_setattr_t setattr;
+    fop_fsetattr_t fsetattr;
+    fop_readdirp_t readdirp;
+
+    /* These 3 are required to keep the index same as GF_FOP_##FOP */
+    void *forget_placeholder;
+    void *release_placeholder;
+    void *releasedir_placeholder;
+
+    fop_getspec_t getspec;
+    fop_fremovexattr_t fremovexattr;
+    fop_fallocate_t fallocate;
+    fop_discard_t discard;
+    fop_zerofill_t zerofill;
+    fop_ipc_t ipc;
+    fop_seek_t seek;
+    fop_lease_t lease;
+    fop_compound_t compound;
+    fop_getactivelk_t getactivelk;
+    fop_setactivelk_t setactivelk;
+    fop_put_t put;
+    fop_icreate_t icreate;
+    fop_namelink_t namelink;
+    fop_copy_file_range_t copy_file_range;
+
+    /* these entries are used for a typechecking hack in STACK_WIND _only_ */
+    /* make sure to add _cbk variables only after defining regular fops as
+       its relative position is used to get the index */
+    fop_stat_cbk_t stat_cbk;
+    fop_readlink_cbk_t readlink_cbk;
+    fop_mknod_cbk_t mknod_cbk;
+    fop_mkdir_cbk_t mkdir_cbk;
+    fop_unlink_cbk_t unlink_cbk;
+    fop_rmdir_cbk_t rmdir_cbk;
+    fop_symlink_cbk_t symlink_cbk;
+    fop_rename_cbk_t rename_cbk;
+    fop_link_cbk_t link_cbk;
+    fop_truncate_cbk_t truncate_cbk;
+    fop_open_cbk_t open_cbk;
+    fop_readv_cbk_t readv_cbk;
+    fop_writev_cbk_t writev_cbk;
+    fop_statfs_cbk_t statfs_cbk;
+    fop_flush_cbk_t flush_cbk;
+    fop_fsync_cbk_t fsync_cbk;
+    fop_setxattr_cbk_t setxattr_cbk;
+    fop_getxattr_cbk_t getxattr_cbk;
+    fop_removexattr_cbk_t removexattr_cbk;
+    fop_opendir_cbk_t opendir_cbk;
+    fop_fsyncdir_cbk_t fsyncdir_cbk;
+    fop_access_cbk_t access_cbk;
+    fop_create_cbk_t create_cbk;
+    fop_ftruncate_cbk_t ftruncate_cbk;
+    fop_fstat_cbk_t fstat_cbk;
+    fop_lk_cbk_t lk_cbk;
+    fop_lookup_cbk_t lookup_cbk;
+    fop_readdir_cbk_t readdir_cbk;
+    fop_inodelk_cbk_t inodelk_cbk;
+    fop_finodelk_cbk_t finodelk_cbk;
+    fop_entrylk_cbk_t entrylk_cbk;
+    fop_fentrylk_cbk_t fentrylk_cbk;
+    fop_xattrop_cbk_t xattrop_cbk;
+    fop_fxattrop_cbk_t fxattrop_cbk;
+    fop_fgetxattr_cbk_t fgetxattr_cbk;
+    fop_fsetxattr_cbk_t fsetxattr_cbk;
+    fop_rchecksum_cbk_t rchecksum_cbk;
+    fop_setattr_cbk_t setattr_cbk;
+    fop_fsetattr_cbk_t fsetattr_cbk;
+    fop_readdirp_cbk_t readdirp_cbk;
+
+    /* These 3 are required to keep the index same as GF_FOP_##FOP */
+    void *forget_placeholder_cbk;
+    void *release_placeholder_cbk;
+    void *releasedir_placeholder_cbk;
+
+    fop_getspec_cbk_t getspec_cbk;
+    fop_fremovexattr_cbk_t fremovexattr_cbk;
+    fop_fallocate_cbk_t fallocate_cbk;
+    fop_discard_cbk_t discard_cbk;
+    fop_zerofill_cbk_t zerofill_cbk;
+    fop_ipc_cbk_t ipc_cbk;
+    fop_seek_cbk_t seek_cbk;
+    fop_lease_cbk_t lease_cbk;
+    fop_compound_cbk_t compound_cbk;
+    fop_getactivelk_cbk_t getactivelk_cbk;
+    fop_setactivelk_cbk_t setactivelk_cbk;
+    fop_put_cbk_t put_cbk;
+    fop_icreate_cbk_t icreate_cbk;
+    fop_namelink_cbk_t namelink_cbk;
+    fop_copy_file_range_cbk_t copy_file_range_cbk;
+};
+
+typedef int32_t (*cbk_forget_t)(xlator_t *this, inode_t *inode);
+
+typedef int32_t (*cbk_release_t)(xlator_t *this, fd_t *fd);
+
+typedef int32_t (*cbk_invalidate_t)(xlator_t *this, inode_t *inode);
+
+typedef int32_t (*cbk_client_t)(xlator_t *this, client_t *client);
+
+typedef void (*cbk_ictxmerge_t)(xlator_t *this, fd_t *fd, inode_t *inode,
+                                inode_t *linked_inode);
+
+typedef size_t (*cbk_inodectx_size_t)(xlator_t *this, inode_t *inode);
+
+typedef size_t (*cbk_fdctx_size_t)(xlator_t *this, fd_t *fd);
+
+typedef void (*cbk_fdclose_t)(xlator_t *this, fd_t *fd);
+
+struct xlator_cbks {
+    cbk_forget_t forget;
+    cbk_release_t release;
+    cbk_release_t releasedir;
+    cbk_invalidate_t invalidate;
+    cbk_client_t client_destroy;
+    cbk_client_t client_disconnect;
+    cbk_ictxmerge_t ictxmerge;
+    cbk_inodectx_size_t ictxsize;
+    cbk_fdctx_size_t fdctxsize;
+    cbk_fdclose_t fdclose;
+    cbk_fdclose_t fdclosedir;
+};
+
+typedef int32_t (*dumpop_priv_t)(xlator_t *this);
+
+typedef int32_t (*dumpop_inode_t)(xlator_t *this);
+
+typedef int32_t (*dumpop_fd_t)(xlator_t *this);
+
+typedef int32_t (*dumpop_inodectx_t)(xlator_t *this, inode_t *ino);
+
+typedef int32_t (*dumpop_fdctx_t)(xlator_t *this, fd_t *fd);
+
+typedef int32_t (*dumpop_priv_to_dict_t)(xlator_t *this, dict_t *dict,
+                                         char *brickname);
+
+typedef int32_t (*dumpop_inode_to_dict_t)(xlator_t *this, dict_t *dict);
+
+typedef int32_t (*dumpop_fd_to_dict_t)(xlator_t *this, dict_t *dict);
+
+typedef int32_t (*dumpop_inodectx_to_dict_t)(xlator_t *this, inode_t *ino,
+                                             dict_t *dict);
+
+typedef int32_t (*dumpop_fdctx_to_dict_t)(xlator_t *this, fd_t *fd,
+                                          dict_t *dict);
+
+typedef int32_t (*dumpop_eh_t)(xlator_t *this);
+
+struct xlator_dumpops {
+    dumpop_priv_t priv;
+    dumpop_inode_t inode;
+    dumpop_fd_t fd;
+    dumpop_inodectx_t inodectx;
+    dumpop_fdctx_t fdctx;
+    dumpop_priv_to_dict_t priv_to_dict;
+    dumpop_inode_to_dict_t inode_to_dict;
+    dumpop_fd_to_dict_t fd_to_dict;
+    dumpop_inodectx_to_dict_t inodectx_to_dict;
+    dumpop_fdctx_to_dict_t fdctx_to_dict;
+    dumpop_eh_t history;
+};
+
+typedef struct xlator_list {
+    xlator_t *xlator;
+    struct xlator_list *next;
+} xlator_list_t;
+
+typedef struct fop_metrics {
+    gf_atomic_t fop;
+    gf_atomic_t cbk; /* only updaed when there is failure */
+} fop_metrics_t;
+
+struct _xlator {
+    /* Built during parsing */
+    char *name;
+    char *type;
+    char *instance_name; /* Used for multi NFSd */
+    xlator_t *next;
+    xlator_t *prev;
+    xlator_list_t *parents;
+    xlator_list_t *children;
+    dict_t *options;
+
+    /* Set after doing dlopen() */
+    void *dlhandle;
+    struct xlator_fops *fops;
+    struct xlator_cbks *cbks;
+    struct xlator_dumpops *dumpops;
+    struct list_head volume_options; /* list of volume_option_t */
+
+    void (*fini)(xlator_t *this);
+    int32_t (*init)(xlator_t *this);
+    int32_t (*reconfigure)(xlator_t *this, dict_t *options);
+    int32_t (*mem_acct_init)(xlator_t *this);
+    int32_t (*dump_metrics)(xlator_t *this, int fd);
+
+    event_notify_fn_t notify;
+
+    gf_loglevel_t loglevel; /* Log level for translator */
+
+    struct {
+        struct {
+            /* for latency measurement */
+            fop_metrics_t metrics[GF_FOP_MAXVALUE];
+
+            gf_atomic_t count;
+        } total;
+
+        struct {
+            /* for latency measurement */
+            gf_latency_t latencies[GF_FOP_MAXVALUE];
+            /* for latency measurement */
+            fop_metrics_t metrics[GF_FOP_MAXVALUE];
+
+            gf_atomic_t count;
+        } interval;
+    } stats;
+
+    /* Misc */
+    eh_t *history; /* event history context */
+    glusterfs_ctx_t *ctx;
+    glusterfs_graph_t *graph; /* not set for fuse */
+    inode_table_t *itable;
+    char init_succeeded;
+    void *private;
+    struct mem_acct *mem_acct;
+    uint64_t winds;
+    char switched;
+
+    /* for the memory pool of 'frame->local' */
+    struct mem_pool *local_pool;
+    gf_boolean_t is_autoloaded;
+
+    /* Saved volfile ID (used for multiplexing) */
+    char *volfile_id;
+
+    /* Its used as an index to inode_ctx*/
+    uint32_t xl_id;
+
+    /* op_version: initialized in xlator code itself */
+    uint32_t op_version[GF_MAX_RELEASES];
+
+    /* flags: initialized in xlator code itself */
+    uint32_t flags;
+
+    /* id: unique, initialized in xlator code itself */
+    uint32_t id;
+
+    /* identifier: a full string which can unique identify the xlator */
+    char *identifier;
+
+    /* Is this pass_through? */
+    gf_boolean_t pass_through;
+    struct xlator_fops *pass_through_fops;
+
+    /* cleanup flag to avoid races during xlator cleanup */
+    uint32_t cleanup_starting;
+
+    /* flag to avoid recall of xlator_mem_cleanup for xame xlator */
+    uint32_t call_cleanup;
+
+    /* Flag to understand how this xlator is categorized */
+    gf_category_t category;
+
+    /* Variable to save xprt associated for detach brick */
+    gf_atomic_t xprtrefcnt;
+
+    /* Flag to notify got CHILD_DOWN event for detach brick */
+    uint32_t notify_down;
+
+    /* Flag to avoid throw duplicate PARENT_DOWN event */
+    uint32_t parent_down;
+};
+
+/* This would be the only structure which needs to be exported by
+   the translators. For the backward compatibility, in 4.x series
+   even the old exported fields will be supported */
+/* XXX: This struct is in use by GD2, and hence SHOULD NOT be modified.
+ * If the struct must be modified, see instructions at the comment with
+ * GD2MARKER below.
+ */
+typedef struct {
+    /* op_version: will be used by volume generation logic to figure
+       out whether to insert it in graph or no, based on cluster's
+       operating version.
+       default value: 0, which means good to insert always */
+    uint32_t op_version[GF_MAX_RELEASES];
+
+    /* flags: will be used by volume generation logic to optimize the
+       placements etc.
+       default value: 0, which means don't treat it specially */
+    uint32_t flags;
+
+    /* xlator_id: unique per xlator. make sure to have no collission
+       in this ID */
+    uint32_t xlator_id;
+
+    /* identifier: a string constant */
+    char *identifier;
+
+    /* struct options: if the translator takes any 'options' from the
+       volume file, then that should be defined here. optional. */
+    volume_option_t *options;
+
+    /* Flag to understand how this xlator is categorized */
+    gf_category_t category;
+
+    /* XXX: GD2MARKER
+     * If a new member that needs to be visible to GD2 is introduced,
+     * add it above this comment.
+     * Any other new members need to be added below this comment, or at the
+     * end of the struct
+     */
+
+    /* init(): mandatory method, will be called during the
+       graph initialization */
+    int32_t (*init)(xlator_t *this);
+
+    /* fini(): optional method, will be initialized to default
+       method which would just free the 'xlator->private' variable.
+       This method is called when the graph is no more in use, and
+       is being destroyed. Also when SIGTERM is received */
+    void (*fini)(xlator_t *this);
+
+    /* reconfigure(): optional method, will be initialized to default
+       method in case not provided by xlator. This method is called
+       when there are only option changes in xlator, and no graph change.
+       eg., a 'gluster volume set' command */
+    int32_t (*reconfigure)(xlator_t *this, dict_t *options);
+
+    /* mem_acct_init(): used for memory accounting inside of the xlator.
+       optional. called during translator initialization */
+    int32_t (*mem_acct_init)(xlator_t *this);
+
+    /* dump_metrics(): used for providing internal metrics. optional */
+    int32_t (*dump_metrics)(xlator_t *this, int fd);
+
+    /* notify(): used for handling the notification of events from either
+       the parent or child in the graph. optional. */
+    event_notify_fn_t notify;
+
+    /* struct fops: mandatory. provides all the filesystem operations
+       methods of the xlator */
+    struct xlator_fops *fops;
+    /* struct cbks: optional. provides methods to handle
+       inode forgets, and fd releases */
+    struct xlator_cbks *cbks;
+
+    /* dumpops: a structure again, with methods to dump the details.
+       optional. */
+    struct xlator_dumpops *dumpops;
+
+    /* struct pass_through_fops: optional. provides all the filesystem
+       operations which should be used if the xlator is marked as pass_through
+     */
+    /* by default, the default_fops would be used */
+    struct xlator_fops *pass_through_fops;
+} xlator_api_t;
+
+#define xlator_has_parent(xl) (xl->parents != NULL)
+
+#define XLATOR_NOTIFY(ret, _xl, params...)                                     \
+    do {                                                                       \
+        xlator_t *_old_THIS = NULL;                                            \
+                                                                               \
+        _old_THIS = THIS;                                                      \
+        THIS = _xl;                                                            \
+                                                                               \
+        ret = _xl->notify(_xl, params);                                        \
+                                                                               \
+        THIS = _old_THIS;                                                      \
+    } while (0);
+
+int32_t
+xlator_set_type_virtual(xlator_t *xl, const char *type);
+
+int32_t
+xlator_set_type(xlator_t *xl, const char *type);
+
+int32_t
+xlator_dynload(xlator_t *xl);
+
+xlator_t *
+file_to_xlator_tree(glusterfs_ctx_t *ctx, FILE *fp);
+
+int
+xlator_notify(xlator_t *this, int32_t event, void *data, ...);
+int
+xlator_init(xlator_t *this);
+int
+xlator_destroy(xlator_t *xl);
+
+int32_t
+xlator_tree_init(xlator_t *xl);
+int32_t
+xlator_tree_free_members(xlator_t *xl);
+int32_t
+xlator_tree_free_memacct(xlator_t *xl);
+
+void
+xlator_tree_fini(xlator_t *xl);
+
+void
+xlator_foreach(xlator_t *this, void (*fn)(xlator_t *each, void *data),
+               void *data);
+
+void
+xlator_foreach_depth_first(xlator_t *this,
+                           void (*fn)(xlator_t *each, void *data), void *data);
+
+xlator_t *
+xlator_search_by_name(xlator_t *any, const char *name);
+xlator_t *
+get_xlator_by_name(xlator_t *this, char *target);
+xlator_t *
+get_xlator_by_type(xlator_t *this, char *target);
+
+void
+xlator_set_inode_lru_limit(xlator_t *this, void *data);
+
+void
+inode_destroy_notify(inode_t *inode, const char *xlname);
+
+int
+loc_copy(loc_t *dst, loc_t *src);
+int
+loc_copy_overload_parent(loc_t *dst, loc_t *src, inode_t *parent);
+#define loc_dup(src, dst) loc_copy(dst, src)
+void
+loc_wipe(loc_t *loc);
+int
+loc_path(loc_t *loc, const char *bname);
+void
+loc_gfid(loc_t *loc, uuid_t gfid);
+void
+loc_pargfid(loc_t *loc, uuid_t pargfid);
+char *
+loc_gfid_utoa(loc_t *loc);
+gf_boolean_t
+loc_is_root(loc_t *loc);
+int32_t
+loc_build_child(loc_t *child, loc_t *parent, char *name);
+gf_boolean_t
+loc_is_nameless(loc_t *loc);
+int
+xlator_mem_acct_init(xlator_t *xl, int num_types);
+void
+xlator_mem_acct_unref(struct mem_acct *mem_acct);
+int
+is_gf_log_command(xlator_t *trans, const char *name, char *value, size_t size);
+int
+glusterd_check_log_level(const char *value);
+int
+xlator_volopt_dynload(char *xlator_type, void **dl_handle,
+                      volume_opt_list_t *vol_opt_handle);
+enum gf_hdsk_event_notify_op {
+    GF_EN_DEFRAG_STATUS,
+    GF_EN_MAX,
+};
+gf_boolean_t
+is_graph_topology_equal(glusterfs_graph_t *graph1, glusterfs_graph_t *graph2);
+int
+glusterfs_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx);
+
+int
+gf_volfile_reconfigure(int oldvollen, FILE *newvolfile_fp, glusterfs_ctx_t *ctx,
+                       const char *oldvolfile);
+
+int
+loc_touchup(loc_t *loc, const char *name);
+
+int
+glusterfs_leaf_position(xlator_t *tgt);
+
+int
+glusterfs_reachable_leaves(xlator_t *base, dict_t *leaves);
+
+int
+xlator_subvolume_count(xlator_t *this);
+
+void
+xlator_init_lock(void);
+void
+xlator_init_unlock(void);
+int
+copy_opts_to_child(xlator_t *src, xlator_t *dst, char *glob);
+
+int
+glusterfs_delete_volfile_checksum(glusterfs_ctx_t *ctx, const char *volfile_id);
+int
+xlator_memrec_free(xlator_t *xl);
+
+void
+xlator_mem_cleanup(xlator_t *this);
+
+void
+handle_default_options(xlator_t *xl, dict_t *options);
+
+void
+gluster_graph_take_reference(xlator_t *tree);
+
+gf_boolean_t
+mgmt_is_multiplexed_daemon(char *name);
+
+gf_boolean_t
+xlator_is_cleanup_starting(xlator_t *this);
+int
+graph_total_client_xlator(glusterfs_graph_t *graph);
+#endif /* _XLATOR_H */
diff --git a/libglusterfs/src/graph-mem-types.h b/libglusterfs/src/graph-mem-types.h
deleted file mode 100644
index 86e0749cd03..00000000000
--- a/libglusterfs/src/graph-mem-types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __GRAPH_MEM_TYPES_H__
-#define __GRAPH_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-#define GF_MEM_TYPE_START (gf_common_mt_end + 1)
-
-enum gf_graph_mem_types {
-        gf_graph_mt_buf = GF_MEM_TYPE_START,
-        gf_graph_mt_end
-};
-#endif
diff --git a/libglusterfs/src/graph-print.c b/libglusterfs/src/graph-print.c
index 53d8643ee0d..595d74330a1 100644
--- a/libglusterfs/src/graph-print.c
+++ b/libglusterfs/src/graph-print.c
@@ -1,200 +1,135 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <sys/uio.h>
 
-#include "common-utils.h"
-#include "xlator.h"
-#include "graph-mem-types.h"
-#include "graph-utils.h"
-
-
+#include "glusterfs/common-utils.h"
+#include "glusterfs/xlator.h"
+#include "glusterfs/graph-utils.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 struct gf_printer {
-        ssize_t (*write) (struct gf_printer *gp, char *buf, size_t len);
-        void *priv;
+    ssize_t (*write)(struct gf_printer *gp, char *buf, size_t len);
+    void *priv;
+    int len;
 };
 
 static ssize_t
-gp_write_file (struct gf_printer *gp, char *buf, size_t len)
+gp_write_file(struct gf_printer *gp, char *buf, size_t len)
 {
-        FILE *f = gp->priv;
+    FILE *f = gp->priv;
 
-        if (fwrite (buf, len, 1, f) != 1) {
-                gf_log ("graph-print", GF_LOG_ERROR, "fwrite failed (%s)",
-                        strerror (errno));
+    if (fwrite(buf, len, 1, f) != 1) {
+        gf_msg("graph-print", GF_LOG_ERROR, errno, LG_MSG_FWRITE_FAILED,
+               "fwrite failed");
 
-                return -1;
-        }
+        return -1;
+    }
 
-        return len;
+    return len;
 }
 
-static ssize_t
-gp_write_buf (struct gf_printer *gp, char *buf, size_t len)
+static int
+gpprintf(struct gf_printer *gp, const char *format, ...)
 {
-        struct iovec *iov = gp->priv;
+    va_list arg;
+    char *str = NULL;
+    int ret = 0;
 
-        if (iov->iov_len < len) {
-                gf_log ("graph-print", GF_LOG_ERROR, "buffer full");
+    va_start(arg, format);
+    ret = gf_vasprintf(&str, format, arg);
+    va_end(arg);
 
-                return -1;
-        }
+    if (ret < 0)
+        return ret;
+
+    ret = gp->write(gp, str, ret);
 
-        memcpy (iov->iov_base, buf, len);
-        iov->iov_base += len;
-        iov->iov_len  -= len;
+    GF_FREE(str);
 
-        return len;
+    return ret;
 }
 
+#define GPPRINTF(gp, fmt, ...)                                                 \
+    do {                                                                       \
+        ret = gpprintf(gp, fmt, ##__VA_ARGS__);                                \
+        if (ret == -1)                                                         \
+            goto out;                                                          \
+        else                                                                   \
+            gp->len += ret;                                                    \
+    } while (0)
+
 static int
-gpprintf (struct gf_printer *gp, const char *format, ...)
+_print_volume_options(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        va_list arg;
-        char *str = NULL;
-        int ret = 0;
+    struct gf_printer *gp = tmp;
+    int ret = 0;
+    GPPRINTF(gp, "    option %s %s\n", k, v->data);
+    return 0;
+out:
+    /* means, it is a failure */
+    return -1;
+}
 
-        va_start (arg, format);
-        ret = gf_vasprintf (&str, format, arg);
-        va_end (arg);
+static int
+glusterfs_graph_print(struct gf_printer *gp, glusterfs_graph_t *graph)
+{
+    xlator_t *trav = NULL;
+    xlator_list_t *xch = NULL;
+    int ret = 0;
+    ssize_t len = 0;
 
-        if (ret < 0)
-                return ret;
+    if (!graph->first)
+        return 0;
 
-        ret = gp->write (gp, str, ret);
+    for (trav = graph->first; trav->next; trav = trav->next)
+        ;
+    for (; trav; trav = trav->prev) {
+        GPPRINTF(gp, "volume %s\n    type %s\n", trav->name, trav->type);
 
-        GF_FREE (str);
+        ret = dict_foreach(trav->options, _print_volume_options, gp);
+        if (ret)
+            goto out;
 
-        return ret;
-}
+        if (trav->children) {
+            GPPRINTF(gp, "    subvolumes");
 
-static int
-glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
-{
-#define GPPRINTF(gp, fmt, ...) do {                    \
-        ret = gpprintf (gp, fmt, ## __VA_ARGS__);      \
-        if (ret == -1)                                 \
-                goto out;                              \
-        else                                           \
-                len += ret;                            \
-} while (0)
-
-        xlator_t      *trav = NULL;
-        data_pair_t   *pair = NULL;
-        xlator_list_t *xch = NULL;
-        int            ret = 0;
-        ssize_t        len = 0;
-
-        if (!graph->first)
-                return 0;
-
-        for (trav = graph->first; trav->next; trav = trav->next);
-        for (; trav; trav = trav->prev) {
-                GPPRINTF (gp, "volume %s\n    type %s\n", trav->name,
-                          trav->type);
-
-                for (pair =  trav->options->members_list; pair && pair->next;
-                     pair = pair->next);
-                for (; pair; pair = pair->prev)
-                        GPPRINTF (gp, "    option %s %s\n", pair->key,
-                                  pair->value->data);
-
-                if (trav->children) {
-                        GPPRINTF (gp, "    subvolumes");
-
-                        for (xch = trav->children; xch; xch = xch->next)
-                                GPPRINTF (gp, " %s", xch->xlator->name);
-
-                        GPPRINTF (gp, "\n");
-                }
-
-                GPPRINTF (gp, "end-volume\n");
-                if (trav != graph->first)
-                        GPPRINTF (gp, "\n");
+            for (xch = trav->children; xch; xch = xch->next)
+                GPPRINTF(gp, " %s", xch->xlator->name);
+
+            GPPRINTF(gp, "\n");
         }
 
- out:
-        if (ret == -1) {
-                gf_log ("graph-print", GF_LOG_ERROR, "printing failed");
+        GPPRINTF(gp, "end-volume\n");
+        if (trav != graph->first)
+            GPPRINTF(gp, "\n");
+    }
 
-                return -1;
-        }
+out:
+    len = gp->len;
+    if (ret == -1) {
+        gf_msg("graph-print", GF_LOG_ERROR, 0, LG_MSG_PRINT_FAILED,
+               "printing failed");
 
-        return len;
+        return -1;
+    }
+
+    return len;
 
 #undef GPPRINTF
 }
 
 int
-glusterfs_graph_print_file (FILE *file, glusterfs_graph_t *graph)
-{
-        struct gf_printer gp = { .write = gp_write_file,
-                                 .priv  = file
-                               };
-
-        return glusterfs_graph_print (&gp, graph);
-}
-
-char *
-glusterfs_graph_print_buf (glusterfs_graph_t *graph)
+glusterfs_graph_print_file(FILE *file, glusterfs_graph_t *graph)
 {
-        FILE *f = NULL;
-        struct iovec iov = {0,};
-        int len = 0;
-        char *buf = NULL;
-        struct gf_printer gp = { .write = gp_write_buf,
-                                 .priv  = &iov
-                               };
-
-        f = fopen ("/dev/null", "a");
-        if (!f) {
-                gf_log ("graph-print", GF_LOG_ERROR,
-                        "cannot open /dev/null (%s)", strerror (errno));
-
-                return NULL;
-        }
-        len = glusterfs_graph_print_file (f, graph);
-        fclose (f);
-        if (len == -1)
-                return NULL;
-
-        buf = GF_CALLOC (1, len + 1, gf_graph_mt_buf);
-        if (!buf) {
-                gf_log ("graph-print", GF_LOG_ERROR, "Out of memory");
-
-                return NULL;
-        }
-        iov.iov_base = buf;
-        iov.iov_len  = len;
-
-        len = glusterfs_graph_print (&gp, graph);
-        if (len == -1) {
-                GF_FREE (buf);
-
-                return NULL;
-        }
+    struct gf_printer gp = {.write = gp_write_file, .priv = file};
 
-        return buf;
+    return glusterfs_graph_print(&gp, graph);
 }
diff --git a/libglusterfs/src/graph-utils.h b/libglusterfs/src/graph-utils.h
deleted file mode 100644
index a6c1ccd5ac4..00000000000
--- a/libglusterfs/src/graph-utils.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _GRAPH_H_
-#define _GRAPH_H_
-
-int
-glusterfs_graph_print_file (FILE *file, glusterfs_graph_t *graph);
-
-char *
-glusterfs_graph_print_buf (glusterfs_graph_t *graph);
-
-int glusterfs_xlator_link (xlator_t *pxl, xlator_t *cxl);
-void glusterfs_graph_set_first (glusterfs_graph_t *graph, xlator_t *xl);
-#endif
diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c
index a2d03fa0b64..13f298eb3bd 100644
--- a/libglusterfs/src/graph.c
+++ b/libglusterfs/src/graph.c
@@ -1,35 +1,39 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include <dlfcn.h>
-#include <netdb.h>
-#include <fnmatch.h>
-#include "defaults.h"
-
-
-
+#include <stdint.h>                  // for uint32_t
+#include <sys/time.h>                // for timeval
+#include <errno.h>                   // for EIO, errno, EINVAL, ENOMEM
+#include <fnmatch.h>                 // for fnmatch, FNM_NOESCAPE
+#include <openssl/sha.h>             // for SHA256_DIGEST_LENGTH
+#include <regex.h>                   // for regmatch_t, regcomp
+#include <stdio.h>                   // for fclose, fopen, snprintf
+#include <stdlib.h>                  // for NULL, atoi, mkstemp
+#include <string.h>                  // for strcmp, strerror, memcpy
+#include <strings.h>                 // for rindex
+#include <sys/stat.h>                // for stat
+#include <sys/time.h>                // for gettimeofday
+#include <unistd.h>                  // for gethostname, getpid
+#include "glusterfs/common-utils.h"  // for gf_strncpy, gf_time_fmt
+#include "glusterfs/defaults.h"
+#include "glusterfs/dict.h"                   // for dict_foreach, dict_set_...
+#include "glusterfs/globals.h"                // for xlator_t, xlator_list_t
+#include "glusterfs/glusterfs.h"              // for glusterfs_graph_t, glus...
+#include "glusterfs/glusterfs-fops.h"         // for GF_EVENT_GRAPH_NEW, GF_...
+#include "glusterfs/libglusterfs-messages.h"  // for LG_MSG_GRAPH_ERROR, LG_...
+#include "glusterfs/list.h"                   // for list_add, list_del_init
+#include "glusterfs/logging.h"                // for gf_msg, GF_LOG_ERROR
+#include "glusterfs/mem-pool.h"               // for GF_FREE, gf_strdup, GF_...
+#include "glusterfs/mem-types.h"              // for gf_common_mt_xlator_list_t
+#include "glusterfs/options.h"                // for xlator_tree_reconfigure
+#include "glusterfs/syscall.h"                // for sys_close, sys_stat
 
 #if 0
 static void
@@ -37,20 +41,17 @@ _gf_dump_details (int argc, char **argv)
 {
         extern FILE *gf_log_logfile;
         int          i = 0;
-        char         timestr[256];
+        char         timestr[GF_TIMESTR_SIZE];
         time_t       utime = 0;
-        struct tm   *tm = NULL;
         pid_t        mypid = 0;
         struct utsname uname_buf = {{0, }, };
         int            uname_ret = -1;
 
-        utime = time (NULL);
-        tm    = localtime (&utime);
         mypid = getpid ();
         uname_ret   = uname (&uname_buf);
 
-        /* Which git? What time? */
-        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+        utime = time (NULL);
+        gf_time_fmt (timestr, sizeof timestr, utime, gf_timefmt_FT);
         fprintf (gf_log_logfile,
                  "========================================"
                  "========================================\n");
@@ -83,459 +84,1797 @@ _gf_dump_details (int argc, char **argv)
 }
 #endif
 
-
-static int
-_log_if_option_is_invalid (xlator_t *xl, data_pair_t *pair)
-{
-        volume_opt_list_t *vol_opt = NULL;
-        volume_option_t   *opt     = NULL;
-        int                i     = 0;
-        int                index = 0;
-        int                found = 0;
-
-        /* Get the first volume_option */
-        list_for_each_entry (vol_opt, &xl->volume_options, list) {
-                /* Warn for extra option */
-                if (!vol_opt->given_opt)
-                        break;
-
-                opt = vol_opt->given_opt;
-                for (index = 0;
-                     ((index < ZR_OPTION_MAX_ARRAY_SIZE) &&
-                      (opt[index].key && opt[index].key[0]));  index++)
-                        for (i = 0; (i < ZR_VOLUME_MAX_NUM_KEY) &&
-                                     opt[index].key[i]; i++) {
-                                if (fnmatch (opt[index].key[i],
-                                             pair->key,
-                                             FNM_NOESCAPE) == 0) {
-                                        found = 1;
-                                        break;
-                                }
-                        }
+int
+glusterfs_read_secure_access_file(void)
+{
+    FILE *fp = NULL;
+    char line[100] = {
+        0,
+    };
+    int cert_depth = 1; /* Default SSL CERT DEPTH */
+    regex_t regcmpl;
+    char *key = {"^option transport.socket.ssl-cert-depth"};
+    char keyval[50] = {
+        0,
+    };
+    int start = 0, end = 0, copy_len = 0;
+    regmatch_t result[1] = {{0}};
+
+    fp = fopen(SECURE_ACCESS_FILE, "r");
+    if (!fp)
+        goto out;
+
+    /* Check if any line matches with key */
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (regcomp(&regcmpl, key, REG_EXTENDED)) {
+            goto out;
         }
-
-        if (!found) {
-                gf_log (xl->name, GF_LOG_WARNING,
-                        "option '%s' is not recognized",
-                        pair->key);
+        if (!regexec(&regcmpl, line, 1, result, 0)) {
+            start = result[0].rm_so;
+            end = result[0].rm_eo;
+            copy_len = end - start;
+            gf_strncpy(keyval, line + copy_len, sizeof(keyval));
+            if (keyval[0]) {
+                cert_depth = atoi(keyval);
+                if (cert_depth == 0)
+                    cert_depth = 1; /* Default SSL CERT DEPTH */
+                break;
+            }
         }
-        return 0;
-}
+        regfree(&regcmpl);
+    }
 
+out:
+    if (fp)
+        fclose(fp);
+    return cert_depth;
+}
 
-int
-glusterfs_xlator_link (xlator_t *pxl, xlator_t *cxl)
+xlator_t *
+glusterfs_get_last_xlator(glusterfs_graph_t *graph)
 {
-        xlator_list_t   *xlchild = NULL;
-        xlator_list_t   *xlparent = NULL;
-        xlator_list_t  **tmp = NULL;
+    xlator_t *trav = graph->first;
+    if (!trav)
+        return NULL;
 
-        xlparent = (void *) GF_CALLOC (1, sizeof (*xlparent),
-                                       gf_common_mt_xlator_list_t);
-        if (!xlparent)
-                return -1;
+    while (trav->next)
+        trav = trav->next;
 
-        xlchild = (void *) GF_CALLOC (1, sizeof (*xlchild),
-                                      gf_common_mt_xlator_list_t);
-        if (!xlchild) {
-                GF_FREE (xlparent);
+    return trav;
+}
 
-                return -1;
+xlator_t *
+glusterfs_mux_xlator_unlink(xlator_t *pxl, xlator_t *cxl)
+{
+    xlator_list_t *unlink = NULL;
+    xlator_list_t *prev = NULL;
+    xlator_list_t **tmp = NULL;
+    xlator_t *next_child = NULL;
+    xlator_t *xl = NULL;
+
+    for (tmp = &pxl->children; *tmp; tmp = &(*tmp)->next) {
+        if ((*tmp)->xlator == cxl) {
+            unlink = *tmp;
+            *tmp = (*tmp)->next;
+            if (*tmp)
+                next_child = (*tmp)->xlator;
+            break;
         }
+        prev = *tmp;
+    }
 
-        xlparent->xlator = pxl;
-        for (tmp = &cxl->parents; *tmp; tmp = &(*tmp)->next);
-        *tmp = xlparent;
+    if (!prev)
+        xl = pxl;
+    else if (prev->xlator)
+        xl = prev->xlator->graph->last_xl;
 
-        xlchild->xlator = cxl;
-        for (tmp = &pxl->children; *tmp; tmp = &(*tmp)->next);
-        *tmp = xlchild;
+    if (xl)
+        xl->next = next_child;
+    if (next_child)
+        next_child->prev = xl;
 
-        return 0;
+    GF_FREE(unlink);
+    return next_child;
 }
 
+int
+glusterfs_xlator_link(xlator_t *pxl, xlator_t *cxl)
+{
+    xlator_list_t *xlchild = NULL;
+    xlator_list_t *xlparent = NULL;
+    xlator_list_t **tmp = NULL;
+
+    xlparent = (void *)GF_CALLOC(1, sizeof(*xlparent),
+                                 gf_common_mt_xlator_list_t);
+    if (!xlparent)
+        return -1;
+
+    xlchild = (void *)GF_CALLOC(1, sizeof(*xlchild),
+                                gf_common_mt_xlator_list_t);
+    if (!xlchild) {
+        GF_FREE(xlparent);
+
+        return -1;
+    }
+
+    xlparent->xlator = pxl;
+    for (tmp = &cxl->parents; *tmp; tmp = &(*tmp)->next)
+        ;
+    *tmp = xlparent;
+
+    xlchild->xlator = cxl;
+    for (tmp = &pxl->children; *tmp; tmp = &(*tmp)->next)
+        ;
+    *tmp = xlchild;
+
+    return 0;
+}
 
 void
-glusterfs_graph_set_first (glusterfs_graph_t *graph, xlator_t *xl)
+glusterfs_graph_set_first(glusterfs_graph_t *graph, xlator_t *xl)
 {
-        xl->next = graph->first;
-        if (graph->first)
-                ((xlator_t *)graph->first)->prev = xl;
-        graph->first = xl;
+    xl->next = graph->first;
+    if (graph->first)
+        ((xlator_t *)graph->first)->prev = xl;
+    graph->first = xl;
 
-        graph->xl_count++;
+    graph->xl_count++;
+    xl->xl_id = graph->xl_count;
 }
 
-
 int
-glusterfs_graph_insert (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
-                        const char *type, const char *name)
+glusterfs_graph_insert(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
+                       const char *type, const char *name,
+                       gf_boolean_t autoload)
 {
-        xlator_t        *ixl = NULL;
+    xlator_t *ixl = NULL;
 
-        if (!ctx->master) {
-                gf_log ("glusterfs", GF_LOG_ERROR,
-                        "volume \"%s\" can be added from command line only "
-                        "on client side", type);
+    if (!ctx->master) {
+        gf_msg("glusterfs", GF_LOG_ERROR, 0, LG_MSG_VOLUME_ERROR,
+               "volume \"%s\" can be added from command line only "
+               "on client side",
+               type);
 
-                return -1;
-        }
+        return -1;
+    }
 
-        ixl = GF_CALLOC (1, sizeof (*ixl), gf_common_mt_xlator_t);
-        if (!ixl)
-                return -1;
-
-        ixl->ctx      = ctx;
-        ixl->graph    = graph;
-        ixl->options  = get_new_dict ();
-        if (!ixl->options)
-                goto err;
-
-        ixl->name  = gf_strdup (name);
-        if (!ixl->name)
-                goto err;
-
-        if (xlator_set_type (ixl, type) == -1) {
-                gf_log ("glusterfs", GF_LOG_ERROR,
-                        "%s (%s) initialization failed",
-                        name, type);
-                return -1;
-        }
+    ixl = GF_CALLOC(1, sizeof(*ixl), gf_common_mt_xlator_t);
+    if (!ixl)
+        return -1;
 
-        if (glusterfs_xlator_link (ixl, graph->top) == -1)
-                goto err;
-        glusterfs_graph_set_first (graph, ixl);
-        graph->top = ixl;
+    ixl->ctx = ctx;
+    ixl->graph = graph;
+    ixl->options = dict_new();
+    if (!ixl->options)
+        goto err;
 
-        return 0;
-err:
-        xlator_destroy (ixl);
+    ixl->name = gf_strdup(name);
+    if (!ixl->name)
+        goto err;
+
+    ixl->is_autoloaded = autoload;
+
+    if (xlator_set_type(ixl, type) == -1) {
+        gf_msg("glusterfs", GF_LOG_ERROR, 0, LG_MSG_INIT_FAILED,
+               "%s (%s) initialization failed", name, type);
         return -1;
+    }
+
+    if (glusterfs_xlator_link(ixl, graph->top) == -1)
+        goto err;
+    glusterfs_graph_set_first(graph, ixl);
+    graph->top = ixl;
+
+    return 0;
+err:
+    xlator_destroy(ixl);
+    return -1;
 }
 
+int
+glusterfs_graph_acl(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args->acl)
+        return 0;
+
+    ret = glusterfs_graph_insert(graph, ctx, "system/posix-acl",
+                                 "posix-acl-autoload", 1);
+    return ret;
+}
 
 int
-glusterfs_graph_readonly (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+glusterfs_graph_worm(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
 {
-        int ret = 0;
-        cmd_args_t      *cmd_args = NULL;
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->read_only)
-                return 0;
+    if (!cmd_args->worm)
+        return 0;
 
-        ret = glusterfs_graph_insert (graph, ctx, "features/read-only",
-                                      "readonly-autoload");
-        return ret;
+    ret = glusterfs_graph_insert(graph, ctx, "features/worm", "worm-autoload",
+                                 1);
+    return ret;
 }
 
+int
+glusterfs_graph_meta(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
+
+    if (!ctx->master)
+        return 0;
+
+    ret = glusterfs_graph_insert(graph, ctx, "meta", "meta-autoload", 1);
+    return ret;
+}
 
 int
-glusterfs_graph_mac_compat (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+glusterfs_graph_mac_compat(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
 {
-        int ret = 0;
-        cmd_args_t      *cmd_args = NULL;
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        if (cmd_args->mac_compat == GF_OPTION_DISABLE)
-                return 0;
+    if (cmd_args->mac_compat == GF_OPTION_DISABLE)
+        return 0;
 
-        ret = glusterfs_graph_insert (graph, ctx, "features/mac-compat",
-                                      "mac-compat-autoload");
+    ret = glusterfs_graph_insert(graph, ctx, "features/mac-compat",
+                                 "mac-compat-autoload", 1);
 
-        return ret;
+    return ret;
 }
 
+int
+glusterfs_graph_gfid_access(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
+    cmd_args_t *cmd_args = NULL;
+
+    cmd_args = &ctx->cmd_args;
+
+    if (!cmd_args->aux_gfid_mount)
+        return 0;
+
+    ret = glusterfs_graph_insert(graph, ctx, "features/gfid-access",
+                                 "gfid-access-autoload", 1);
+    return ret;
+}
 
 static void
-gf_add_cmdline_options (glusterfs_graph_t *graph, cmd_args_t *cmd_args)
-{
-        int                      ret = 0;
-        xlator_t                *trav = NULL;
-        xlator_cmdline_option_t *cmd_option = NULL;
-
-        trav = graph->first;
-
-        while (trav) {
-                list_for_each_entry (cmd_option,
-                                     &cmd_args->xlator_options, cmd_args) {
-                        if (!fnmatch (cmd_option->volume,
-                                      trav->name, FNM_NOESCAPE)) {
-                                ret = dict_set_str (trav->options,
-                                                    cmd_option->key,
-                                                    cmd_option->value);
-                                if (ret == 0) {
-                                        gf_log (trav->name, GF_LOG_WARNING,
-                                                "adding option '%s' for "
-                                                "volume '%s' with value '%s'",
-                                                cmd_option->key, trav->name,
-                                                cmd_option->value);
-                                } else {
-                                        gf_log (trav->name, GF_LOG_WARNING,
-                                                "adding option '%s' for "
-                                                "volume '%s' failed: %s",
-                                                cmd_option->key, trav->name,
-                                                strerror (-ret));
-                                }
-                        }
+gf_add_cmdline_options(glusterfs_graph_t *graph, cmd_args_t *cmd_args)
+{
+    int ret = 0;
+    xlator_t *trav = NULL;
+    xlator_cmdline_option_t *cmd_option = NULL;
+
+    trav = graph->first;
+
+    while (trav) {
+        list_for_each_entry(cmd_option, &cmd_args->xlator_options, cmd_args)
+        {
+            if (!fnmatch(cmd_option->volume, trav->name, FNM_NOESCAPE)) {
+                ret = dict_set_str(trav->options, cmd_option->key,
+                                   cmd_option->value);
+                if (ret == 0) {
+                    gf_msg(trav->name, GF_LOG_TRACE, 0, LG_MSG_VOL_OPTION_ADD,
+                           "adding option '%s' for "
+                           "volume '%s' with value '%s'",
+                           cmd_option->key, trav->name, cmd_option->value);
+                } else {
+                    gf_msg(trav->name, GF_LOG_WARNING, -ret,
+                           LG_MSG_VOL_OPTION_ADD,
+                           "adding option '%s' for "
+                           "volume '%s' failed",
+                           cmd_option->key, trav->name);
                 }
-                trav = trav->next;
+            }
         }
+        trav = trav->next;
+    }
 }
 
-
 int
-glusterfs_graph_validate_options (glusterfs_graph_t *graph)
+glusterfs_graph_validate_options(glusterfs_graph_t *graph)
 {
-        volume_opt_list_t  *vol_opt = NULL;
-        xlator_t           *trav = NULL;
-        int                 ret = -1;
-
-        trav = graph->first;
+    xlator_t *trav = NULL;
+    int ret = -1;
+    char *errstr = NULL;
 
-        while (trav) {
-                if (list_empty (&trav->volume_options))
-                        continue;
+    trav = graph->first;
 
-                vol_opt = list_entry (trav->volume_options.next,
-                                      volume_opt_list_t, list);
+    while (trav) {
+        if (list_empty(&trav->volume_options)) {
+            trav = trav->next;
+            continue;
+        }
 
-                ret = validate_xlator_volume_options (trav,
-                                                      vol_opt->given_opt);
-                if (ret) {
-                        gf_log (trav->name, GF_LOG_ERROR,
-                                "validating translator failed");
-                        return ret;
-                }
-                trav = trav->next;
+        ret = xlator_options_validate(trav, trav->options, &errstr);
+        if (ret) {
+            gf_msg(trav->name, GF_LOG_ERROR, 0, LG_MSG_VALIDATION_FAILED,
+                   "validation failed: "
+                   "%s",
+                   errstr);
+            return ret;
         }
+        trav = trav->next;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-glusterfs_graph_init (glusterfs_graph_t *graph)
+glusterfs_graph_init(glusterfs_graph_t *graph)
 {
-        xlator_t           *trav = NULL;
-        int                 ret = -1;
+    xlator_t *trav = NULL;
+    int ret = -1;
 
-        trav = graph->first;
+    trav = graph->first;
 
-        while (trav) {
-                ret = xlator_init (trav);
-                if (ret) {
-                        gf_log (trav->name, GF_LOG_ERROR,
-                                "initializing translator failed");
-                        return ret;
-                }
-                trav = trav->next;
+    while (trav) {
+        ret = xlator_init(trav);
+        if (ret) {
+            gf_msg(trav->name, GF_LOG_ERROR, 0, LG_MSG_TRANSLATOR_INIT_FAILED,
+                   "initializing translator failed");
+            return ret;
         }
+        trav = trav->next;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-glusterfs_graph_unknown_options (glusterfs_graph_t *graph)
+glusterfs_graph_deactivate(glusterfs_graph_t *graph)
 {
-        data_pair_t        *pair = NULL;
-        xlator_t           *trav = NULL;
+    xlator_t *top = NULL;
 
-        trav = graph->first;
+    if (graph == NULL)
+        goto out;
 
-        /* Validate again phase */
-        while (trav) {
-                pair = trav->options->members_list;
-                while (pair) {
-                        _log_if_option_is_invalid (trav, pair);
-                        pair = pair->next;
-                }
-                trav = trav->next;
-        }
+    top = graph->top;
+    xlator_tree_fini(top);
+out:
+    return 0;
+}
 
-        return 0;
+static int
+_log_if_unknown_option(dict_t *dict, char *key, data_t *value, void *data)
+{
+    volume_option_t *found = NULL;
+    xlator_t *xl = NULL;
+
+    xl = data;
+
+    found = xlator_volume_option_get(xl, key);
+
+    if (!found) {
+        gf_msg(xl->name, GF_LOG_DEBUG, 0, LG_MSG_XLATOR_OPTION_INVALID,
+               "option '%s' is not recognized", key);
+    }
+
+    return 0;
 }
 
+static void
+_xlator_check_unknown_options(xlator_t *xl, void *data)
+{
+    dict_foreach(xl->options, _log_if_unknown_option, xl);
+}
 
-void
-fill_uuid (char *uuid, int size)
+static int
+glusterfs_graph_unknown_options(glusterfs_graph_t *graph)
 {
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
+    xlator_foreach(graph->first, _xlator_check_unknown_options, NULL);
+    return 0;
+}
 
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("graph", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
+static void
+fill_uuid(char *uuid, int size, struct timeval tv)
+{
+    char hostname[50] = {
+        0,
+    };
+    char now_str[GF_TIMESTR_SIZE];
 
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("graph", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
-        }
+    if (gethostname(hostname, sizeof(hostname) - 1) != 0) {
+        gf_msg("graph", GF_LOG_ERROR, errno, LG_MSG_GETHOSTNAME_FAILED,
+               "gethostname failed");
+        hostname[sizeof(hostname) - 1] = '\0';
+    }
 
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (uuid, size, "%s-%d-%s:%"GF_PRI_SUSECONDS,
-                  hostname, getpid(), now_str, tv.tv_usec);
+    gf_time_fmt_tv(now_str, sizeof now_str, &tv, gf_timefmt_dirent);
+    snprintf(uuid, size, "%s-%d-%s", hostname, getpid(), now_str);
 
-        return;
+    return;
 }
 
+static int
+glusterfs_graph_settop(glusterfs_graph_t *graph, char *volume_name,
+                       gf_boolean_t exact_match)
+{
+    int ret = -1;
+    xlator_t *trav = NULL;
+
+    if (!volume_name || !exact_match) {
+        graph->top = graph->first;
+        ret = 0;
+    } else {
+        for (trav = graph->first; trav; trav = trav->next) {
+            if (strcmp(trav->name, volume_name) == 0) {
+                graph->top = trav;
+                ret = 0;
+                break;
+            }
+        }
+    }
+
+    return ret;
+}
 
 int
-glusterfs_graph_settop (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+glusterfs_graph_parent_up(glusterfs_graph_t *graph)
 {
-        const char *volume_name = NULL;
-        xlator_t   *trav = NULL;
+    xlator_t *trav = NULL;
+    int ret = -1;
 
-        volume_name = ctx->cmd_args.volume_name;
+    trav = graph->first;
 
-        if (!volume_name) {
-                graph->top = graph->first;
-                return 0;
+    while (trav) {
+        if (!xlator_has_parent(trav)) {
+            ret = xlator_notify(trav, GF_EVENT_PARENT_UP, trav);
         }
 
-        for (trav = graph->first; trav; trav = trav->next) {
-                if (strcmp (trav->name, volume_name) == 0) {
-                        graph->top = trav;
-                        return 0;
-                }
-        }
+        if (ret)
+            break;
 
+        trav = trav->next;
+    }
+
+    return ret;
+}
+
+int
+glusterfs_graph_prepare(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
+                        char *volume_name)
+{
+    xlator_t *trav = NULL;
+    int ret = 0;
+
+    /* XXX: CHECKSUM */
+
+    /* XXX: attach to -n volname */
+    /* A '/' in the volume name suggests brick multiplexing is used, find
+     * the top of the (sub)graph. The volname MUST match the subvol in this
+     * case. In other cases (like for gfapi) the default top for the
+     * (sub)graph is ok. */
+    if (!volume_name) {
+        /* GlusterD does not pass a volume_name */
+        ret = glusterfs_graph_settop(graph, volume_name, _gf_false);
+    } else if (strncmp(volume_name, "/snaps/", 7) == 0) {
+        /* snap shots have their top xlator named like "/snaps/..."  */
+        ret = glusterfs_graph_settop(graph, volume_name, _gf_false);
+    } else if (volume_name[0] == '/') {
+        /* brick multiplexing passes the brick path */
+        ret = glusterfs_graph_settop(graph, volume_name, _gf_true);
+    } else {
+        ret = glusterfs_graph_settop(graph, volume_name, _gf_false);
+    }
+
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph settop failed");
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* XXX: WORM VOLUME */
+    ret = glusterfs_graph_worm(graph, ctx);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph worm failed");
+        return -1;
+    }
+    ret = glusterfs_graph_acl(graph, ctx);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph ACL failed");
         return -1;
+    }
+
+    /* XXX: MAC COMPAT */
+    ret = glusterfs_graph_mac_compat(graph, ctx);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph mac compat failed");
+        return -1;
+    }
+
+    /* XXX: gfid-access */
+    ret = glusterfs_graph_gfid_access(graph, ctx);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph 'gfid-access' failed");
+        return -1;
+    }
+
+    /* XXX: topmost xlator */
+    ret = glusterfs_graph_meta(graph, ctx);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_ERROR,
+               "glusterfs graph meta failed");
+        return -1;
+    }
+
+    /* XXX: this->ctx setting */
+    for (trav = graph->first; trav; trav = trav->next) {
+        trav->ctx = ctx;
+    }
+
+    /* XXX: DOB setting */
+    gettimeofday(&graph->dob, NULL);
+
+    fill_uuid(graph->graph_uuid, sizeof(graph->graph_uuid), graph->dob);
+
+    graph->id = ctx->graph_id++;
+
+    /* XXX: --xlator-option additions */
+    gf_add_cmdline_options(graph, &ctx->cmd_args);
+
+    return 0;
+}
+
+static xlator_t *
+glusterfs_root(glusterfs_graph_t *graph)
+{
+    return graph->first;
+}
+
+static int
+glusterfs_is_leaf(xlator_t *xl)
+{
+    int ret = 0;
+
+    if (!xl->children)
+        ret = 1;
+
+    return ret;
 }
 
+static uint32_t
+glusterfs_count_leaves(xlator_t *xl)
+{
+    int n = 0;
+    xlator_list_t *list = NULL;
+
+    if (glusterfs_is_leaf(xl))
+        n = 1;
+    else
+        for (list = xl->children; list; list = list->next)
+            n += glusterfs_count_leaves(list->xlator);
+
+    return n;
+}
 
 int
-glusterfs_graph_parent_up (glusterfs_graph_t *graph)
+glusterfs_get_leaf_count(glusterfs_graph_t *graph)
 {
-        xlator_t *trav = NULL;
-        int       ret = -1;
+    return graph->leaf_count;
+}
+
+static int
+_glusterfs_leaf_position(xlator_t *tgt, int *id, xlator_t *xl)
+{
+    xlator_list_t *list = NULL;
+    int found = 0;
+
+    if (xl == tgt)
+        found = 1;
+    else if (glusterfs_is_leaf(xl))
+        *id += 1;
+    else
+        for (list = xl->children; !found && list; list = list->next)
+            found = _glusterfs_leaf_position(tgt, id, list->xlator);
+
+    return found;
+}
 
-        trav = graph->first;
+int
+glusterfs_leaf_position(xlator_t *tgt)
+{
+    xlator_t *root = NULL;
+    int pos = 0;
 
-        while (trav) {
-                if (!xlator_has_parent (trav)) {
-                        ret = xlator_notify (trav, GF_EVENT_PARENT_UP, trav);
-                }
+    root = glusterfs_root(tgt->graph);
 
-                if (ret)
-                        break;
+    if (!_glusterfs_leaf_position(tgt, &pos, root))
+        pos = -1;
 
-                trav = trav->next;
+    return pos;
+}
+
+static int
+_glusterfs_reachable_leaves(xlator_t *base, xlator_t *xl, dict_t *leaves)
+{
+    xlator_list_t *list = NULL;
+    int err = 1;
+    int pos = 0;
+    char *strpos = NULL;
+
+    if (glusterfs_is_leaf(xl)) {
+        pos = glusterfs_leaf_position(xl);
+        if (pos < 0)
+            goto out;
+
+        err = gf_asprintf(&strpos, "%d", pos);
+
+        if (err >= 0) {
+            err = dict_set_static_ptr(leaves, strpos, base);
+            GF_FREE(strpos);
         }
+    } else {
+        for (err = 0, list = xl->children; !err && list; list = list->next)
+            err = _glusterfs_reachable_leaves(base, list->xlator, leaves);
+    }
 
-        return ret;
+out:
+    return err;
 }
 
+/*
+ * This function determines which leaves are children (or grandchildren)
+ * of the given base. The base may have multiple sub volumes. Each sub
+ * volumes in turn may have sub volumes.. until the leaves are reached.
+ * Each leaf is numbered 1,2,3,...etc.
+ *
+ * The base translator calls this function to see which of *its* subvolumes
+ * it would forward an FOP to, to *get to* a particular leaf.
+ * That information is built into the "leaves" dictionary.
+ * key:destination leaf# -> value:base subvolume xlator.
+ */
 
 int
-glusterfs_graph_prepare (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+glusterfs_reachable_leaves(xlator_t *base, dict_t *leaves)
 {
-        xlator_t    *trav = NULL;
-        int          ret = 0;
+    xlator_list_t *list = NULL;
+    int err = 0;
 
-        /* XXX: CHECKSUM */
+    for (list = base->children; !err && list; list = list->next)
+        err = _glusterfs_reachable_leaves(list->xlator, list->xlator, leaves);
 
-        /* XXX: attach to -n volname */
-        ret = glusterfs_graph_settop (graph, ctx);
-        if (ret)
-                return -1;
+    return err;
+}
 
-        /* XXX: RO VOLUME */
-        ret = glusterfs_graph_readonly (graph, ctx);
-        if (ret)
-                return -1;
+int
+glusterfs_graph_activate(glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+{
+    int ret = 0;
+    xlator_t *root = NULL;
 
-        /* XXX: MAC COMPAT */
-        ret = glusterfs_graph_mac_compat (graph, ctx);
-        if (ret)
-                return -1;
+    root = glusterfs_root(graph);
 
-        /* XXX: this->ctx setting */
-        for (trav = graph->first; trav; trav = trav->next) {
-                trav->ctx = ctx;
+    graph->leaf_count = glusterfs_count_leaves(root);
+
+    /* XXX: all xlator options validation */
+    ret = glusterfs_graph_validate_options(graph);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_VALIDATION_FAILED,
+               "validate options failed");
+        return ret;
+    }
+
+    /* XXX: perform init () */
+    ret = glusterfs_graph_init(graph);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_GRAPH_INIT_FAILED,
+               "init failed");
+        return ret;
+    }
+
+    ret = glusterfs_graph_unknown_options(graph);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_UNKNOWN_OPTIONS_FAILED,
+               "unknown options "
+               "failed");
+        return ret;
+    }
+
+    /* XXX: log full graph (_gf_dump_details) */
+
+    list_add(&graph->list, &ctx->graphs);
+    ctx->active = graph;
+
+    /* XXX: attach to master and set active pointer */
+    if (ctx->master) {
+        ret = xlator_notify(ctx->master, GF_EVENT_GRAPH_NEW, graph);
+        if (ret) {
+            gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED,
+                   "graph new notification failed");
+            return ret;
         }
+        ((xlator_t *)ctx->master)->next = graph->top;
+    }
+
+    /* XXX: perform parent up */
+    ret = glusterfs_graph_parent_up(graph);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED,
+               "parent up notification failed");
+        return ret;
+    }
 
-        /* XXX: DOB setting */
-        gettimeofday (&graph->dob, NULL);
+    return 0;
+}
 
-        fill_uuid (graph->graph_uuid, 128);
+int
+xlator_equal_rec(xlator_t *xl1, xlator_t *xl2)
+{
+    xlator_list_t *trav1 = NULL;
+    xlator_list_t *trav2 = NULL;
+    int ret = 0;
 
-        /* XXX: --xlator-option additions */
-        gf_add_cmdline_options (graph, &ctx->cmd_args);
+    if (xl1 == NULL || xl2 == NULL) {
+        gf_msg_debug("xlator", 0, "invalid argument");
+        return -1;
+    }
+
+    trav1 = xl1->children;
+    trav2 = xl2->children;
+
+    while (trav1 && trav2) {
+        ret = xlator_equal_rec(trav1->xlator, trav2->xlator);
+        if (ret) {
+            gf_msg_debug("glusterfsd-mgmt", 0,
+                         "xlators children "
+                         "not equal");
+            goto out;
+        }
 
+        trav1 = trav1->next;
+        trav2 = trav2->next;
+    }
+
+    if (trav1 || trav2) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(xl1->name, xl2->name)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* type could have changed even if xlator names match,
+       e.g cluster/distribute and cluster/nufa share the same
+       xlator name
+    */
+    if (strcmp(xl1->type, xl2->type)) {
+        ret = -1;
+        goto out;
+    }
+out:
+    return ret;
+}
 
-        return 0;
+gf_boolean_t
+is_graph_topology_equal(glusterfs_graph_t *graph1, glusterfs_graph_t *graph2)
+{
+    xlator_t *trav1 = NULL;
+    xlator_t *trav2 = NULL;
+    gf_boolean_t ret = _gf_true;
+    xlator_list_t *ltrav;
+
+    trav1 = graph1->first;
+    trav2 = graph2->first;
+
+    if (strcmp(trav2->type, "protocol/server") == 0) {
+        trav2 = trav2->children->xlator;
+        for (ltrav = trav1->children; ltrav; ltrav = ltrav->next) {
+            trav1 = ltrav->xlator;
+            if (!trav1->cleanup_starting && !strcmp(trav1->name, trav2->name)) {
+                break;
+            }
+        }
+        if (!ltrav) {
+            return _gf_false;
+        }
+    }
+
+    ret = xlator_equal_rec(trav1, trav2);
+
+    if (ret) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "graphs are not equal");
+        ret = _gf_false;
+        goto out;
+    }
+
+    ret = _gf_true;
+    gf_msg_debug("glusterfsd-mgmt", 0, "graphs are equal");
+
+out:
+    return ret;
+}
+
+/* Function has 3types of return value 0, -ve , 1
+ *   return 0          =======> reconfiguration of options has succeeded
+ *   return 1          =======> the graph has to be reconstructed and all the
+ * xlators should be inited return -1(or -ve) =======> Some Internal Error
+ * occurred during the operation
+ */
+int
+glusterfs_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx)
+{
+    glusterfs_graph_t *oldvolfile_graph = NULL;
+    glusterfs_graph_t *newvolfile_graph = NULL;
+
+    int ret = -1;
+
+    if (!ctx) {
+        gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL,
+               "ctx is NULL");
+        goto out;
+    }
+
+    oldvolfile_graph = ctx->active;
+    if (!oldvolfile_graph) {
+        ret = 1;
+        goto out;
+    }
+
+    newvolfile_graph = glusterfs_graph_construct(newvolfile_fp);
+
+    if (!newvolfile_graph) {
+        goto out;
+    }
+
+    glusterfs_graph_prepare(newvolfile_graph, ctx, ctx->cmd_args.volume_name);
+
+    if (!is_graph_topology_equal(oldvolfile_graph, newvolfile_graph)) {
+        ret = 1;
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "Graph topology not "
+                     "equal(should call INIT)");
+        goto out;
+    }
+
+    gf_msg_debug("glusterfsd-mgmt", 0,
+                 "Only options have changed in the"
+                 " new graph");
+
+    ret = glusterfs_graph_reconfigure(oldvolfile_graph, newvolfile_graph);
+    if (ret) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "Could not reconfigure "
+                     "new options in old graph");
+        goto out;
+    }
+
+    ret = 0;
+out:
+
+    if (newvolfile_graph)
+        glusterfs_graph_destroy(newvolfile_graph);
+
+    return ret;
 }
 
+/* This function need to remove. This added to support gfapi volfile
+ * reconfigure.
+ */
 
 int
-glusterfs_graph_activate (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+gf_volfile_reconfigure(int oldvollen, FILE *newvolfile_fp, glusterfs_ctx_t *ctx,
+                       const char *oldvolfile)
 {
-        int ret = 0;
+    glusterfs_graph_t *oldvolfile_graph = NULL;
+    glusterfs_graph_t *newvolfile_graph = NULL;
+    FILE *oldvolfile_fp = NULL;
+    /*Since the function mkstemp() replaces XXXXXX,
+     * assigning it to a variable
+     */
+    char temp_file[] = "/tmp/temp_vol_file_XXXXXX";
+    gf_boolean_t active_graph_found = _gf_true;
+
+    int ret = -1;
+    int u_ret = -1;
+    int file_desc = -1;
+
+    if (!oldvollen) {
+        ret = 1;  // Has to call INIT for the whole graph
+        goto out;
+    }
+
+    if (!ctx) {
+        gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL,
+               "ctx is NULL");
+        goto out;
+    }
+
+    oldvolfile_graph = ctx->active;
+    if (!oldvolfile_graph) {
+        active_graph_found = _gf_false;
+        gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_ACTIVE_GRAPH_NULL,
+               "glusterfs_ctx->active is NULL");
+
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+        file_desc = mkstemp(temp_file);
+        if (file_desc < 0) {
+            gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, errno,
+                   LG_MSG_TMPFILE_CREATE_FAILED,
+                   "Unable to "
+                   "create temporary volfile");
+            goto out;
+        }
 
-        /* XXX: all xlator options validation */
-        ret = glusterfs_graph_validate_options (graph);
-        if (ret)
-                return ret;
+        /*Calling unlink so that when the file is closed or program
+         *terminates the tempfile is deleted.
+         */
+        u_ret = sys_unlink(temp_file);
+
+        if (u_ret < 0) {
+            gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, errno,
+                   LG_MSG_TMPFILE_DELETE_FAILED,
+                   "Temporary file"
+                   " delete failed.");
+            sys_close(file_desc);
+            goto out;
+        }
 
-        /* XXX: perform init () */
-        ret = glusterfs_graph_init (graph);
-        if (ret)
-                return ret;
+        oldvolfile_fp = fdopen(file_desc, "w+b");
+        if (!oldvolfile_fp)
+            goto out;
 
-        ret = glusterfs_graph_unknown_options (graph);
-        if (ret)
-                return ret;
+        fwrite(oldvolfile, oldvollen, 1, oldvolfile_fp);
+        fflush(oldvolfile_fp);
+        if (ferror(oldvolfile_fp)) {
+            goto out;
+        }
 
-        /* XXX: log full graph (_gf_dump_details) */
+        oldvolfile_graph = glusterfs_graph_construct(oldvolfile_fp);
+        if (!oldvolfile_graph)
+            goto out;
+    }
+
+    newvolfile_graph = glusterfs_graph_construct(newvolfile_fp);
+    if (!newvolfile_graph) {
+        goto out;
+    }
+
+    glusterfs_graph_prepare(newvolfile_graph, ctx, ctx->cmd_args.volume_name);
+
+    if (!is_graph_topology_equal(oldvolfile_graph, newvolfile_graph)) {
+        ret = 1;
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "Graph topology not "
+                     "equal(should call INIT)");
+        goto out;
+    }
+
+    gf_msg_debug("glusterfsd-mgmt", 0,
+                 "Only options have changed in the"
+                 " new graph");
+
+    /* */
+    ret = glusterfs_graph_reconfigure(oldvolfile_graph, newvolfile_graph);
+    if (ret) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "Could not reconfigure "
+                     "new options in old graph");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (oldvolfile_fp)
+        fclose(oldvolfile_fp);
+
+    /*  Do not simply destroy the old graph here. If the oldgraph
+        is constructed here in this function itself instead of getting
+        it from ctx->active (which happens only of ctx->active is NULL),
+        then destroy the old graph. If some i/o is still happening in
+        the old graph and the old graph is obtained from ctx->active,
+        then destroying the graph will cause problems.
+    */
+    if (!active_graph_found && oldvolfile_graph)
+        glusterfs_graph_destroy(oldvolfile_graph);
+    if (newvolfile_graph)
+        glusterfs_graph_destroy(newvolfile_graph);
+
+    return ret;
+}
 
-        list_add (&graph->list, &ctx->graphs);
-        ctx->active = graph;
+int
+glusterfs_graph_reconfigure(glusterfs_graph_t *oldgraph,
+                            glusterfs_graph_t *newgraph)
+{
+    xlator_t *old_xl = NULL;
+    xlator_t *new_xl = NULL;
+    xlator_list_t *trav;
+
+    GF_ASSERT(oldgraph);
+    GF_ASSERT(newgraph);
+
+    old_xl = oldgraph->first;
+    while (old_xl->is_autoloaded) {
+        old_xl = old_xl->children->xlator;
+    }
+
+    new_xl = newgraph->first;
+    while (new_xl->is_autoloaded) {
+        new_xl = new_xl->children->xlator;
+    }
+
+    if (strcmp(old_xl->type, "protocol/server") != 0) {
+        return xlator_tree_reconfigure(old_xl, new_xl);
+    }
+
+    /* Some options still need to be handled by the server translator. */
+    if (old_xl->reconfigure) {
+        old_xl->reconfigure(old_xl, new_xl->options);
+    }
+
+    (void)copy_opts_to_child(new_xl, FIRST_CHILD(new_xl), "*auth*");
+    new_xl = FIRST_CHILD(new_xl);
+
+    for (trav = old_xl->children; trav; trav = trav->next) {
+        if (!trav->xlator->cleanup_starting &&
+            !strcmp(trav->xlator->name, new_xl->name)) {
+            return xlator_tree_reconfigure(trav->xlator, new_xl);
+        }
+    }
 
-        /* XXX: attach to master and set active pointer */
-        if (ctx->master)
-                ret = xlator_notify (ctx->master, GF_EVENT_GRAPH_NEW, graph);
-        if (ret)
-                return ret;
+    return -1;
+}
 
-        /* XXX: perform parent up */
-        ret = glusterfs_graph_parent_up (graph);
-        if (ret)
-                return ret;
+int
+glusterfs_graph_destroy_residual(glusterfs_graph_t *graph)
+{
+    int ret = -1;
 
+    if (graph == NULL)
+        return ret;
 
-        return 0;
+    ret = xlator_tree_free_memacct(graph->first);
+
+    list_del_init(&graph->list);
+    pthread_mutex_destroy(&graph->mutex);
+    pthread_cond_destroy(&graph->child_down_cond);
+    GF_FREE(graph);
+
+    return ret;
 }
 
+/* This function destroys all the xlator members except for the
+ * xlator strcuture and its mem accounting field.
+ *
+ * If otherwise, it would destroy the master xlator object as well
+ * its mem accounting, which would mean after calling glusterfs_graph_destroy()
+ * there cannot be any reference to GF_FREE() from the master xlator, this is
+ * not possible because of the following dependencies:
+ * - glusterfs_ctx_t will have mem pools allocated by the master xlators
+ * - xlator objects will have references to those mem pools(g: dict)
+ *
+ * Ordering the freeing in any of the order will also not solve the dependency:
+ * - Freeing xlator objects(including memory accounting) before mem pools
+ *   destruction will mean not use GF_FREE while destroying mem pools.
+ * - Freeing mem pools and then destroying xlator objects would lead to crashes
+ *   when xlator tries to unref dict or other mem pool objects.
+ *
+ * Hence the way chosen out of this interdependency is to split xlator object
+ * free into two stages:
+ * - Free all the xlator members excpet for its mem accounting structure
+ * - Free all the mem accouting structures of xlator along with the xlator
+ *   object itself.
+ */
 int
-glusterfs_graph_reconfigure (glusterfs_graph_t *oldgraph,
-                             glusterfs_graph_t *newgraph)
+glusterfs_graph_destroy(glusterfs_graph_t *graph)
 {
-        xlator_t *old_xl = NULL;
-        xlator_t *new_xl = NULL;
+    int ret = 0;
 
-        GF_ASSERT (oldgraph);
-        GF_ASSERT (newgraph);
+    GF_VALIDATE_OR_GOTO("graph", graph, out);
 
-        old_xl   = oldgraph->first;
-        new_xl   = newgraph->first;
+    ret = xlator_tree_free_members(graph->first);
+
+    ret = glusterfs_graph_destroy_residual(graph);
+out:
+    return ret;
+}
+
+int
+glusterfs_graph_fini(glusterfs_graph_t *graph)
+{
+    xlator_t *trav = NULL;
+
+    trav = graph->first;
+
+    while (trav) {
+        if (trav->init_succeeded) {
+            trav->cleanup_starting = 1;
+            trav->fini(trav);
+            if (trav->local_pool) {
+                mem_pool_destroy(trav->local_pool);
+                trav->local_pool = NULL;
+            }
+            if (trav->itable) {
+                inode_table_destroy(trav->itable);
+                trav->itable = NULL;
+            }
+            trav->init_succeeded = 0;
+        }
+        trav = trav->next;
+    }
 
-        return xlator_tree_reconfigure (old_xl, new_xl);
+    return 0;
 }
 
 int
-glusterfs_graph_destroy (glusterfs_graph_t *graph)
+glusterfs_graph_attach(glusterfs_graph_t *orig_graph, char *path,
+                       glusterfs_graph_t **newgraph)
+{
+    xlator_t *this = THIS;
+    FILE *fp;
+    glusterfs_graph_t *graph;
+    xlator_t *xl;
+    char *volfile_id = NULL;
+    char *volfile_content = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    size_t file_len = -1;
+    gf_volfile_t *volfile_obj = NULL;
+    int ret = -1;
+    char sha256_hash[SHA256_DIGEST_LENGTH] = {
+        0,
+    };
+
+    if (!orig_graph) {
+        return -EINVAL;
+    }
+
+    ret = sys_stat(path, &stbuf);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Unable to stat %s (%s)", path,
+               strerror(errno));
+        return -EINVAL;
+    }
+
+    file_len = stbuf.st_size;
+    volfile_content = GF_MALLOC(file_len + 1, gf_common_mt_char);
+    if (!volfile_content)
+        return -ENOMEM;
+
+    fp = fopen(path, "r");
+    if (!fp) {
+        gf_log(THIS->name, GF_LOG_WARNING, "oops, %s disappeared on us", path);
+        GF_FREE(volfile_content);
+        return -EIO;
+    }
+
+    ret = fread(volfile_content, sizeof(char), file_len, fp);
+    if (ret == file_len) {
+        glusterfs_compute_sha256((const unsigned char *)volfile_content,
+                                 file_len, sha256_hash);
+    } else {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "read failed on path %s. File size=%" GF_PRI_SIZET
+               "read size=%d",
+               path, file_len, ret);
+        GF_FREE(volfile_content);
+        fclose(fp);
+        return -EIO;
+    }
+
+    GF_FREE(volfile_content);
+
+    graph = glusterfs_graph_construct(fp);
+    fclose(fp);
+    if (!graph) {
+        gf_log(this->name, GF_LOG_WARNING, "could not create graph from %s",
+               path);
+        return -EIO;
+    }
+
+    /*
+     * If there's a server translator on top, we want whatever's below
+     * that.
+     */
+    xl = graph->first;
+    if (strcmp(xl->type, "protocol/server") == 0) {
+        (void)copy_opts_to_child(xl, FIRST_CHILD(xl), "*auth*");
+        xl = FIRST_CHILD(xl);
+    }
+    graph->first = xl;
+    *newgraph = graph;
+
+    volfile_id = strstr(path, "/snaps/");
+    if (!volfile_id) {
+        volfile_id = rindex(path, '/');
+        if (volfile_id) {
+            ++volfile_id;
+        }
+    }
+    if (volfile_id) {
+        xl->volfile_id = gf_strdup(volfile_id);
+        /* There's a stray ".vol" at the end. */
+        xl->volfile_id[strlen(xl->volfile_id) - 4] = '\0';
+    }
+
+    /* TODO memory leaks everywhere need to free graph in case of error */
+    if (glusterfs_graph_prepare(graph, this->ctx, xl->name)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "failed to prepare graph for xlator %s", xl->name);
+        return -EIO;
+    } else if (glusterfs_graph_init(graph)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "failed to initialize graph for xlator %s", xl->name);
+        return -EIO;
+    } else if (glusterfs_xlator_link(orig_graph->top, graph->top)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "failed to link the graphs for xlator %s ", xl->name);
+        return -EIO;
+    }
+
+    if (!volfile_obj) {
+        volfile_obj = GF_CALLOC(1, sizeof(gf_volfile_t), gf_common_volfile_t);
+        if (!volfile_obj) {
+            return -EIO;
+        }
+    }
+
+    INIT_LIST_HEAD(&volfile_obj->volfile_list);
+    snprintf(volfile_obj->vol_id, sizeof(volfile_obj->vol_id), "%s",
+             xl->volfile_id);
+    memcpy(volfile_obj->volfile_checksum, sha256_hash,
+           sizeof(volfile_obj->volfile_checksum));
+    list_add(&volfile_obj->volfile_list, &this->ctx->volfile_list);
+
+    return 0;
+}
+int
+glusterfs_muxsvc_cleanup_parent(glusterfs_ctx_t *ctx,
+                                glusterfs_graph_t *parent_graph)
 {
+    if (parent_graph) {
+        if (parent_graph->first) {
+            xlator_destroy(parent_graph->first);
+        }
+        ctx->active = NULL;
+        GF_FREE(parent_graph);
+        parent_graph = NULL;
+    }
+    return 0;
+}
+
+void *
+glusterfs_graph_cleanup(void *arg)
+{
+    glusterfs_graph_t *graph = NULL;
+    glusterfs_ctx_t *ctx = THIS->ctx;
+    int ret = -1;
+    graph = arg;
+
+    if (!graph)
+        return NULL;
+
+    /* To destroy the graph, fitst sent a GF_EVENT_PARENT_DOWN
+     * Then wait for GF_EVENT_CHILD_DOWN to get on the top
+     * xl. Once we have GF_EVENT_CHILD_DOWN event, then proceed
+     * to fini.
+     *
+     * During fini call, this will take a last unref on rpc and
+     * rpc_transport_object.
+     */
+    if (graph->first)
+        default_notify(graph->first, GF_EVENT_PARENT_DOWN, graph->first);
+
+    ret = pthread_mutex_lock(&graph->mutex);
+    if (ret != 0) {
+        gf_msg("glusterfs", GF_LOG_ERROR, EAGAIN, LG_MSG_GRAPH_CLEANUP_FAILED,
+               "Failed to acquire a lock");
+        goto out;
+    }
+    /* check and wait for CHILD_DOWN for top xlator*/
+    while (graph->used) {
+        ret = pthread_cond_wait(&graph->child_down_cond, &graph->mutex);
+        if (ret != 0)
+            gf_msg("glusterfs", GF_LOG_INFO, 0, LG_MSG_GRAPH_CLEANUP_FAILED,
+                   "cond wait failed ");
+    }
+
+    ret = pthread_mutex_unlock(&graph->mutex);
+    if (ret != 0) {
+        gf_msg("glusterfs", GF_LOG_ERROR, EAGAIN, LG_MSG_GRAPH_CLEANUP_FAILED,
+               "Failed to release a lock");
+    }
+
+    /* Though we got a child down on top xlator, we have to wait until
+     * all the notifier to exit. Because there should not be any threads
+     * that access xl variables.
+     */
+    pthread_mutex_lock(&ctx->notify_lock);
+    {
+        while (ctx->notifying)
+            pthread_cond_wait(&ctx->notify_cond, &ctx->notify_lock);
+    }
+    pthread_mutex_unlock(&ctx->notify_lock);
+
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    {
+        glusterfs_graph_fini(graph);
+        glusterfs_graph_destroy(graph);
+    }
+    pthread_mutex_unlock(&ctx->cleanup_lock);
+out:
+    return NULL;
+}
+
+glusterfs_graph_t *
+glusterfs_muxsvc_setup_parent_graph(glusterfs_ctx_t *ctx, char *name,
+                                    char *type)
+{
+    glusterfs_graph_t *parent_graph = NULL;
+    xlator_t *ixl = NULL;
+    int ret = -1;
+    parent_graph = GF_CALLOC(1, sizeof(*parent_graph),
+                             gf_common_mt_glusterfs_graph_t);
+    if (!parent_graph)
+        goto out;
+
+    INIT_LIST_HEAD(&parent_graph->list);
+
+    ctx->active = parent_graph;
+    ixl = GF_CALLOC(1, sizeof(*ixl), gf_common_mt_xlator_t);
+    if (!ixl)
+        goto out;
+
+    ixl->ctx = ctx;
+    ixl->graph = parent_graph;
+    ixl->options = dict_new();
+    if (!ixl->options)
+        goto out;
+
+    ixl->name = gf_strdup(name);
+    if (!ixl->name)
+        goto out;
+
+    ixl->is_autoloaded = 1;
+
+    if (xlator_set_type(ixl, type) == -1) {
+        gf_msg("glusterfs", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED,
+               "%s (%s) set type failed", name, type);
+        goto out;
+    }
+
+    glusterfs_graph_set_first(parent_graph, ixl);
+    parent_graph->top = ixl;
+    ixl = NULL;
+
+    gettimeofday(&parent_graph->dob, NULL);
+    fill_uuid(parent_graph->graph_uuid, 128, parent_graph->dob);
+    parent_graph->id = ctx->graph_id++;
+    ret = 0;
+out:
+    if (ixl)
+        xlator_destroy(ixl);
+
+    if (ret) {
+        glusterfs_muxsvc_cleanup_parent(ctx, parent_graph);
+        parent_graph = NULL;
+    }
+    return parent_graph;
+}
+
+int
+glusterfs_svc_mux_pidfile_cleanup(gf_volfile_t *volfile_obj)
+{
+    if (!volfile_obj || !volfile_obj->pidfp)
         return 0;
+
+    gf_msg_trace("glusterfsd", 0, "pidfile %s cleanup", volfile_obj->vol_id);
+
+    lockf(fileno(volfile_obj->pidfp), F_ULOCK, 0);
+    fclose(volfile_obj->pidfp);
+    volfile_obj->pidfp = NULL;
+
+    return 0;
 }
 
+int
+glusterfs_process_svc_detach(glusterfs_ctx_t *ctx, gf_volfile_t *volfile_obj)
+{
+    xlator_t *last_xl = NULL;
+    glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *parent_graph = NULL;
+    pthread_t clean_graph = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *xl = NULL;
+
+    if (!ctx || !ctx->active || !volfile_obj)
+        goto out;
+
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    {
+        parent_graph = ctx->active;
+        graph = volfile_obj->graph;
+        if (!graph)
+            goto unlock;
+        if (graph->first)
+            xl = graph->first;
+
+        last_xl = graph->last_xl;
+        if (last_xl)
+            last_xl->next = NULL;
+        if (!xl || xl->cleanup_starting)
+            goto unlock;
+
+        xl->cleanup_starting = 1;
+        gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_DETACH_STARTED,
+               "detaching child %s", volfile_obj->vol_id);
+
+        list_del_init(&volfile_obj->volfile_list);
+        glusterfs_mux_xlator_unlink(parent_graph->top, xl);
+        glusterfs_svc_mux_pidfile_cleanup(volfile_obj);
+        parent_graph->last_xl = glusterfs_get_last_xlator(parent_graph);
+        parent_graph->xl_count -= graph->xl_count;
+        parent_graph->leaf_count -= graph->leaf_count;
+        parent_graph->id++;
+        ret = 0;
+    }
+unlock:
+    pthread_mutex_unlock(&ctx->cleanup_lock);
+out:
+    if (!ret) {
+        list_del_init(&volfile_obj->volfile_list);
+        if (graph) {
+            ret = gf_thread_create_detached(
+                &clean_graph, glusterfs_graph_cleanup, graph, "graph_clean");
+            if (ret) {
+                gf_msg("glusterfs", GF_LOG_ERROR, EINVAL,
+                       LG_MSG_GRAPH_CLEANUP_FAILED,
+                       "%s failed to create clean "
+                       "up thread",
+                       volfile_obj->vol_id);
+                ret = 0;
+            }
+        }
+        GF_FREE(volfile_obj);
+    }
+    return ret;
+}
+
+int
+glusterfs_svc_mux_pidfile_setup(gf_volfile_t *volfile_obj, const char *pid_file)
+{
+    int ret = -1;
+    FILE *pidfp = NULL;
+
+    if (!pid_file || !volfile_obj)
+        goto out;
+
+    if (volfile_obj->pidfp) {
+        ret = 0;
+        goto out;
+    }
+    pidfp = fopen(pid_file, "a+");
+    if (!pidfp) {
+        goto out;
+    }
+    volfile_obj->pidfp = pidfp;
+
+    ret = lockf(fileno(pidfp), F_TLOCK, 0);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+out:
+    return ret;
+}
 
+int
+glusterfs_svc_mux_pidfile_update(gf_volfile_t *volfile_obj,
+                                 const char *pid_file, pid_t pid)
+{
+    int ret = 0;
+    FILE *pidfp = NULL;
+    int old_pid;
+
+    if (!volfile_obj->pidfp) {
+        ret = glusterfs_svc_mux_pidfile_setup(volfile_obj, pid_file);
+        if (ret == -1)
+            goto out;
+    }
+    pidfp = volfile_obj->pidfp;
+    ret = fscanf(pidfp, "%d", &old_pid);
+    if (ret <= 0) {
+        goto update;
+    }
+    if (old_pid == pid) {
+        ret = 0;
+        goto out;
+    } else {
+        gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED,
+               "Old pid=%d found in pidfile %s. Cleaning the old pid and "
+               "Updating new pid=%d",
+               old_pid, pid_file, pid);
+    }
+update:
+    ret = sys_ftruncate(fileno(pidfp), 0);
+    if (ret) {
+        gf_msg("glusterfsd", GF_LOG_ERROR, errno,
+               LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED,
+               "pidfile %s truncation failed", pid_file);
+        goto out;
+    }
+
+    ret = fprintf(pidfp, "%d\n", pid);
+    if (ret <= 0) {
+        gf_msg("glusterfsd", GF_LOG_ERROR, errno,
+               LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, "pidfile %s write failed",
+               pid_file);
+        goto out;
+    }
+
+    ret = fflush(pidfp);
+    if (ret) {
+        gf_msg("glusterfsd", GF_LOG_ERROR, errno,
+               LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED, "pidfile %s write failed",
+               pid_file);
+        goto out;
+    }
+out:
+    return ret;
+}
+
+int
+glusterfs_update_mux_pid(dict_t *dict, gf_volfile_t *volfile_obj)
+{
+    char *file = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("graph", dict, out);
+    GF_VALIDATE_OR_GOTO("graph", volfile_obj, out);
+
+    ret = dict_get_str(dict, "pidfile", &file);
+    if (ret < 0) {
+        gf_msg("mgmt", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED,
+               "Failed to get pidfile from dict for  volfile_id=%s",
+               volfile_obj->vol_id);
+    }
+
+    ret = glusterfs_svc_mux_pidfile_update(volfile_obj, file, getpid());
+    if (ret < 0) {
+        ret = -1;
+        gf_msg("mgmt", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_SETUP_FAILED,
+               "Failed to update "
+               "the pidfile for volfile_id=%s",
+               volfile_obj->vol_id);
+
+        goto out;
+    }
+
+    if (ret == 1)
+        gf_msg("mgmt", GF_LOG_INFO, 0, LG_MSG_GRAPH_ATTACH_PID_FILE_UPDATED,
+               "PID %d updated in pidfile=%s", getpid(), file);
+    ret = 0;
+out:
+    return ret;
+}
+int
+glusterfs_process_svc_attach_volfp(glusterfs_ctx_t *ctx, FILE *fp,
+                                   char *volfile_id, char *checksum,
+                                   dict_t *dict)
+{
+    glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *parent_graph = NULL;
+    glusterfs_graph_t *clean_graph = NULL;
+    int ret = -1;
+    xlator_t *xl = NULL;
+    xlator_t *last_xl = NULL;
+    gf_volfile_t *volfile_obj = NULL;
+    pthread_t thread_id = {
+        0,
+    };
+
+    if (!ctx)
+        goto out;
+    parent_graph = ctx->active;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_msg("glusterfsd", GF_LOG_ERROR, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED,
+               "failed to construct the graph");
+        goto out;
+    }
+    graph->parent_down = 0;
+    graph->last_xl = glusterfs_get_last_xlator(graph);
+
+    for (xl = graph->first; xl; xl = xl->next) {
+        if (strcmp(xl->type, "mount/fuse") == 0) {
+            gf_msg("glusterfsd", GF_LOG_ERROR, EINVAL,
+                   LG_MSG_GRAPH_ATTACH_FAILED,
+                   "fuse xlator cannot be specified in volume file");
+            goto out;
+        }
+    }
+
+    graph->leaf_count = glusterfs_count_leaves(glusterfs_root(graph));
+    xl = graph->first;
+    /* TODO memory leaks everywhere need to free graph in case of error */
+    if (glusterfs_graph_prepare(graph, ctx, xl->name)) {
+        gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED,
+               "failed to prepare graph for xlator %s", xl->name);
+        ret = -1;
+        goto out;
+    } else if (glusterfs_graph_init(graph)) {
+        gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED,
+               "failed to initialize graph for xlator %s", xl->name);
+        ret = -1;
+        goto out;
+    } else if (glusterfs_graph_parent_up(graph)) {
+        gf_msg("glusterfsd", GF_LOG_WARNING, EINVAL, LG_MSG_GRAPH_ATTACH_FAILED,
+               "failed to link the graphs for xlator %s ", xl->name);
+        ret = -1;
+        goto out;
+    }
+
+    if (!parent_graph) {
+        parent_graph = glusterfs_muxsvc_setup_parent_graph(ctx, "glustershd",
+                                                           "debug/io-stats");
+        if (!parent_graph)
+            goto out;
+        ((xlator_t *)parent_graph->top)->next = xl;
+        clean_graph = parent_graph;
+    } else {
+        last_xl = parent_graph->last_xl;
+        if (last_xl)
+            last_xl->next = xl;
+        xl->prev = last_xl;
+    }
+    parent_graph->last_xl = graph->last_xl;
+
+    ret = glusterfs_xlator_link(parent_graph->top, xl);
+    if (ret) {
+        gf_msg("graph", GF_LOG_ERROR, 0, LG_MSG_EVENT_NOTIFY_FAILED,
+               "parent up notification failed");
+        goto out;
+    }
+    parent_graph->xl_count += graph->xl_count;
+    parent_graph->leaf_count += graph->leaf_count;
+    parent_graph->id++;
+
+    volfile_obj = GF_CALLOC(1, sizeof(gf_volfile_t), gf_common_volfile_t);
+    if (!volfile_obj) {
+        ret = -1;
+        goto out;
+    }
+    volfile_obj->pidfp = NULL;
+    snprintf(volfile_obj->vol_id, sizeof(volfile_obj->vol_id), "%s",
+             volfile_id);
+
+    if (strcmp(ctx->cmd_args.process_name, "glustershd") == 0) {
+        ret = glusterfs_update_mux_pid(dict, volfile_obj);
+        if (ret == -1) {
+            GF_FREE(volfile_obj);
+            goto out;
+        }
+    }
+
+    graph->used = 1;
+    parent_graph->id++;
+    list_add(&graph->list, &ctx->graphs);
+    INIT_LIST_HEAD(&volfile_obj->volfile_list);
+    volfile_obj->graph = graph;
+    memcpy(volfile_obj->volfile_checksum, checksum,
+           sizeof(volfile_obj->volfile_checksum));
+    list_add_tail(&volfile_obj->volfile_list, &ctx->volfile_list);
+    gf_log_dump_graph(fp, graph);
+    graph = NULL;
+
+    ret = 0;
+out:
+    if (ret) {
+        if (graph) {
+            gluster_graph_take_reference(graph->first);
+            ret = gf_thread_create_detached(&thread_id, glusterfs_graph_cleanup,
+                                            graph, "graph_clean");
+            if (ret) {
+                gf_msg("glusterfs", GF_LOG_ERROR, EINVAL,
+                       LG_MSG_GRAPH_CLEANUP_FAILED,
+                       "%s failed to create clean "
+                       "up thread",
+                       volfile_id);
+                ret = 0;
+            }
+        }
+        if (clean_graph)
+            glusterfs_muxsvc_cleanup_parent(ctx, clean_graph);
+    }
+    return ret;
+}
+
+int
+glusterfs_mux_volfile_reconfigure(FILE *newvolfile_fp, glusterfs_ctx_t *ctx,
+                                  gf_volfile_t *volfile_obj, char *checksum,
+                                  dict_t *dict)
+{
+    glusterfs_graph_t *oldvolfile_graph = NULL;
+    glusterfs_graph_t *newvolfile_graph = NULL;
+    char vol_id[NAME_MAX + 1];
+
+    int ret = -1;
+
+    if (!ctx) {
+        gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL,
+               "ctx is NULL");
+        goto out;
+    }
+
+    /* Change the message id */
+    if (!volfile_obj) {
+        gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, 0, LG_MSG_CTX_NULL,
+               "failed to get volfile object");
+        goto out;
+    }
+
+    oldvolfile_graph = volfile_obj->graph;
+    if (!oldvolfile_graph) {
+        goto out;
+    }
+
+    newvolfile_graph = glusterfs_graph_construct(newvolfile_fp);
+
+    if (!newvolfile_graph) {
+        goto out;
+    }
+    newvolfile_graph->last_xl = glusterfs_get_last_xlator(newvolfile_graph);
+
+    glusterfs_graph_prepare(newvolfile_graph, ctx, newvolfile_graph->first);
+
+    if (!is_graph_topology_equal(oldvolfile_graph, newvolfile_graph)) {
+        ret = snprintf(vol_id, sizeof(vol_id), "%s", volfile_obj->vol_id);
+        if (ret < 0)
+            goto out;
+        ret = glusterfs_process_svc_detach(ctx, volfile_obj);
+        if (ret) {
+            gf_msg("glusterfsd-mgmt", GF_LOG_ERROR, EINVAL,
+                   LG_MSG_GRAPH_CLEANUP_FAILED,
+                   "Could not detach "
+                   "old graph. Aborting the reconfiguration operation");
+            goto out;
+        }
+        volfile_obj = NULL;
+        ret = glusterfs_process_svc_attach_volfp(ctx, newvolfile_fp, vol_id,
+                                                 checksum, dict);
+        goto out;
+    }
+
+    gf_msg_debug("glusterfsd-mgmt", 0,
+                 "Only options have changed in the"
+                 " new graph");
+
+    ret = glusterfs_graph_reconfigure(oldvolfile_graph, newvolfile_graph);
+    if (ret) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "Could not reconfigure "
+                     "new options in old graph");
+        goto out;
+    }
+    memcpy(volfile_obj->volfile_checksum, checksum,
+           sizeof(volfile_obj->volfile_checksum));
+
+    ret = 0;
+out:
+
+    if (newvolfile_graph)
+        glusterfs_graph_destroy(newvolfile_graph);
+
+    return ret;
+}
diff --git a/libglusterfs/src/graph.l b/libglusterfs/src/graph.l
index 08f7d536736..b9d4b2b6828 100644
--- a/libglusterfs/src/graph.l
+++ b/libglusterfs/src/graph.l
@@ -1,60 +1,40 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 %x STRING
 %option yylineno
 %option noinput
 %{
 
 #define YYSTYPE char *
-#include "xlator.h"
+#include "glusterfs/xlator.h"
 #include "y.tab.h"
 #include <string.h>
-#define START_STRSIZE	32
 
 static char *text;
-static int text_asize;
 static int text_size;
 
 void append_string(const char *str, int size)
 {
-	int new_size = text_size + size + 1;
-	if (new_size > text_asize) {
-		new_size += START_STRSIZE - 1;
-		new_size &= -START_STRSIZE;
-                if (!text) {
-                        text = GF_CALLOC (1, new_size,
-                                          gf_common_mt_char);
-                } else {
-                        text = GF_REALLOC (text, new_size);
-                }
-                if (!text) {
-                        gf_log ("parser", GF_LOG_ERROR,
-                                "out of memory");
-                        return;
-                }
-		text_asize = new_size;
-	}
-        memcpy(text + text_size, str, size);
-	text_size += size;
-	text[text_size] = 0;
+    int new_size = text_size + size + 1;
+    if (!text) {
+        text = GF_CALLOC (1, new_size, gf_common_mt_char);
+    } else {
+        text = GF_REALLOC (text, new_size);
+    }
+    if (!text) {
+        return;
+    }
+    memcpy(text + text_size, str, size);
+    text_size += size;
+    text[text_size] = 0;
 }
 
 %}
@@ -77,13 +57,15 @@ TYPE   [t][y][p][e]
        \\.        {    append_string (yytext + 1, yyleng - 1); }
        \"         {
                        if (0) {
-                              yyunput (0, NULL);
+                           yyunput (0, NULL);
                        }
                        BEGIN (INITIAL);
-                       yylval = text;
+                       graphyylval = text;
+                       text = NULL;
+                       text_size = 0;
                        return STRING_TOK;
-       }
+                  }
 }
-[^ \t\r\n\"\\]+ {    yylval = gf_strdup (yytext) ; return ID; }
+[^ \t\r\n\"\\]+ {    graphyylval = gf_strdup (yytext) ; return ID; }
 [ \t\r\n]+             ;
 %%
diff --git a/libglusterfs/src/graph.y b/libglusterfs/src/graph.y
index 52e15940ac7..e63febdc08b 100644
--- a/libglusterfs/src/graph.y
+++ b/libglusterfs/src/graph.y
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 %token VOLUME_BEGIN VOLUME_END OPTION NEWLINE SUBVOLUME ID WHITESPACE COMMENT TYPE STRING_TOK
 
 %{
@@ -28,10 +18,15 @@
 #include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <pthread.h>
 
-#include "xlator.h"
-#include "graph-utils.h"
-#include "logging.h"
+#define RELAX_POISONING
+
+#include "glusterfs/xlator.h"
+#include "glusterfs/graph-utils.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/syscall.h"
+#include "glusterfs/libglusterfs-messages.h"
 
 static int new_volume (char *name);
 static int volume_type (char *type);
@@ -45,8 +40,8 @@ static void option_error (void);
 #define YYSTYPE char *
 #define GF_CMD_BUFFER_LEN (8 * GF_UNIT_KB)
 
-int yyerror (const char *);
-int yylex ();
+int graphyyerror (const char *);
+int graphyylex ();
 %}
 
 
@@ -86,11 +81,11 @@ glusterfs_graph_t *construct;
 static void
 type_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
-        gf_log ("parser", GF_LOG_ERROR,
+        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLFILE_PARSE_ERROR,
                 "Volume %s, before line %d: Please specify volume type",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -98,11 +93,11 @@ type_error (void)
 static void
 sub_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
-        gf_log ("parser", GF_LOG_ERROR,
+        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLFILE_PARSE_ERROR,
                 "Volume %s, before line %d: Please specify subvolumes",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -110,12 +105,12 @@ sub_error (void)
 static void
 option_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
-        gf_log ("parser", GF_LOG_ERROR,
+        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLFILE_PARSE_ERROR,
                 "Volume %s, before line %d: Please specify "
                 "option <key> <value>",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -123,21 +118,20 @@ option_error (void)
 static int
 new_volume (char *name)
 {
-        extern int   yylineno;
+        extern int   graphyylineno;
         xlator_t    *trav = NULL;
         int          ret = 0;
 
         if (!name) {
-                gf_log ("parser", GF_LOG_DEBUG,
-			"Invalid argument name: '%s'", name);
+                gf_msg_debug ("parser", 0,"Invalid argument name");
                 ret = -1;
                 goto out;
         }
 
         if (curr) {
-                gf_log ("parser", GF_LOG_ERROR,
-                        "new volume (%s) defintion in line %d unexpected",
-                        name, yylineno);
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
+                        "new volume (%s) definition in line %d unexpected",
+                        name, graphyylineno);
                 ret = -1;
                 goto out;
         }
@@ -146,7 +140,6 @@ new_volume (char *name)
                                    gf_common_mt_xlator_t);
 
         if (!curr) {
-                gf_log ("parser", GF_LOG_ERROR, "Out of memory");
                 ret = -1;
                 goto out;
         }
@@ -155,9 +148,9 @@ new_volume (char *name)
 
         while (trav) {
                 if (!strcmp (name, trav->name)) {
-                        gf_log ("parser", GF_LOG_ERROR,
-				"Line %d: volume '%s' defined again",
-                                yylineno, name);
+                        gf_msg ("parser", GF_LOG_ERROR, 0,
+                                LG_MSG_VOLFILE_PARSE_ERROR, "Line %d: volume "
+                                "'%s' defined again", graphyylineno, name);
                         ret = -1;
                         goto out;
                 }
@@ -171,7 +164,8 @@ new_volume (char *name)
                 goto out;
         }
 
-        curr->options = get_new_dict ();
+        INIT_LIST_HEAD(&curr->volume_options);
+        curr->options = dict_new ();
 
         if (!curr->options) {
                 GF_FREE (curr->name);
@@ -189,8 +183,9 @@ new_volume (char *name)
         construct->first = curr;
 
         construct->xl_count++;
+        curr->xl_id = construct->xl_count;
 
-        gf_log ("parser", GF_LOG_TRACE, "New node for '%s'", name);
+        gf_msg_trace ("parser", 0, "New node for '%s'", name);
 
 out:
         GF_FREE (name);
@@ -202,26 +197,26 @@ out:
 static int
 volume_type (char *type)
 {
-        extern int   yylineno;
+        extern int   graphyylineno;
         int32_t      ret = 0;
 
         if (!type) {
-                gf_log ("parser", GF_LOG_DEBUG, "Invalid argument type");
+                gf_msg_debug ("parser", 0, "Invalid argument type");
                 ret = -1;
                 goto out;
         }
 
         ret = xlator_set_type (curr, type);
         if (ret) {
-                gf_log ("parser", GF_LOG_ERROR,
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
                         "Volume '%s', line %d: type '%s' is not valid or "
 			"not found on this machine",
-                        curr->name, yylineno, type);
+                        curr->name, graphyylineno, type);
                 ret = -1;
                 goto out;
         }
 
-        gf_log ("parser", GF_LOG_TRACE, "Type:%s:%s", curr->name, type);
+        gf_msg_trace ("parser", 0, "Type:%s:%s", curr->name, type);
 
 out:
         GF_FREE (type);
@@ -233,30 +228,30 @@ out:
 static int
 volume_option (char *key, char *value)
 {
-        extern int  yylineno;
+        extern int  graphyylineno;
         int         ret = 0;
         char       *set_value = NULL;
 
         if (!key || !value){
-                gf_log ("parser", GF_LOG_ERROR, "Invalid argument");
+                gf_msg ("parser", GF_LOG_ERROR, 0,
+                        LG_MSG_INVALID_VOLFILE_ENTRY, "Invalid argument");
                 ret = -1;
                 goto out;
         }
 
         set_value = gf_strdup (value);
-	ret = dict_set_dynstr (curr->options, key, set_value);
+	ret = dict_set_option (curr->options, key, set_value);
 
         if (ret == 1) {
-                gf_log ("parser", GF_LOG_ERROR,
-                        "Volume '%s', line %d: duplicate entry "
-			"('option %s') present",
-                        curr->name, yylineno, key);
+                gf_msg ("parser", GF_LOG_ERROR, 0,
+                        LG_MSG_INVALID_VOLFILE_ENTRY, "Volume '%s', line %d: "
+                        "duplicate entry ('option %s') present",
+                        curr->name, graphyylineno, key);
                 ret = -1;
                 goto out;
         }
 
-        gf_log ("parser", GF_LOG_TRACE, "Option:%s:%s:%s",
-                curr->name, key, value);
+        gf_msg_trace ("parser", 0, "Option:%s:%s:%s", curr->name, key, value);
 
 out:
         GF_FREE (key);
@@ -269,12 +264,13 @@ out:
 static int
 volume_sub (char *sub)
 {
-        extern int       yylineno;
+        extern int       graphyylineno;
         xlator_t        *trav = NULL;
         int              ret = 0;
 
         if (!sub) {
-                gf_log ("parser", GF_LOG_ERROR, "Invalid subvolumes argument");
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
+                        "Invalid subvolumes argument");
                 ret = -1;
                 goto out;
         }
@@ -288,30 +284,28 @@ volume_sub (char *sub)
         }
 
         if (!trav) {
-                gf_log ("parser", GF_LOG_ERROR,
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_SUB_VOLUME_ERROR,
                         "Volume '%s', line %d: subvolume '%s' is not defined "
-			"prior to usage",
-                        curr->name, yylineno, sub);
+			"prior to usage",curr->name, graphyylineno, sub);
                 ret = -1;
                 goto out;
         }
 
         if (trav == curr) {
-                gf_log ("parser", GF_LOG_ERROR,
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_INVALID_ENTRY,
                         "Volume '%s', line %d: has '%s' itself as subvolume",
-                        curr->name, yylineno, sub);
+                        curr->name, graphyylineno, sub);
                 ret = -1;
                 goto out;
         }
 
 	ret = glusterfs_xlator_link (curr, trav);
 	if (ret) {
-                gf_log ("parser", GF_LOG_ERROR, "Out of memory");
                 ret = -1;
                 goto out;
         }
 
-        gf_log ("parser", GF_LOG_TRACE, "child:%s->%s", curr->name, sub);
+        gf_msg_trace ("parser", 0, "child:%s->%s", curr->name, sub);
 
 out:
         GF_FREE (sub);
@@ -324,11 +318,11 @@ static int
 volume_end (void)
 {
         if (!curr->fops) {
-                gf_log ("parser", GF_LOG_ERROR,
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLUME_ERROR,
                         "\"type\" not specified for volume %s", curr->name);
                 return -1;
         }
-        gf_log ("parser", GF_LOG_TRACE, "end:%s", curr->name);
+        gf_msg_trace ("parser", 0, "end:%s", curr->name);
 
         curr = NULL;
         return 0;
@@ -336,55 +330,52 @@ volume_end (void)
 
 
 int
-yywrap ()
+graphyywrap ()
 {
         return 1;
 }
 
 
 int
-yyerror (const char *str)
+graphyyerror (const char *str)
 {
-        extern char  *yytext;
-        extern int    yylineno;
-
-        if (curr && curr->name && yytext) {
-                if (!strcmp (yytext, "volume")) {
-                        gf_log ("parser", GF_LOG_ERROR,
-                                "'end-volume' not defined for volume '%s'",
-				curr->name);
-                } else if (!strcmp (yytext, "type")) {
-                        gf_log ("parser", GF_LOG_ERROR,
+        extern char  *graphyytext;
+        extern int    graphyylineno;
+
+        if (curr && curr->name && graphyytext) {
+                if (!strcmp (graphyytext, "volume")) {
+                        gf_msg ("parser", GF_LOG_ERROR, 0,
+                                LG_MSG_VOLUME_ERROR, "'end-volume' not"
+                                " defined for volume '%s'", curr->name);
+                } else if (!strcmp (graphyytext, "type")) {
+                        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLUME_ERROR,
                                 "line %d: duplicate 'type' defined for "
-				"volume '%s'",
-                                yylineno, curr->name);
-                } else if (!strcmp (yytext, "subvolumes")) {
-                        gf_log ("parser", GF_LOG_ERROR,
-                                "line %d: duplicate 'subvolumes' defined for "
-				"volume '%s'",
-                                yylineno, curr->name);
+				"volume '%s'", graphyylineno, curr->name);
+                } else if (!strcmp (graphyytext, "subvolumes")) {
+                        gf_msg ("parser", GF_LOG_ERROR, 0,
+                                LG_MSG_SUB_VOLUME_ERROR, "line %d: duplicate "
+                                "'subvolumes' defined for volume '%s'",
+                                graphyylineno, curr->name);
                 } else if (curr) {
-                        gf_log ("parser", GF_LOG_ERROR,
+                        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_SYNTAX_ERROR,
                                 "syntax error: line %d (volume '%s'): \"%s\""
 				"\nallowed tokens are 'volume', 'type', "
 				"'subvolumes', 'option', 'end-volume'()",
-                                yylineno, curr->name,
-				yytext);
+                                graphyylineno, curr->name, graphyytext);
                 } else {
-                        gf_log ("parser", GF_LOG_ERROR,
+                        gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_SYNTAX_ERROR,
                                 "syntax error: line %d (just after volume "
 				"'%s'): \"%s\"\n(%s)",
-                                yylineno, curr->name,
-				yytext,
+                                graphyylineno, curr->name, graphyytext,
                                 "allowed tokens are 'volume', 'type', "
 				"'subvolumes', 'option', 'end-volume'");
                 }
         } else {
-                gf_log ("parser", GF_LOG_ERROR,
-                        "syntax error in line %d: \"%s\" \n"
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_SYNTAX_ERROR,
+                        "syntax error in line %d: \"%s\"\n"
                         "(allowed tokens are 'volume', 'type', "
 			"'subvolumes', 'option', 'end-volume')\n",
-                        yylineno, yytext);
+                        graphyylineno, graphyytext);
         }
 
         return -1;
@@ -402,7 +393,8 @@ execute_cmd (char *cmd, char **result, size_t size)
 
 	fpp = popen (cmd, "r");
 	if (!fpp) {
-		gf_log ("parser", GF_LOG_ERROR, "%s: failed to popen", cmd);
+		gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_FILE_OP_FAILED,
+                        "%s: failed to popen", cmd);
 		return -1;
 	}
 
@@ -443,8 +435,6 @@ preprocess (FILE *srcfp, FILE *dstfp)
         char    in_backtick = 0;
 	int     line = 1;
         int     column = 0;
-        int     backtick_line = 0;
-        int     backtick_column = 0;
         int     character = 0;
 
 
@@ -454,7 +444,6 @@ preprocess (FILE *srcfp, FILE *dstfp)
 	cmd = GF_CALLOC (cmd_buf_size, 1,
                          gf_common_mt_char);
         if (cmd == NULL) {
-                gf_log ("parser", GF_LOG_ERROR, "Out of memory");
                 return -1;
         }
 
@@ -462,7 +451,6 @@ preprocess (FILE *srcfp, FILE *dstfp)
                             gf_common_mt_char);
         if (result == NULL) {
                 GF_FREE (cmd);
-                gf_log ("parser", GF_LOG_ERROR, "Out of memory");
                 return -1;
         }
 
@@ -482,9 +470,6 @@ preprocess (FILE *srcfp, FILE *dstfp)
 			} else {
 				i = 0;
 				cmd[i] = '\0';
-
-				backtick_column = column;
-				backtick_line = line;
 			}
 
 			in_backtick = !in_backtick;
@@ -494,6 +479,7 @@ preprocess (FILE *srcfp, FILE *dstfp)
 					cmd_buf_size *= 2;
 					cmd = GF_REALLOC (cmd, cmd_buf_size);
                                         if (cmd == NULL) {
+                                                GF_FREE (result);
                                                 return -1;
                                         }
 
@@ -526,15 +512,16 @@ preprocess (FILE *srcfp, FILE *dstfp)
         }
 
 	if (in_backtick) {
-		gf_log ("parser", GF_LOG_ERROR,
-			"Unterminated backtick in volume specfication file at line (%d), column (%d).",
-			line, column);
+		gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_VOLUME_ERROR,
+			"Unterminated backtick in volume specification file at "
+                        "line (%d), column (%d).", line, column);
                 ret = -1;
 	}
 
 out:
 	fseek (srcfp, 0L, SEEK_SET);
 	fseek (dstfp, 0L, SEEK_SET);
+
 	GF_FREE (cmd);
 	GF_FREE (result);
 
@@ -542,7 +529,7 @@ out:
 }
 
 
-extern FILE *yyin;
+extern FILE *graphyyin;
 
 glusterfs_graph_t *
 glusterfs_graph_new ()
@@ -556,6 +543,9 @@ glusterfs_graph_new ()
 
         INIT_LIST_HEAD (&graph->list);
 
+        pthread_mutex_init(&graph->mutex, NULL);
+        pthread_cond_init(&graph->child_down_cond, NULL);
+
         gettimeofday (&graph->dob, NULL);
 
         return graph;
@@ -566,52 +556,66 @@ glusterfs_graph_t *
 glusterfs_graph_construct (FILE *fp)
 {
         int                ret = 0;
+        int                tmp_fd = -1;
         glusterfs_graph_t *graph = NULL;
-	FILE              *tmp_file = NULL;
+        FILE              *tmp_file = NULL;
+        char               template[] = "/tmp/tmp.XXXXXX";
+	static pthread_mutex_t graph_mutex = PTHREAD_MUTEX_INITIALIZER;
 
         graph = glusterfs_graph_new ();
         if (!graph)
-                return NULL;
+                goto err;
 
-	tmp_file = tmpfile ();
+        /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+        tmp_fd = mkstemp (template);
+        if (-1 == tmp_fd)
+                goto err;
 
-	if (tmp_file == NULL) {
-		gf_log ("parser", GF_LOG_ERROR,
-			"cannot create temparory file");
+        ret = sys_unlink (template);
+        if (ret < 0) {
+                gf_msg ("parser", GF_LOG_WARNING, 0, LG_MSG_FILE_OP_FAILED,
+                        "Unable to delete file: %s", template);
+        }
 
-                glusterfs_graph_destroy (graph);
-		return NULL;
-	}
+        tmp_file = fdopen (tmp_fd, "w+b");
+        if (!tmp_file)
+                goto err;
 
-	ret = preprocess (fp, tmp_file);
-	if (ret < 0) {
-		gf_log ("parser", GF_LOG_ERROR,
-			"parsing of backticks failed");
+        ret = preprocess (fp, tmp_file);
+        if (ret < 0) {
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_BACKTICK_PARSE_FAILED,
+                        "parsing of backticks failed");
+                goto err;
+        }
 
-                glusterfs_graph_destroy (graph);
-		fclose (tmp_file);
-		return NULL;
+	pthread_mutex_lock (&graph_mutex);
+	{
+		graphyyin = tmp_file;
+		construct = graph;
+		ret = yyparse ();
+		construct = NULL;
 	}
-
-        yyin = tmp_file;
-
-        construct = graph;
-
-        ret = yyparse ();
-
-        construct = NULL;
-
-	fclose (tmp_file);
+	pthread_mutex_unlock (&graph_mutex);
 
         if (ret == 1) {
-                gf_log ("parser", GF_LOG_DEBUG,
-			"parsing of volfile failed, please review it "
-			"once more");
-
-                glusterfs_graph_destroy (graph);
-                return NULL;
+                gf_msg_debug ("parser", 0, "parsing of volfile failed, please "
+                              "review it once more");
+                goto err;
         }
 
+        fclose (tmp_file);
         return graph;
+err:
+        if (tmp_file) {
+                fclose (tmp_file);
+        } else {
+                gf_msg ("parser", GF_LOG_ERROR, 0, LG_MSG_FILE_OP_FAILED,
+                        "cannot create temporary file");
+                if (-1 != tmp_fd)
+                        sys_close (tmp_fd);
+        }
+
+        glusterfs_graph_destroy (graph);
+        return NULL;
 }
 
diff --git a/libglusterfs/src/hashfn.c b/libglusterfs/src/hashfn.c
index 98f74188599..d2237e99f83 100644
--- a/libglusterfs/src/hashfn.c
+++ b/libglusterfs/src/hashfn.c
@@ -1,48 +1,21 @@
 /*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <stdint.h>
 #include <stdlib.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "hashfn.h"
-
-#define get16bits(d) (*((const uint16_t *) (d)))
+#define get16bits(d) (*((const uint16_t *)(d)))
 
 #define DM_DELTA 0x9E3779B9
-#define DM_FULLROUNDS 10        /* 32 is overkill, 16 is strong crypto */
-#define DM_PARTROUNDS 6	        /* 6 gets complete mixing */
-
-
-uint32_t
-ReallySimpleHash (char *path, int len)
-{
-        uint32_t        hash = 0;
-        for (;len > 0; len--)
-                hash ^= (char)path[len];
-
-        return hash;
-}
+#define DM_FULLROUNDS 10 /* 32 is overkill, 16 is strong crypto */
+#define DM_PARTROUNDS 6  /* 6 gets complete mixing */
 
 /*
   This is apparently the "fastest hash function for strings".
@@ -51,146 +24,145 @@ ReallySimpleHash (char *path, int len)
 
 /* In any case make sure, you return 1 */
 
-uint32_t SuperFastHash (const char * data, int32_t len) {
-  uint32_t hash = len, tmp;
-  int32_t rem;
-
-  if (len <= 1 || data == NULL) return 1;
-
-  rem = len & 3;
-  len >>= 2;
-
-  /* Main loop */
-  for (;len > 0; len--) {
-    hash  += get16bits (data);
-    tmp    = (get16bits (data+2) << 11) ^ hash;
-    hash   = (hash << 16) ^ tmp;
-    data  += 2*sizeof (uint16_t);
-    hash  += hash >> 11;
-  }
-
-  /* Handle end cases */
-  switch (rem) {
-  case 3: hash += get16bits (data);
-    hash ^= hash << 16;
-    hash ^= data[sizeof (uint16_t)] << 18;
-    hash += hash >> 11;
-    break;
-  case 2: hash += get16bits (data);
-    hash ^= hash << 11;
+uint32_t
+SuperFastHash(const char *data, int32_t len)
+{
+    uint32_t hash = len, tmp;
+    int32_t rem;
+
+    if (len <= 1 || data == NULL)
+        return 1;
+
+    rem = len & 3;
+    len >>= 2;
+
+    /* Main loop */
+    for (; len > 0; len--) {
+        hash += get16bits(data);
+        tmp = (get16bits(data + 2) << 11) ^ hash;
+        hash = (hash << 16) ^ tmp;
+        data += 2 * sizeof(uint16_t);
+        hash += hash >> 11;
+    }
+
+    /* Handle end cases */
+    switch (rem) {
+        case 3:
+            hash += get16bits(data);
+            hash ^= hash << 16;
+            hash ^= data[sizeof(uint16_t)] << 18;
+            hash += hash >> 11;
+            break;
+        case 2:
+            hash += get16bits(data);
+            hash ^= hash << 11;
+            hash += hash >> 17;
+            break;
+        case 1:
+            hash += *data;
+            hash ^= hash << 10;
+            hash += hash >> 1;
+    }
+
+    /* Force "avalanching" of final 127 bits */
+    hash ^= hash << 3;
+    hash += hash >> 5;
+    hash ^= hash << 4;
     hash += hash >> 17;
-    break;
-  case 1: hash += *data;
-    hash ^= hash << 10;
-    hash += hash >> 1;
-  }
-
-  /* Force "avalanching" of final 127 bits */
-  hash ^= hash << 3;
-  hash += hash >> 5;
-  hash ^= hash << 4;
-  hash += hash >> 17;
-  hash ^= hash << 25;
-  hash += hash >> 6;
-
-  return hash;
-}
+    hash ^= hash << 25;
+    hash += hash >> 6;
 
+    return hash;
+}
 
 /* Davies-Meyer hashing function implementation
  */
 static int
-dm_round (int rounds, uint32_t *array, uint32_t *h0, uint32_t *h1)
+dm_round(int rounds, uint32_t *array, uint32_t *h0, uint32_t *h1)
 {
-	uint32_t sum = 0;
-	int      n = 0;
-	uint32_t b0  = 0;
-	uint32_t b1  = 0;
-
-	b0 = *h0;
-	b1 = *h1;
-
-	n = rounds;
-
-	do {
-		sum += DM_DELTA;
-		b0  += ((b1 << 4) + array[0])
-			^ (b1 + sum)
-			^ ((b1 >> 5) + array[1]);
-		b1  += ((b0 << 4) + array[2])
-			^ (b0 + sum)
-			^ ((b0 >> 5) + array[3]);
-	} while (--n);
-
-	*h0 += b0;
-	*h1 += b1;
-
-	return 0;
-}
+    uint32_t sum = 0;
+    int n = 0;
+    uint32_t b0 = 0;
+    uint32_t b1 = 0;
+
+    b0 = *h0;
+    b1 = *h1;
 
+    n = rounds;
+
+    do {
+        sum += DM_DELTA;
+        b0 += ((b1 << 4) + array[0]) ^ (b1 + sum) ^ ((b1 >> 5) + array[1]);
+        b1 += ((b0 << 4) + array[2]) ^ (b0 + sum) ^ ((b0 >> 5) + array[3]);
+    } while (--n);
+
+    *h0 += b0;
+    *h1 += b1;
+
+    return 0;
+}
 
 uint32_t
-__pad (int len)
+__pad(int len)
 {
-	uint32_t pad = 0;
+    uint32_t pad = 0;
 
-	pad = (uint32_t) len | ((uint32_t) len << 8);
-	pad |= pad << 16;
+    pad = (uint32_t)len | ((uint32_t)len << 8);
+    pad |= pad << 16;
 
-	return pad;
+    return pad;
 }
 
 uint32_t
-gf_dm_hashfn (const char *msg, int len)
+gf_dm_hashfn(const char *msg, int len)
 {
-	uint32_t  h0 = 0x9464a485;
-	uint32_t  h1 = 0x542e1a94;
-	uint32_t  array[4];
-	uint32_t  pad = 0;
-	int       i = 0;
-	int       j = 0;
-	int       full_quads = 0;
-	int       full_words = 0;
-	int       full_bytes = 0;
-	uint32_t *intmsg = NULL;
-	int       word = 0;
-
-
-	intmsg = (uint32_t *) msg;
-	pad = __pad (len);
-
-	full_bytes   = len;
-	full_words   = len / 4;
-	full_quads   = len / 16;
-
-	for (i = 0; i < full_quads; i++) {
-		for (j = 0; j < 4; j++) {
-			word     = *intmsg;
-			array[j] = word;
-			intmsg++;
-			full_words--;
-			full_bytes -= 4;
-		}
-		dm_round (DM_PARTROUNDS, &array[0], &h0, &h1);
-	}
-
-	for (j = 0; j < 4; j++) {
-		if (full_words) {
-			word     = *intmsg;
-			array[j] = word;
-			intmsg++;
-			full_words--;
-			full_bytes -= 4;
-		} else {
-			array[j] = pad;
-			while (full_bytes) {
-				array[j] <<= 8;
-				array[j] |= msg[len - full_bytes];
-				full_bytes--;
-			}
-		}
-	}
-	dm_round (DM_FULLROUNDS, &array[0], &h0, &h1);
-
-	return h0 ^ h1;
+    uint32_t h0 = 0x9464a485;
+    uint32_t h1 = 0x542e1a94;
+    uint32_t array[4];
+    uint32_t pad = 0;
+    int i = 0;
+    int j = 0;
+    int full_quads = 0;
+    int full_words = 0;
+    int full_bytes = 0;
+    uint32_t *intmsg = NULL;
+    int word = 0;
+
+    intmsg = (uint32_t *)msg;
+    pad = __pad(len);
+
+    full_bytes = len;
+    full_words = len / 4;
+    full_quads = len / 16;
+
+    for (i = 0; i < full_quads; i++) {
+        for (j = 0; j < 4; j++) {
+            word = *intmsg;
+            array[j] = word;
+            intmsg++;
+            full_words--;
+            full_bytes -= 4;
+        }
+        dm_round(DM_PARTROUNDS, &array[0], &h0, &h1);
+    }
+
+    for (j = 0; j < 4; j++) {
+        if (full_words) {
+            word = *intmsg;
+            array[j] = word;
+            intmsg++;
+            full_words--;
+            full_bytes -= 4;
+        } else {
+            array[j] = pad;
+            while (full_bytes) {
+                array[j] <<= 8;
+                array[j] |= msg[len - full_bytes];
+                full_bytes--;
+            }
+        }
+    }
+    dm_round(DM_FULLROUNDS, &array[0], &h0, &h1);
+
+    return h0 ^ h1;
 }
diff --git a/libglusterfs/src/hashfn.h b/libglusterfs/src/hashfn.h
deleted file mode 100644
index 8261e4c7895..00000000000
--- a/libglusterfs/src/hashfn.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
-   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __HASHFN_H__
-#define __HASHFN_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/types.h>
-#include <stdint.h>
-
-uint32_t SuperFastHash (const char * data, int32_t len);
-
-uint32_t gf_dm_hashfn (const char *msg, int len);
-
-uint32_t
-ReallySimpleHash (char *path, int len);
-#endif /* __HASHFN_H__ */
diff --git a/libglusterfs/src/iatt.h b/libglusterfs/src/iatt.h
deleted file mode 100644
index ce4cc495d0d..00000000000
--- a/libglusterfs/src/iatt.h
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _IATT_H
-#define _IATT_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/types.h>
-#include <sys/stat.h> /* for iatt <--> stat conversions */
-#include <unistd.h>
-
-#include "compat.h"
-#include "uuid.h"
-
-typedef enum {
-        IA_INVAL = 0,
-        IA_IFREG,
-        IA_IFDIR,
-        IA_IFLNK,
-        IA_IFBLK,
-        IA_IFCHR,
-        IA_IFIFO,
-        IA_IFSOCK
-} ia_type_t;
-
-
-typedef struct {
-        uint8_t    suid:1;
-        uint8_t    sgid:1;
-        uint8_t    sticky:1;
-        struct {
-                uint8_t    read:1;
-                uint8_t    write:1;
-                uint8_t    exec:1;
-        } owner, group, other;
-} ia_prot_t;
-
-
-struct iatt {
-        uint64_t     ia_ino;        /* inode number */
-        uuid_t       ia_gfid;
-        uint64_t     ia_dev;        /* backing device ID */
-        ia_type_t    ia_type;       /* type of file */
-        ia_prot_t    ia_prot;       /* protection */
-        uint32_t     ia_nlink;      /* Link count */
-        uint32_t     ia_uid;        /* user ID of owner */
-        uint32_t     ia_gid;        /* group ID of owner */
-        uint64_t     ia_rdev;       /* device ID (if special file) */
-        uint64_t     ia_size;       /* file size in bytes */
-        uint32_t     ia_blksize;    /* blocksize for filesystem I/O */
-        uint64_t     ia_blocks;     /* number of 512B blocks allocated */
-        uint32_t     ia_atime;      /* last access time */
-        uint32_t     ia_atime_nsec;
-        uint32_t     ia_mtime;      /* last modification time */
-        uint32_t     ia_mtime_nsec;
-        uint32_t     ia_ctime;      /* last status change time */
-        uint32_t     ia_ctime_nsec;
-};
-
-
-#define IA_ISREG(t) (t == IA_IFREG)
-#define IA_ISDIR(t) (t == IA_IFDIR)
-#define IA_ISLNK(t) (t == IA_IFLNK)
-#define IA_ISBLK(t) (t == IA_IFBLK)
-#define IA_ISCHR(t) (t == IA_IFCHR)
-#define IA_ISFIFO(t) (t == IA_IFIFO)
-#define IA_ISSOCK(t) (t == IA_IFSOCK)
-
-#define IA_PROT_RUSR(prot)      ((prot).owner.read == 1)
-#define IA_PROT_WUSR(prot)      ((prot).owner.write == 1)
-#define IA_PROT_XUSR(prot)      ((prot).owner.exec == 1)
-
-#define IA_PROT_RGRP(prot)      ((prot).group.read == 1)
-#define IA_PROT_WGRP(prot)      ((prot).group.write == 1)
-#define IA_PROT_XGRP(prot)      ((prot).group.exec == 1)
-
-#define IA_PROT_ROTH(prot)      ((prot).other.read == 1)
-#define IA_PROT_WOTH(prot)      ((prot).other.write == 1)
-#define IA_PROT_XOTH(prot)      ((prot).other.exec == 1)
-
-#define IA_PROT_SUID(prot)      ((prot).suid == 1)
-#define IA_PROT_SGID(prot)      ((prot).sgid == 1)
-#define IA_PROT_STCKY(prot)     ((prot).sticky == 1)
-
-static inline uint32_t
-ia_major (uint64_t ia_dev)
-{
-        return (uint32_t) (ia_dev >> 32);
-}
-
-
-static inline uint32_t
-ia_minor (uint64_t ia_dev)
-{
-        return (uint32_t) (ia_dev & 0xffffffff);
-}
-
-
-static inline uint64_t
-ia_makedev (uint32_t ia_maj, uint32_t ia_min)
-{
-        return ((((uint64_t) ia_maj) << 32) | ia_min);
-}
-
-
-static inline ia_prot_t
-ia_prot_from_st_mode (mode_t mode)
-{
-        ia_prot_t ia_prot = {0, };
-
-        if (mode & S_ISUID)
-                ia_prot.suid = 1;
-        if (mode & S_ISGID)
-                ia_prot.sgid = 1;
-        if (mode & S_ISVTX)
-                ia_prot.sticky = 1;
-
-        if (mode & S_IRUSR)
-                ia_prot.owner.read = 1;
-        if (mode & S_IWUSR)
-                ia_prot.owner.write = 1;
-        if (mode & S_IXUSR)
-                ia_prot.owner.exec = 1;
-
-        if (mode & S_IRGRP)
-                ia_prot.group.read = 1;
-        if (mode & S_IWGRP)
-                ia_prot.group.write = 1;
-        if (mode & S_IXGRP)
-                ia_prot.group.exec = 1;
-
-        if (mode & S_IROTH)
-                ia_prot.other.read = 1;
-        if (mode & S_IWOTH)
-                ia_prot.other.write = 1;
-        if (mode & S_IXOTH)
-                ia_prot.other.exec = 1;
-
-        return ia_prot;
-}
-
-
-static inline ia_type_t
-ia_type_from_st_mode (mode_t mode)
-{
-        ia_type_t type = IA_INVAL;
-
-        if (S_ISREG (mode))
-                type = IA_IFREG;
-        if (S_ISDIR (mode))
-                type = IA_IFDIR;
-        if (S_ISLNK (mode))
-                type = IA_IFLNK;
-        if (S_ISBLK (mode))
-                type = IA_IFBLK;
-        if (S_ISCHR (mode))
-                type = IA_IFCHR;
-        if (S_ISFIFO (mode))
-                type = IA_IFIFO;
-        if (S_ISSOCK (mode))
-                type = IA_IFSOCK;
-
-        return type;
-}
-
-
-static inline mode_t
-st_mode_from_ia (ia_prot_t prot, ia_type_t type)
-{
-        mode_t    st_mode = 0;
-        uint32_t  type_bit = 0;
-        uint32_t  prot_bit = 0;
-
-        switch (type) {
-        case IA_IFREG:
-                type_bit = S_IFREG;
-                break;
-        case IA_IFDIR:
-                type_bit = S_IFDIR;
-                break;
-        case IA_IFLNK:
-                type_bit = S_IFLNK;
-                break;
-        case IA_IFBLK:
-                type_bit = S_IFBLK;
-                break;
-        case IA_IFCHR:
-                type_bit = S_IFCHR;
-                break;
-        case IA_IFIFO:
-                type_bit = S_IFIFO;
-                break;
-        case IA_IFSOCK:
-                type_bit = S_IFSOCK;
-                break;
-        case IA_INVAL:
-                break;
-        }
-
-        if (prot.suid)
-                prot_bit |= S_ISUID;
-        if (prot.sgid)
-                prot_bit |= S_ISGID;
-        if (prot.sticky)
-                prot_bit |= S_ISVTX;
-
-        if (prot.owner.read)
-                prot_bit |= S_IRUSR;
-        if (prot.owner.write)
-                prot_bit |= S_IWUSR;
-        if (prot.owner.exec)
-                prot_bit |= S_IXUSR;
-
-        if (prot.group.read)
-                prot_bit |= S_IRGRP;
-        if (prot.group.write)
-                prot_bit |= S_IWGRP;
-        if (prot.group.exec)
-                prot_bit |= S_IXGRP;
-
-        if (prot.other.read)
-                prot_bit |= S_IROTH;
-        if (prot.other.write)
-                prot_bit |= S_IWOTH;
-        if (prot.other.exec)
-                prot_bit |= S_IXOTH;
-
-        st_mode = (type_bit | prot_bit);
-
-        return st_mode;
-}
-
-
-static inline int
-iatt_from_stat (struct iatt *iatt, struct stat *stat)
-{
-        iatt->ia_dev        = stat->st_dev;
-        iatt->ia_ino        = stat->st_ino;
-
-        iatt->ia_type       = ia_type_from_st_mode (stat->st_mode);
-        iatt->ia_prot       = ia_prot_from_st_mode (stat->st_mode);
-
-        iatt->ia_nlink      = stat->st_nlink;
-        iatt->ia_uid        = stat->st_uid;
-        iatt->ia_gid        = stat->st_gid;
-
-        iatt->ia_rdev       = ia_makedev (major (stat->st_rdev),
-                                          minor (stat->st_rdev));
-
-        iatt->ia_size       = stat->st_size;
-        iatt->ia_blksize    = stat->st_blksize;
-        iatt->ia_blocks     = stat->st_blocks;
-
-        iatt->ia_atime      = stat->st_atime;
-        iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
-
-        iatt->ia_mtime      = stat->st_mtime;
-        iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
-
-        iatt->ia_ctime      = stat->st_ctime;
-        iatt->ia_ctime_nsec = ST_CTIM_NSEC (stat);
-
-        return 0;
-}
-
-
-static inline int
-iatt_to_stat (struct iatt *iatt, struct stat *stat)
-{
-        stat->st_dev        = iatt->ia_dev;
-        stat->st_ino        = iatt->ia_ino;
-
-        stat->st_mode       = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
-
-        stat->st_nlink      = iatt->ia_nlink;
-        stat->st_uid        = iatt->ia_uid;
-        stat->st_gid        = iatt->ia_gid;
-
-        stat->st_rdev       = makedev (ia_major (iatt->ia_rdev),
-                                       ia_minor (iatt->ia_rdev));
-
-        stat->st_size       = iatt->ia_size;
-        stat->st_blksize    = iatt->ia_blksize;
-        stat->st_blocks     = iatt->ia_blocks;
-
-        stat->st_atime      = iatt->ia_atime;
-        ST_ATIM_NSEC_SET (stat, iatt->ia_atime_nsec);
-
-        stat->st_mtime      = iatt->ia_mtime;
-        ST_MTIM_NSEC_SET (stat, iatt->ia_mtime_nsec);
-
-        stat->st_ctime      = iatt->ia_ctime;
-        ST_CTIM_NSEC_SET (stat, iatt->ia_ctime_nsec);
-
-        return 0;
-}
-
-
-#endif /* _IATT_H */
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 388485283ff..dbadf77442d 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -1,1577 +1,2677 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "inode.h"
-#include "fd.h"
-#include "common-utils.h"
-#include "statedump.h"
+#include "glusterfs/inode.h"
+#include "glusterfs/common-utils.h"
+#include "glusterfs/statedump.h"
 #include <pthread.h>
 #include <sys/types.h>
 #include <stdint.h>
-#include "list.h"
-#include <time.h>
+#include "glusterfs/list.h"
 #include <assert.h>
+#include "glusterfs/libglusterfs-messages.h"
 
 /* TODO:
    move latest accessed dentry to list_head of inode
 */
 
-#define INODE_DUMP_LIST(head, key_buf, key_prefix, list_type) \
-        {                                                               \
-                int i = 1;                                              \
-                inode_t *inode = NULL;                                  \
-                list_for_each_entry (inode, head, list) {               \
-                        gf_proc_dump_build_key(key_buf, key_prefix, "%s.%d",list_type, \
-                                               i++);                    \
-                        gf_proc_dump_add_section(key_buf);              \
-                        inode_dump(inode, key);                         \
-                }                                                       \
+// clang-format off
+/*
+
+Details as per Xavi:
+
+ I think we should have 3 lists: active, lru and invalidate.
+
+We'll need 3 things: refs, nlookups and invalidate_sent flag. Any change of
+refs, invalidate_sent flag and moving from one list to another must be done
+atomically.
+
+With this information, these are the states that cause a transition:
+
+    refs nlookups inv_sent    op
+      1      0        0      unref  -> refs = 0, active--->destroy
+      1      1        0      unref  -> refs = 0, active--->lru
+      1      1        0     forget  -> nlookups = 0, active--->active
+     *0      1        0     forget  -> nlookups = 0, lru--->destroy
+     *0      1        1     forget  -> nlookups = 0, invalidate--->destroy
+      0      1        0       ref   -> refs = 1, lru--->active
+      0      1        1       ref   -> refs = 1, inv_sent = 0, invalidate--->active
+      0      1        0    overflow -> refs = 1, inv_sent = 1, lru--->invalidate
+      1      1        1      unref  -> refs = 0, invalidate--->invalidate
+      1      1        1     forget  -> nlookups = 0, inv_sent = 0, invalidate--->active
+
+(*) technically these combinations cannot happen because a forget sent by the
+kernel first calls ref() and then unref(). However it's equivalent.
+
+overflow means that lru list has grown beyond the limit and the inode needs to
+be invalidated. All other combinations do not cause a change in state or are not
+possible.
+
+Based on this, the code could be similar to this:
+
+    ref(inode, inv)
+    {
+        if (refs == 0) {
+            if (inv_sent) {
+                invalidate_count--;
+                inv_sent = 0;
+            } else {
+                lru_count--;
+            }
+            if (inv) {
+                inv_sent = 1;
+                invalidate_count++;
+                list_move(inode, invalidate);
+            } else {
+                active_count++;
+                list_move(inode, active);
+            }
+        }
+        refs++;
+    }
+
+    unref(inode, clear)
+    {
+        if (clear && inv_sent) {
+            // there is a case of fuse itself sending forget, without
+            // invalidate, after entry delete, like unlink(), rmdir().
+            inv_sent = 0;
+            invalidate_count--;
+            active_count++;
+            list_move(inode, active);
+        }
+        refs--;
+        if ((refs == 0) && !inv_sent) {
+            active_count--;
+            if (nlookups == 0) {
+                destroy(inode);
+            } else {
+                lru_count++;
+                list_move(inode, lru);
+            }
         }
+    }
+
+    forget(inode)
+    {
+        ref(inode, false);
+        nlookups--;
+        unref(inode, true);
+    }
+
+    overflow(inode)
+    {
+        ref(inode, true);
+        invalidator(inode);
+        unref(inode, false);
+    }
+
+*/
+// clang-format on
+
+#define INODE_DUMP_LIST(head, key_buf, key_prefix, list_type)                  \
+    {                                                                          \
+        int i = 1;                                                             \
+        inode_t *inode = NULL;                                                 \
+        list_for_each_entry(inode, head, list)                                 \
+        {                                                                      \
+            gf_proc_dump_build_key(key_buf, key_prefix, "%s.%d", list_type,    \
+                                   i++);                                       \
+            gf_proc_dump_add_section("%s", key_buf);                           \
+            inode_dump(inode, key);                                            \
+        }                                                                      \
+    }
 
 static inode_t *
-__inode_unref (inode_t *inode);
+__inode_unref(inode_t *inode, bool clear);
 
 static int
-inode_table_prune (inode_table_t *table);
+inode_table_prune(inode_table_t *table);
 
 void
-fd_dump (struct list_head *head, char *prefix);
+fd_dump(struct list_head *head, char *prefix);
 
 static int
-hash_dentry (inode_t *parent, const char *name, int mod)
+hash_dentry(inode_t *parent, const char *name, int mod)
 {
-        int hash = 0;
-        int ret = 0;
+    int hash = 0;
+    int ret = 0;
 
-        hash = *name;
-        if (hash) {
-                for (name += 1; *name != '\0'; name++) {
-                        hash = (hash << 5) - hash + *name;
-                }
+    hash = *name;
+    if (hash) {
+        for (name += 1; *name != '\0'; name++) {
+            hash = (hash << 5) - hash + *name;
         }
-        ret = (hash + (unsigned long)parent) % mod;
+    }
+    ret = (hash + (unsigned long)parent) % mod;
 
-        return ret;
+    return ret;
 }
 
-
 static int
-hash_gfid (uuid_t uuid, int mod)
+hash_gfid(uuid_t uuid, int mod)
 {
-        int ret = 0;
-
-        ret = uuid[15] + (uuid[14] << 8);
-
-        return ret;
+    return ((uuid[15] + (uuid[14] << 8)) % mod);
 }
 
-
 static void
-__dentry_hash (dentry_t *dentry)
+__dentry_hash(dentry_t *dentry, const int hash)
 {
-        inode_table_t   *table = NULL;
-        int              hash = 0;
-
-        if (!dentry)
-                return;
+    inode_table_t *table = NULL;
 
-        table = dentry->inode->table;
-        hash = hash_dentry (dentry->parent, dentry->name,
-                            table->hashsize);
+    table = dentry->inode->table;
 
-        list_del_init (&dentry->hash);
-        list_add (&dentry->hash, &table->name_hash[hash]);
+    list_del_init(&dentry->hash);
+    list_add(&dentry->hash, &table->name_hash[hash]);
 }
 
-
 static int
-__is_dentry_hashed (dentry_t *dentry)
+__is_dentry_hashed(dentry_t *dentry)
 {
-        if (!dentry)
-                return 0;
-
-        return !list_empty (&dentry->hash);
+    return !list_empty(&dentry->hash);
 }
 
-
 static void
-__dentry_unhash (dentry_t *dentry)
+__dentry_unhash(dentry_t *dentry)
 {
-        if (!dentry)
-                return;
-
-        list_del_init (&dentry->hash);
+    list_del_init(&dentry->hash);
 }
 
-
 static void
-__dentry_unset (dentry_t *dentry)
+dentry_destroy(dentry_t *dentry)
 {
-        struct mem_pool *tmp_pool = NULL;
+    if (!dentry)
+        return;
 
-        if (!dentry)
-                return;
+    GF_FREE(dentry->name);
+    dentry->name = NULL;
+    mem_put(dentry);
 
-        tmp_pool = dentry->inode->table->dentry_pool;
-        __dentry_unhash (dentry);
+    return;
+}
 
-        list_del_init (&dentry->inode_list);
+static dentry_t *
+__dentry_unset(dentry_t *dentry)
+{
+    if (!dentry)
+        return NULL;
 
-        if (dentry->name)
-                GF_FREE (dentry->name);
+    __dentry_unhash(dentry);
 
-        if (dentry->parent) {
-                __inode_unref (dentry->parent);
-                dentry->parent = NULL;
-        }
+    list_del_init(&dentry->inode_list);
 
-        mem_put (tmp_pool, dentry);
-        tmp_pool = NULL;
+    if (dentry->parent) {
+        __inode_unref(dentry->parent, false);
+        dentry->parent = NULL;
+    }
 
+    return dentry;
 }
 
-
 static int
-__foreach_ancestor_dentry (dentry_t *dentry,
-                           int (per_dentry_fn) (dentry_t *dentry,
-                                                void *data),
-                           void *data)
+__foreach_ancestor_dentry(dentry_t *dentry,
+                          int(per_dentry_fn)(dentry_t *dentry, void *data),
+                          void *data)
 {
-        inode_t  *parent = NULL;
-        dentry_t *each = NULL;
-        int       ret = 0;
+    inode_t *parent = NULL;
+    dentry_t *each = NULL;
+    int ret = 0;
 
-        if (!dentry)
-                return 0;
-
-        ret = per_dentry_fn (dentry, data);
+    if (!dentry) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_DENTRY_NOT_FOUND,
+                         "dentry not found");
+        return 0;
+    }
+
+    ret = per_dentry_fn(dentry, data);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_PER_DENTRY_FAILED,
+                "ret=%d", ret, NULL);
+        goto out;
+    }
+
+    parent = dentry->parent;
+    if (!parent) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_PARENT_DENTRY_NOT_FOUND,
+                NULL);
+        goto out;
+    }
+
+    list_for_each_entry(each, &parent->dentry_list, inode_list)
+    {
+        ret = __foreach_ancestor_dentry(each, per_dentry_fn, data);
         if (ret)
-                goto out;
-
-        parent = dentry->parent;
-        if (!parent)
-                goto out;
-
-        list_for_each_entry (each, &parent->dentry_list, inode_list) {
-                ret = __foreach_ancestor_dentry (each, per_dentry_fn, data);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-__check_cycle (dentry_t *a_dentry, void *data)
+__check_cycle(dentry_t *a_dentry, void *data)
 {
-        inode_t *link_inode = NULL;
+    inode_t *link_inode = NULL;
 
-        link_inode = data;
+    link_inode = data;
 
-        if (a_dentry->parent == link_inode)
-                return 1;
+    if (a_dentry->parent == link_inode)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
-
 static int
-__is_dentry_cyclic (dentry_t *dentry)
-{
-        int       ret = 0;
-        inode_t  *inode = NULL;
-        char      uuidbuf[64];
-        char     *name = "<nul>";
-
-        ret = __foreach_ancestor_dentry (dentry, __check_cycle,
-                                         dentry->inode);
-        if (ret) {
-                inode = dentry->inode;
-
-                if (dentry->name)
-                        name = dentry->name;
-                uuid_unparse (inode->gfid, uuidbuf);
-
-                gf_log (dentry->inode->table->name, GF_LOG_CRITICAL,
-                        "detected cyclic loop formation during inode linkage."
-                        " inode (%"PRId64"/%s) linking under itself as %s",
-                        inode->ino, uuidbuf, name);
-        }
+__is_dentry_cyclic(dentry_t *dentry)
+{
+    int ret = 0;
 
-        return ret;
-}
+    ret = __foreach_ancestor_dentry(dentry, __check_cycle, dentry->inode);
+    if (ret) {
+        gf_smsg(dentry->inode->table->name, GF_LOG_CRITICAL, 0,
+                LG_MSG_DENTRY_CYCLIC_LOOP, "gfid=%s name=-%s",
+                uuid_utoa(dentry->inode->gfid), dentry->name, NULL);
+    }
 
+    return ret;
+}
 
 static void
-__inode_unhash (inode_t *inode)
+__inode_unhash(inode_t *inode)
 {
-        if (!inode)
-                return;
-
-        list_del_init (&inode->hash);
+    list_del_init(&inode->hash);
 }
 
-
 static int
-__is_inode_hashed (inode_t *inode)
+__is_inode_hashed(inode_t *inode)
 {
-        if (!inode)
-                return 0;
-
-        return !list_empty (&inode->hash);
+    return !list_empty(&inode->hash);
 }
 
-
 static void
-__inode_hash (inode_t *inode)
+__inode_hash(inode_t *inode, const int hash)
 {
-        inode_table_t *table = NULL;
-        int            hash = 0;
+    inode_table_t *table = inode->table;
 
-        if (!inode)
-                return;
-
-        table = inode->table;
-        hash = hash_gfid (inode->gfid, 65536);
-
-        list_del_init (&inode->hash);
-        list_add (&inode->hash, &table->inode_hash[hash]);
+    list_del_init(&inode->hash);
+    list_add(&inode->hash, &table->inode_hash[hash]);
 }
 
-
 static dentry_t *
-__dentry_search_for_inode (inode_t *inode, ino_t par, const char *name)
+__dentry_search_for_inode(inode_t *inode, uuid_t pargfid, const char *name)
 {
-        dentry_t *dentry = NULL;
-        dentry_t *tmp = NULL;
+    dentry_t *dentry = NULL;
+    dentry_t *tmp = NULL;
 
-        if (!inode || !name)
-                return NULL;
+    /* earlier, just the ino was sent, which could have been 0, now
+       we deal with gfid, and if sent gfid is null or 0, no need to
+       continue with the check */
+    if (!pargfid || gf_uuid_is_null(pargfid))
+        return NULL;
 
-        list_for_each_entry (tmp, &inode->dentry_list, inode_list) {
-                if (tmp->parent->ino == par && !strcmp (tmp->name, name)) {
-                        dentry = tmp;
-                        break;
-                }
+    list_for_each_entry(tmp, &inode->dentry_list, inode_list)
+    {
+        if ((gf_uuid_compare(tmp->parent->gfid, pargfid) == 0) &&
+            !strcmp(tmp->name, name)) {
+            dentry = tmp;
+            break;
         }
+    }
 
-        return dentry;
+    return dentry;
 }
 
-
 static void
-__inode_destroy (inode_t *inode)
+__inode_ctx_free(inode_t *inode)
 {
-        int          index = 0;
-        xlator_t    *xl = NULL;
-        xlator_t    *old_THIS = NULL;
-        struct mem_pool *tmp_pool = NULL;
-
-        if (!inode)
-                return;
-
-        if (!inode->_ctx)
-                goto noctx;
-
-        tmp_pool = inode->table->inode_pool;
-
-        for (index = 0; index < inode->table->xl->graph->xl_count; index++) {
-                if (inode->_ctx[index].xl_key) {
-                        xl = (xlator_t *)(long)inode->_ctx[index].xl_key;
-                        old_THIS = THIS;
-                        THIS = xl;
-                        if (xl->cbks->forget)
-                                xl->cbks->forget (xl, inode);
-                        THIS = old_THIS;
-                }
+    int index = 0;
+    xlator_t *xl = NULL;
+    xlator_t *old_THIS = NULL;
+
+    if (!inode->_ctx) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_CTX_NULL, NULL);
+        goto noctx;
+    }
+
+    for (index = 0; index < inode->table->ctxcount; index++) {
+        if (inode->_ctx[index].value1 || inode->_ctx[index].value2) {
+            xl = (xlator_t *)(long)inode->_ctx[index].xl_key;
+            if (xl && !xl->call_cleanup && xl->cbks->forget) {
+                old_THIS = THIS;
+                THIS = xl;
+                xl->cbks->forget(xl, inode);
+                THIS = old_THIS;
+            }
         }
+    }
 
-        GF_FREE (inode->_ctx);
-noctx:
-        LOCK_DESTROY (&inode->lock);
-        //  memset (inode, 0xb, sizeof (*inode));
-        mem_put (tmp_pool, inode);
-	 tmp_pool = NULL;
+    GF_FREE(inode->_ctx);
+    inode->_ctx = NULL;
 
+noctx:
+    return;
 }
 
-
 static void
-__inode_activate (inode_t *inode)
+__inode_destroy(inode_t *inode)
 {
-        if (!inode)
-                return;
+    __inode_ctx_free(inode);
 
-        list_move (&inode->list, &inode->table->active);
-        inode->table->active_size++;
+    LOCK_DESTROY(&inode->lock);
+    //  memset (inode, 0xb, sizeof (*inode));
+    mem_put(inode);
 }
 
-
-static void
-__inode_passivate (inode_t *inode)
+void
+inode_ctx_merge(fd_t *fd, inode_t *inode, inode_t *linked_inode)
 {
-        dentry_t      *dentry = NULL;
-        dentry_t      *t = NULL;
+    int index = 0;
+    xlator_t *xl = NULL;
+    xlator_t *old_THIS = NULL;
+
+    if (!fd || !inode || !linked_inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid inode");
+        return;
+    }
 
-        if (!inode)
-                return;
+    if (!inode->_ctx || !linked_inode->_ctx) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid inode context");
+        return;
+    }
 
-        list_move_tail (&inode->list, &inode->table->lru);
-        inode->table->lru_size++;
+    for (; index < inode->table->ctxcount; index++) {
+        if (inode->_ctx[index].xl_key) {
+            xl = (xlator_t *)(long)inode->_ctx[index].xl_key;
 
-        list_for_each_entry_safe (dentry, t, &inode->dentry_list, inode_list) {
-                if (!__is_dentry_hashed (dentry))
-                        __dentry_unset (dentry);
+            old_THIS = THIS;
+            THIS = xl;
+            if (xl->cbks->ictxmerge)
+                xl->cbks->ictxmerge(xl, fd, inode, linked_inode);
+            THIS = old_THIS;
         }
+    }
 }
 
+static void
+__inode_activate(inode_t *inode)
+{
+    list_move(&inode->list, &inode->table->active);
+    inode->table->active_size++;
+}
 
 static void
-__inode_retire (inode_t *inode)
+__inode_passivate(inode_t *inode)
 {
-        dentry_t      *dentry = NULL;
-        dentry_t      *t = NULL;
+    dentry_t *dentry = NULL;
+    dentry_t *t = NULL;
+
+    list_move_tail(&inode->list, &inode->table->lru);
+    inode->table->lru_size++;
 
-        if (!inode)
-                return;
+    list_for_each_entry_safe(dentry, t, &inode->dentry_list, inode_list)
+    {
+        if (!__is_dentry_hashed(dentry))
+            dentry_destroy(__dentry_unset(dentry));
+    }
+}
 
-        list_move_tail (&inode->list, &inode->table->purge);
-        inode->table->purge_size++;
+static void
+__inode_retire(inode_t *inode)
+{
+    dentry_t *dentry = NULL;
+    dentry_t *t = NULL;
 
-        __inode_unhash (inode);
+    list_move_tail(&inode->list, &inode->table->purge);
+    inode->table->purge_size++;
 
-        list_for_each_entry_safe (dentry, t, &inode->dentry_list, inode_list) {
-                __dentry_unset (dentry);
-        }
+    __inode_unhash(inode);
+
+    list_for_each_entry_safe(dentry, t, &inode->dentry_list, inode_list)
+    {
+        dentry_destroy(__dentry_unset(dentry));
+    }
 }
 
+static int
+__inode_get_xl_index(inode_t *inode, xlator_t *xlator)
+{
+    int set_idx = -1;
+
+    if ((inode->_ctx[xlator->xl_id].xl_key != NULL) &&
+        (inode->_ctx[xlator->xl_id].xl_key != xlator))
+        goto out;
+
+    set_idx = xlator->xl_id;
+    inode->_ctx[set_idx].xl_key = xlator;
+
+out:
+    return set_idx;
+}
 
 static inode_t *
-__inode_unref (inode_t *inode)
+__inode_unref(inode_t *inode, bool clear)
 {
-        if (!inode)
-                return NULL;
+    int index = 0;
+    xlator_t *this = NULL;
+    uint64_t nlookup = 0;
+
+    /*
+     * Root inode should always be in active list of inode table. So unrefs
+     * on root inode are no-ops.
+     */
+    if (__is_root_gfid(inode->gfid))
+        return inode;
 
-        if (inode->ino == 1)
-                return inode;
+    /*
+     * No need to acquire inode table's lock
+     * as __inode_unref is called after acquiding
+     * the inode table's lock.
+     */
+    if (inode->table->cleanup_started && !inode->ref)
+        /*
+         * There is a good chance that, the inode
+         * on which unref came has already been
+         * zero refed and added to the purge list.
+         * This can happen when inode table is
+         * being destroyed (glfs_fini is something
+         * which destroys the inode table).
+         *
+         * Consider a directory 'a' which has a file
+         * 'b'. Now as part of inode table destruction
+         * zero refing of inodes does not happen from
+         * leaf to the root. It happens in the order
+         * inodes are present in the list. So, in this
+         * example, the dentry of 'b' would have its
+         * parent set to the inode of 'a'. So if
+         * 'a' gets zero refed first (as part of
+         * inode table cleanup) and then 'b' has to
+         * zero refed, then dentry_unset is called on
+         * the dentry of 'b' and it further goes on to
+         * call inode_unref on b's parent which is 'a'.
+         * In this situation, GF_ASSERT would be called
+         * below as the refcount of 'a' has been already set
+         * to zero.
+         *
+         * So return the inode if the inode table cleanup
+         * has already started and inode refcount is 0.
+         */
+        return inode;
 
-        GF_ASSERT (inode->ref);
+    this = THIS;
 
-        --inode->ref;
+    if (clear && inode->in_invalidate_list) {
+        inode->in_invalidate_list = false;
+        inode->table->invalidate_size--;
+        __inode_activate(inode);
+    }
+    GF_ASSERT(inode->ref);
 
-        if (!inode->ref) {
-                inode->table->active_size--;
+    --inode->ref;
 
-                if (inode->nlookup)
-                        __inode_passivate (inode);
-                else
-                        __inode_retire (inode);
-        }
+    index = __inode_get_xl_index(inode, this);
+    if (index >= 0) {
+        inode->_ctx[index].xl_key = this;
+        inode->_ctx[index].ref--;
+    }
 
-        return inode;
-}
+    if (!inode->ref && !inode->in_invalidate_list) {
+        inode->table->active_size--;
 
+        nlookup = GF_ATOMIC_GET(inode->nlookup);
+        if (nlookup)
+            __inode_passivate(inode);
+        else
+            __inode_retire(inode);
+    }
+
+    return inode;
+}
 
 static inode_t *
-__inode_ref (inode_t *inode)
+__inode_ref(inode_t *inode, bool is_invalidate)
 {
-        if (!inode)
-                return NULL;
+    int index = 0;
+    xlator_t *this = NULL;
 
-        if (!inode->ref) {
-                inode->table->lru_size--;
-                __inode_activate (inode);
-        }
-        inode->ref++;
+    if (!inode)
+        return NULL;
 
+    this = THIS;
+
+    /*
+     * Root inode should always be in active list of inode table. So unrefs
+     * on root inode are no-ops. If we do not allow unrefs but allow refs,
+     * it leads to refcount overflows and deleting and adding the inode
+     * to active-list, which is ugly. active_size (check __inode_activate)
+     * in inode table increases which is wrong. So just keep the ref
+     * count as 1 always
+     */
+    if (__is_root_gfid(inode->gfid) && inode->ref)
         return inode;
-}
 
+    if (!inode->ref) {
+        if (inode->in_invalidate_list) {
+            inode->in_invalidate_list = false;
+            inode->table->invalidate_size--;
+        } else {
+            inode->table->lru_size--;
+        }
+        if (is_invalidate) {
+            inode->in_invalidate_list = true;
+            inode->table->invalidate_size++;
+            list_move_tail(&inode->list, &inode->table->invalidate);
+        } else {
+            __inode_activate(inode);
+        }
+    }
+
+    inode->ref++;
+
+    index = __inode_get_xl_index(inode, this);
+    if (index >= 0) {
+        inode->_ctx[index].xl_key = this;
+        inode->_ctx[index].ref++;
+    }
+
+    return inode;
+}
 
 inode_t *
-inode_unref (inode_t *inode)
+inode_unref(inode_t *inode)
 {
-        inode_table_t *table = NULL;
+    inode_table_t *table = NULL;
 
-        if (!inode)
-                return NULL;
+    if (!inode)
+        return NULL;
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                inode = __inode_unref (inode);
-        }
-        pthread_mutex_unlock (&table->lock);
+    pthread_mutex_lock(&table->lock);
+    {
+        inode = __inode_unref(inode, false);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        inode_table_prune (table);
+    inode_table_prune(table);
 
-        return inode;
+    return inode;
 }
 
-
 inode_t *
-inode_ref (inode_t *inode)
+inode_ref(inode_t *inode)
 {
-        inode_table_t *table = NULL;
+    inode_table_t *table = NULL;
 
-        if (!inode)
-                return NULL;
+    if (!inode)
+        return NULL;
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                inode = __inode_ref (inode);
-        }
-        pthread_mutex_unlock (&table->lock);
+    pthread_mutex_lock(&table->lock);
+    {
+        inode = __inode_ref(inode, false);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        return inode;
+    return inode;
 }
 
-
 static dentry_t *
-__dentry_create (inode_t *inode, inode_t *parent, const char *name)
+dentry_create(inode_t *inode, inode_t *parent, const char *name)
 {
-        dentry_t      *newd = NULL;
-
-        if (!inode || !parent || !name)
-                return NULL;
+    dentry_t *newd = NULL;
 
-        newd = mem_get0 (parent->table->dentry_pool);
+    newd = mem_get0(parent->table->dentry_pool);
+    if (newd == NULL) {
+        goto out;
+    }
 
-        if (newd == NULL) {
-                gf_log ("inode", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&newd->inode_list);
-        INIT_LIST_HEAD (&newd->hash);
-
-        newd->name = gf_strdup (name);
-        if (newd->name == NULL) {
-                gf_log ("inode", GF_LOG_ERROR, "out of memory");
-                mem_put (parent->table->dentry_pool, newd);
-                newd = NULL;
-                goto out;
-        }
+    INIT_LIST_HEAD(&newd->inode_list);
+    INIT_LIST_HEAD(&newd->hash);
 
-        if (parent)
-                newd->parent = __inode_ref (parent);
+    newd->name = gf_strdup(name);
+    if (newd->name == NULL) {
+        mem_put(newd);
+        newd = NULL;
+        goto out;
+    }
 
-        list_add (&newd->inode_list, &inode->dentry_list);
-        newd->inode = inode;
+    newd->inode = inode;
 
 out:
-        return newd;
+    return newd;
 }
 
-
 static inode_t *
-__inode_create (inode_table_t *table)
+inode_create(inode_table_t *table)
 {
-        inode_t  *newi = NULL;
-
-        if (!table)
-                return NULL;
+    inode_t *newi = NULL;
 
-        newi = mem_get0 (table->inode_pool);
-        if (!newi) {
-                gf_log ("inode", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-
-        newi->table = table;
+    newi = mem_get0(table->inode_pool);
+    if (!newi) {
+        goto out;
+    }
 
-        LOCK_INIT (&newi->lock);
+    newi->table = table;
 
-        INIT_LIST_HEAD (&newi->fd_list);
-        INIT_LIST_HEAD (&newi->list);
-        INIT_LIST_HEAD (&newi->hash);
-        INIT_LIST_HEAD (&newi->dentry_list);
+    LOCK_INIT(&newi->lock);
 
-        newi->_ctx = GF_CALLOC (1, (sizeof (struct _inode_ctx) *
-                                    table->xl->graph->xl_count),
-                                    gf_common_mt_inode_ctx);
-
-        if (newi->_ctx == NULL) {
-                gf_log ("inode", GF_LOG_ERROR, "out of memory");
-                LOCK_DESTROY (&newi->lock);
-                mem_put (table->inode_pool, newi);
-                newi = NULL;
-                goto out;
-        }
+    INIT_LIST_HEAD(&newi->fd_list);
+    INIT_LIST_HEAD(&newi->list);
+    INIT_LIST_HEAD(&newi->hash);
+    INIT_LIST_HEAD(&newi->dentry_list);
 
-        list_add (&newi->list, &table->lru);
-        table->lru_size++;
+    newi->_ctx = GF_CALLOC(1, (sizeof(struct _inode_ctx) * table->ctxcount),
+                           gf_common_mt_inode_ctx);
+    if (newi->_ctx == NULL) {
+        LOCK_DESTROY(&newi->lock);
+        mem_put(newi);
+        newi = NULL;
+        goto out;
+    }
 
 out:
-
-        return newi;
+    return newi;
 }
 
-
 inode_t *
-inode_new (inode_table_t *table)
+inode_new(inode_table_t *table)
 {
-        inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-        if (!table)
-                return NULL;
+    if (!table) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INODE_TABLE_NOT_FOUND,
+                         "inode not "
+                         "found");
+        return NULL;
+    }
 
-        pthread_mutex_lock (&table->lock);
+    inode = inode_create(table);
+    if (inode) {
+        pthread_mutex_lock(&table->lock);
         {
-                inode = __inode_create (table);
-                if (inode != NULL) {
-                        __inode_ref (inode);
-                }
+            list_add(&inode->list, &table->lru);
+            table->lru_size++;
+            __inode_ref(inode, false);
         }
-        pthread_mutex_unlock (&table->lock);
+        pthread_mutex_unlock(&table->lock);
+    }
 
-        return inode;
+    return inode;
 }
 
-
+/* Reduce the ref count by value 'nref'
+ * Args:
+ * inode - address of the inode to operate on
+ * nref - number to subtracted from inode->ref
+ *        if nref is 0, then the ref count is overwritten 0
+ *
+ * This function may cause the purging of the inode,
+ * hence to be used only in destructor functions and not otherwise.
+ */
 static inode_t *
-__inode_lookup (inode_t *inode)
+__inode_ref_reduce_by_n(inode_t *inode, uint64_t nref)
 {
-        if (!inode)
-                return NULL;
+    uint64_t nlookup = 0;
 
-        inode->nlookup++;
+    GF_ASSERT(inode->ref >= nref);
 
-        return inode;
-}
+    inode->ref -= nref;
+
+    if (!nref)
+        inode->ref = 0;
+
+    if (!inode->ref) {
+        inode->table->active_size--;
 
+        nlookup = GF_ATOMIC_GET(inode->nlookup);
+        if (nlookup)
+            __inode_passivate(inode);
+        else
+            __inode_retire(inode);
+    }
+
+    return inode;
+}
 
 static inode_t *
-__inode_forget (inode_t *inode, uint64_t nlookup)
+inode_forget_atomic(inode_t *inode, uint64_t nlookup)
 {
-        if (!inode)
-                return NULL;
-
-        GF_ASSERT (inode->nlookup >= nlookup);
+    uint64_t inode_lookup = 0;
 
-        inode->nlookup -= nlookup;
+    if (!inode)
+        return NULL;
 
-        if (!nlookup)
-                inode->nlookup = 0;
+    if (nlookup == 0) {
+        GF_ATOMIC_INIT(inode->nlookup, 0);
+    } else {
+        inode_lookup = GF_ATOMIC_FETCH_SUB(inode->nlookup, nlookup);
+        GF_ASSERT(inode_lookup >= nlookup);
+    }
 
-        return inode;
+    return inode;
 }
 
-
 dentry_t *
-__dentry_grep (inode_table_t *table, inode_t *parent, const char *name)
+__dentry_grep(inode_table_t *table, inode_t *parent, const char *name,
+              const int hash)
 {
-        int       hash = 0;
-        dentry_t *dentry = NULL;
-        dentry_t *tmp = NULL;
-
-        if (!table || !name || !parent)
-                return NULL;
-
-        hash = hash_dentry (parent, name, table->hashsize);
-
-        list_for_each_entry (tmp, &table->name_hash[hash], hash) {
-                if (tmp->parent == parent && !strcmp (tmp->name, name)) {
-                        dentry = tmp;
-                        break;
-                }
+    dentry_t *dentry = NULL;
+    dentry_t *tmp = NULL;
+
+    list_for_each_entry(tmp, &table->name_hash[hash], hash)
+    {
+        if (tmp->parent == parent && !strcmp(tmp->name, name)) {
+            dentry = tmp;
+            break;
         }
+    }
 
-        return dentry;
+    return dentry;
 }
 
-
 inode_t *
-inode_grep (inode_table_t *table, inode_t *parent, const char *name)
+inode_grep(inode_table_t *table, inode_t *parent, const char *name)
 {
-        inode_t   *inode = NULL;
-        dentry_t  *dentry = NULL;
-
-        if (!table || !parent || !name)
-                return NULL;
+    inode_t *inode = NULL;
+    dentry_t *dentry = NULL;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                dentry = __dentry_grep (table, parent, name);
+    if (!table || !parent || !name) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "table || parent || name"
+                         " not found");
+        return NULL;
+    }
 
-                if (dentry)
-                        inode = dentry->inode;
+    int hash = hash_dentry(parent, name, table->hashsize);
 
-                if (inode)
-                        __inode_ref (inode);
+    pthread_mutex_lock(&table->lock);
+    {
+        dentry = __dentry_grep(table, parent, name, hash);
+        if (dentry) {
+            inode = dentry->inode;
+            if (inode)
+                __inode_ref(inode, false);
         }
-        pthread_mutex_unlock (&table->lock);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        return inode;
+    return inode;
 }
 
-
 inode_t *
-inode_get (inode_table_t *table, ino_t ino, uint64_t gen)
+inode_resolve(inode_table_t *table, char *path)
 {
-        return NULL;
-}
+    char *tmp = NULL, *bname = NULL, *str = NULL, *saveptr = NULL;
+    inode_t *inode = NULL, *parent = NULL;
+
+    if ((path == NULL) || (table == NULL)) {
+        goto out;
+    }
+
+    parent = inode_ref(table->root);
+    str = tmp = gf_strdup(path);
+    if (str == NULL) {
+        goto out;
+    }
+
+    while (1) {
+        bname = strtok_r(str, "/", &saveptr);
+        if (bname == NULL) {
+            break;
+        }
 
+        if (inode != NULL) {
+            inode_unref(inode);
+        }
 
-static int
-__is_root_gfid (uuid_t gfid)
-{
-        uuid_t  root;
-        int     ret;
+        inode = inode_grep(table, parent, bname);
+        if (inode == NULL) {
+            break;
+        }
 
-        memset (root, 0, 16);
-        root[15] = 1;
+        if (parent != NULL) {
+            inode_unref(parent);
+        }
 
-        ret = uuid_compare (gfid, root);
+        parent = inode_ref(inode);
+        str = NULL;
+    }
 
-        return ret;
+    inode_unref(parent);
+    GF_FREE(tmp);
+out:
+    return inode;
 }
 
+int
+inode_grep_for_gfid(inode_table_t *table, inode_t *parent, const char *name,
+                    uuid_t gfid, ia_type_t *type)
+{
+    inode_t *inode = NULL;
+    dentry_t *dentry = NULL;
+    int ret = -1;
+
+    if (!table || !parent || !name) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "table || parent || name"
+                         " not found");
+        return ret;
+    }
+
+    int hash = hash_dentry(parent, name, table->hashsize);
+
+    pthread_mutex_lock(&table->lock);
+    {
+        dentry = __dentry_grep(table, parent, name, hash);
+        if (dentry) {
+            inode = dentry->inode;
+            if (inode) {
+                gf_uuid_copy(gfid, inode->gfid);
+                *type = inode->ia_type;
+                ret = 0;
+            }
+        }
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    return ret;
+}
 
-inode_t *
-__inode_find (inode_table_t *table, uuid_t gfid)
+/* return 1 if gfid is of root, 0 if not */
+gf_boolean_t
+__is_root_gfid(uuid_t gfid)
 {
-        inode_t   *inode = NULL;
-        inode_t   *tmp = NULL;
-        int        hash = 0;
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        if (!table)
-                goto out;
+    if (gf_uuid_compare(gfid, root) == 0)
+        return _gf_true;
 
-        if (__is_root_gfid (gfid) == 0)
-                return table->root;
+    return _gf_false;
+}
 
-        hash = hash_gfid (gfid, 65536);
+inode_t *
+__inode_find(inode_table_t *table, uuid_t gfid, const int hash)
+{
+    inode_t *inode = NULL;
+    inode_t *tmp = NULL;
 
-        list_for_each_entry (tmp, &table->inode_hash[hash], hash) {
-                if (uuid_compare (tmp->gfid, gfid) == 0) {
-                        inode = tmp;
-                        break;
-                }
+    if (__is_root_gfid(gfid))
+        return table->root;
+
+    list_for_each_entry(tmp, &table->inode_hash[hash], hash)
+    {
+        if (gf_uuid_compare(tmp->gfid, gfid) == 0) {
+            inode = tmp;
+            break;
         }
+    }
 
-out:
-        return inode;
+    return inode;
 }
 
-
 inode_t *
-inode_find (inode_table_t *table, uuid_t gfid)
+inode_find(inode_table_t *table, uuid_t gfid)
 {
-        inode_t   *inode = NULL;
+    inode_t *inode = NULL;
 
-        if (!table)
-                return NULL;
+    if (!table) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                         LG_MSG_INODE_TABLE_NOT_FOUND,
+                         "table not "
+                         "found");
+        return NULL;
+    }
 
-        pthread_mutex_lock (&table->lock);
-        {
-                inode = __inode_find (table, gfid);
-                if (inode)
-                        __inode_ref (inode);
-        }
-        pthread_mutex_unlock (&table->lock);
+    int hash = hash_gfid(gfid, 65536);
 
-        return inode;
-}
+    pthread_mutex_lock(&table->lock);
+    {
+        inode = __inode_find(table, gfid, hash);
+        if (inode)
+            __inode_ref(inode, false);
+    }
+    pthread_mutex_unlock(&table->lock);
 
+    return inode;
+}
 
 static inode_t *
-__inode_link (inode_t *inode, inode_t *parent, const char *name,
-              struct iatt *iatt)
+__inode_link(inode_t *inode, inode_t *parent, const char *name,
+             struct iatt *iatt, const int dhash)
 {
-        dentry_t      *dentry = NULL;
-        dentry_t      *old_dentry = NULL;
-        inode_t       *old_inode = NULL;
-        inode_table_t *table = NULL;
-        inode_t       *link_inode = NULL;
+    dentry_t *dentry = NULL;
+    dentry_t *old_dentry = NULL;
+    inode_t *old_inode = NULL;
+    inode_table_t *table = NULL;
+    inode_t *link_inode = NULL;
+    char link_uuid_str[64] = {0}, parent_uuid_str[64] = {0};
+
+    table = inode->table;
+
+    if (parent) {
+        /* We should prevent inode linking between different
+           inode tables. This can cause errors which is very
+           hard to catch/debug. */
+        if (inode->table != parent->table) {
+            errno = EINVAL;
+            GF_ASSERT(!"link attempted b/w inodes of diff table");
+        }
 
-        if (!inode)
-                return NULL;
+        if (parent->ia_type != IA_IFDIR) {
+            errno = EINVAL;
+            GF_ASSERT(!"link attempted on non-directory parent");
+            return NULL;
+        }
 
-        table = inode->table;
-        if (!table)
-                return NULL;
+        if (!name || strlen(name) == 0) {
+            errno = EINVAL;
+            GF_ASSERT (!"link attempted with no basename on "
+                                    "parent");
+            return NULL;
+        }
+    }
 
-        link_inode = inode;
+    link_inode = inode;
 
-        if (!__is_inode_hashed (inode)) {
-                if (!iatt)
-                        return NULL;
+    if (!__is_inode_hashed(inode)) {
+        if (!iatt) {
+            errno = EINVAL;
+            return NULL;
+        }
 
-                if (uuid_is_null (iatt->ia_gfid))
-                        return NULL;
+        if (gf_uuid_is_null(iatt->ia_gfid)) {
+            errno = EINVAL;
+            return NULL;
+        }
 
-                uuid_copy (inode->gfid, iatt->ia_gfid);
-                inode->ino        = iatt->ia_ino;
-                inode->ia_type    = iatt->ia_type;
+        int ihash = hash_gfid(iatt->ia_gfid, 65536);
 
-                old_inode = __inode_find (table, inode->gfid);
+        old_inode = __inode_find(table, iatt->ia_gfid, ihash);
 
-                if (old_inode) {
-                        link_inode = old_inode;
-                } else {
-                        __inode_hash (inode);
-                }
+        if (old_inode) {
+            link_inode = old_inode;
+        } else {
+            gf_uuid_copy(inode->gfid, iatt->ia_gfid);
+            inode->ia_type = iatt->ia_type;
+            __inode_hash(inode, ihash);
         }
+    } else {
+        /* @old_inode serves another important purpose - it indicates
+           to the code further below whether a dentry cycle check is
+           required or not (a new inode linkage can never result in
+           creation of a loop.)
+
+           if the given @inode is already hashed, it actually means
+           it is an "old" inode and deserves to undergo the cyclic
+           check.
+        */
+        old_inode = inode;
+    }
+
+    if (name && (!strcmp(name, ".") || !strcmp(name, ".."))) {
+        return link_inode;
+    }
+
+    /* use only link_inode beyond this point */
+    if (parent) {
+        old_dentry = __dentry_grep(table, parent, name, dhash);
+
+        if (!old_dentry || old_dentry->inode != link_inode) {
+            dentry = dentry_create(link_inode, parent, name);
+            if (!dentry) {
+                gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0,
+                                 LG_MSG_DENTRY_CREATE_FAILED,
+                                 "dentry create failed on "
+                                 "inode %s with parent %s",
+                                 uuid_utoa_r(link_inode->gfid, link_uuid_str),
+                                 uuid_utoa_r(parent->gfid, parent_uuid_str));
+                errno = ENOMEM;
+                return NULL;
+            }
 
-        /* use only link_inode beyond this point */
-        if (parent) {
-                old_dentry = __dentry_grep (table, parent, name);
+            /* dentry linking needs to happen inside lock */
+            dentry->parent = __inode_ref(parent, false);
+            list_add(&dentry->inode_list, &link_inode->dentry_list);
 
-                if (!old_dentry || old_dentry->inode != link_inode) {
-                        dentry = __dentry_create (link_inode, parent, name);
-                        if (old_inode && __is_dentry_cyclic (dentry)) {
-                                __dentry_unset (dentry);
-                                return NULL;
-                        }
-                        __dentry_hash (dentry);
+            if (old_inode && __is_dentry_cyclic(dentry)) {
+                errno = ELOOP;
+                dentry_destroy(__dentry_unset(dentry));
+                return NULL;
+            }
+            __dentry_hash(dentry, dhash);
 
-                        if (old_dentry)
-                                __dentry_unset (old_dentry);
-                }
+            if (old_dentry)
+                dentry_destroy(__dentry_unset(old_dentry));
         }
+    }
 
-        return link_inode;
+    return link_inode;
 }
 
-
 inode_t *
-inode_link (inode_t *inode, inode_t *parent, const char *name,
-            struct iatt *iatt)
+inode_link(inode_t *inode, inode_t *parent, const char *name, struct iatt *iatt)
 {
-        inode_table_t *table = NULL;
-        inode_t       *linked_inode = NULL;
+    int hash = 0;
+    inode_table_t *table = NULL;
+    inode_t *linked_inode = NULL;
 
-        if (!inode)
-                return NULL;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return NULL;
+    }
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                linked_inode = __inode_link (inode, parent, name, iatt);
+    if (parent && name) {
+        hash = hash_dentry(parent, name, table->hashsize);
+    }
 
-                if (linked_inode)
-                        __inode_ref (linked_inode);
-        }
-        pthread_mutex_unlock (&table->lock);
+    if (name && strchr(name, '/')) {
+        GF_ASSERT(!"inode link attempted with '/' in name");
+        return NULL;
+    }
 
-        inode_table_prune (table);
+    pthread_mutex_lock(&table->lock);
+    {
+        linked_inode = __inode_link(inode, parent, name, iatt, hash);
+        if (linked_inode)
+            __inode_ref(linked_inode, false);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        return linked_inode;
+    inode_table_prune(table);
+
+    return linked_inode;
 }
 
+int
+inode_lookup(inode_t *inode)
+{
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
+
+    GF_ATOMIC_INC(inode->nlookup);
+
+    return 0;
+}
 
 int
-inode_lookup (inode_t *inode)
+inode_ref_reduce_by_n(inode_t *inode, uint64_t nref)
 {
-        inode_table_t *table = NULL;
+    inode_table_t *table = NULL;
 
-        if (!inode)
-                return -1;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                __inode_lookup (inode);
-        }
-        pthread_mutex_unlock (&table->lock);
+    pthread_mutex_lock(&table->lock);
+    {
+        __inode_ref_reduce_by_n(inode, nref);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        return 0;
-}
+    inode_table_prune(table);
 
+    return 0;
+}
 
 int
-inode_forget (inode_t *inode, uint64_t nlookup)
+inode_forget(inode_t *inode, uint64_t nlookup)
 {
-        inode_table_t *table = NULL;
+    inode_table_t *table = NULL;
 
-        if (!inode)
-                return -1;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                __inode_forget (inode, nlookup);
-        }
-        pthread_mutex_unlock (&table->lock);
+    inode_forget_atomic(inode, nlookup);
 
-        inode_table_prune (table);
+    inode_table_prune(table);
 
-        return 0;
+    return 0;
 }
 
+int
+inode_forget_with_unref(inode_t *inode, uint64_t nlookup)
+{
+    inode_table_t *table = NULL;
 
-static void
-__inode_unlink (inode_t *inode, inode_t *parent, const char *name)
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
+
+    table = inode->table;
+
+    pthread_mutex_lock(&table->lock);
+    {
+        inode_forget_atomic(inode, nlookup);
+        __inode_unref(inode, true);
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    inode_table_prune(table);
+
+    return 0;
+}
+
+/*
+ * Invalidate an inode. This is invoked when a translator decides that an
+ * inode's cache is no longer valid. Any translator interested in taking action
+ * in this situation can define the invalidate callback.
+ */
+int
+inode_invalidate(inode_t *inode)
 {
-        dentry_t *dentry = NULL;
+    int ret = 0;
+    xlator_t *xl = NULL;
+    xlator_t *old_THIS = NULL;
+
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
+
+    /*
+     * The master xlator is not in the graph but it can define an invalidate
+     * handler.
+     */
+    xl = inode->table->xl->ctx->master;
+    if (xl && xl->cbks->invalidate) {
+        old_THIS = THIS;
+        THIS = xl;
+        ret = xl->cbks->invalidate(xl, inode);
+        THIS = old_THIS;
+        if (ret)
+            return ret;
+    }
 
-        if (!inode || !parent || !name)
-                return;
+    xl = inode->table->xl->graph->first;
+    while (xl) {
+        old_THIS = THIS;
+        THIS = xl;
+        if (xl->cbks->invalidate)
+            ret = xl->cbks->invalidate(xl, inode);
+        THIS = old_THIS;
 
-        dentry = __dentry_search_for_inode (inode, parent->ino, name);
+        if (ret)
+            break;
 
-        /* dentry NULL for corrupted backend */
-        if (dentry)
-                __dentry_unset (dentry);
+        xl = xl->next;
+    }
+
+    return ret;
 }
 
+static dentry_t *
+__inode_unlink(inode_t *inode, inode_t *parent, const char *name)
+{
+    dentry_t *dentry = NULL;
+    char pgfid[64] = {0};
+    char gfid[64] = {0};
+
+    dentry = __dentry_search_for_inode(inode, parent->gfid, name);
+
+    /* dentry NULL for corrupted backend */
+    if (dentry) {
+        dentry = __dentry_unset(dentry);
+    } else {
+        gf_smsg("inode", GF_LOG_WARNING, 0, LG_MSG_DENTRY_NOT_FOUND,
+                "parent-gfid=%s name=%s gfid%s",
+                uuid_utoa_r(parent->gfid, pgfid), name,
+                uuid_utoa_r(inode->gfid, gfid), NULL);
+    }
+
+    return dentry;
+}
 
 void
-inode_unlink (inode_t *inode, inode_t *parent, const char *name)
+inode_unlink(inode_t *inode, inode_t *parent, const char *name)
 {
-        inode_table_t *table = NULL;
+    inode_table_t *table;
+    dentry_t *dentry;
 
-        if (!inode)
-                return;
+    if (!inode || !parent || !name)
+        return;
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                __inode_unlink (inode, parent, name);
-        }
-        pthread_mutex_unlock (&table->lock);
+    pthread_mutex_lock(&table->lock);
+    {
+        dentry = __inode_unlink(inode, parent, name);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        inode_table_prune (table);
-}
+    dentry_destroy(dentry);
 
+    inode_table_prune(table);
+}
 
 int
-inode_rename (inode_table_t *table, inode_t *srcdir, const char *srcname,
-              inode_t *dstdir, const char *dstname, inode_t *inode,
-              struct iatt *iatt)
+inode_rename(inode_table_t *table, inode_t *srcdir, const char *srcname,
+             inode_t *dstdir, const char *dstname, inode_t *inode,
+             struct iatt *iatt)
 {
-        if (!inode)
-                return -1;
+    int hash = 0;
+    dentry_t *dentry = NULL;
 
-        table = inode->table;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return -1;
+    }
 
-        pthread_mutex_lock (&table->lock);
-        {
-                __inode_link (inode, dstdir, dstname, iatt);
-                __inode_unlink (inode, srcdir, srcname);
-        }
-        pthread_mutex_unlock (&table->lock);
+    table = inode->table;
 
-        inode_table_prune (table);
+    if (dstname && strchr(dstname, '/')) {
+        GF_ASSERT(!"inode link attempted with '/' in name");
+        return -1;
+    }
 
-        return 0;
-}
+    if (dstdir && dstname) {
+        hash = hash_dentry(dstdir, dstname, table->hashsize);
+    }
+
+    pthread_mutex_lock(&table->lock);
+    {
+        __inode_link(inode, dstdir, dstname, iatt, hash);
+        /* pick the old dentry */
+        dentry = __inode_unlink(inode, srcdir, srcname);
+    }
+    pthread_mutex_unlock(&table->lock);
 
+    /* free the old dentry */
+    dentry_destroy(dentry);
+
+    inode_table_prune(table);
+
+    return 0;
+}
 
 static dentry_t *
-__dentry_search_arbit (inode_t *inode)
+__dentry_search_arbit(inode_t *inode)
 {
-        dentry_t *dentry = NULL;
-        dentry_t *trav = NULL;
+    dentry_t *dentry = NULL;
+    dentry_t *trav = NULL;
 
-        if (!inode)
-                return NULL;
+    if (!inode)
+        return NULL;
 
-        list_for_each_entry (trav, &inode->dentry_list, inode_list) {
-                if (__is_dentry_hashed (trav)) {
-                        dentry = trav;
-                        break;
-                }
+    list_for_each_entry(trav, &inode->dentry_list, inode_list)
+    {
+        if (__is_dentry_hashed(trav)) {
+            dentry = trav;
+            break;
         }
+    }
 
-        if (!dentry) {
-                list_for_each_entry (trav, &inode->dentry_list, inode_list) {
-                        dentry = trav;
-                        break;
-                }
+    if (!dentry) {
+        list_for_each_entry(trav, &inode->dentry_list, inode_list)
+        {
+            dentry = trav;
+            break;
         }
+    }
 
-        return dentry;
+    return dentry;
 }
 
-
 inode_t *
-inode_parent (inode_t *inode, ino_t par, const char *name)
+inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
 {
-        inode_t       *parent = NULL;
-        inode_table_t *table = NULL;
-        dentry_t      *dentry = NULL;
+    inode_t *parent = NULL;
+    inode_table_t *table = NULL;
+    dentry_t *dentry = NULL;
 
-        if (!inode)
-                return NULL;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return NULL;
+    }
 
-        table = inode->table;
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                if (par && name) {
-                        dentry = __dentry_search_for_inode (inode, par, name);
-                } else {
-                        dentry = __dentry_search_arbit (inode);
-                }
+    pthread_mutex_lock(&table->lock);
+    {
+        if (pargfid && !gf_uuid_is_null(pargfid) && name) {
+            dentry = __dentry_search_for_inode(inode, pargfid, name);
+        } else {
+            dentry = __dentry_search_arbit(inode);
+        }
 
-                if (dentry)
-                        parent = dentry->parent;
+        if (dentry)
+            parent = dentry->parent;
 
-                if (parent)
-                        __inode_ref (parent);
-        }
-        pthread_mutex_unlock (&table->lock);
+        if (parent)
+            __inode_ref(parent, false);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-        return parent;
+    return parent;
 }
 
+static int
+__inode_has_dentry(inode_t *inode)
+{
+    return !list_empty(&inode->dentry_list);
+}
 
 int
-inode_path (inode_t *inode, const char *name, char **bufp)
+inode_has_dentry(inode_t *inode)
 {
-        inode_table_t *table = NULL;
-        dentry_t      *trav = NULL;
-        size_t         i = 0, size = 0;
-        int64_t        ret = 0;
-        int            len = 0;
-        char          *buf = NULL;
+    int dentry_present = 0;
 
-        if (!inode)
-                return -1;
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return 0;
+    }
 
-        table = inode->table;
+    LOCK(&inode->lock);
+    {
+        dentry_present = __inode_has_dentry(inode);
+    }
+    UNLOCK(&inode->lock);
 
-        pthread_mutex_lock (&table->lock);
-        {
-                for (trav = __dentry_search_arbit (inode); trav;
-                     trav = __dentry_search_arbit (trav->parent)) {
-                        i ++; /* "/" */
-                        i += strlen (trav->name);
-                        if (i > PATH_MAX) {
-                                gf_log (table->name, GF_LOG_CRITICAL,
-                                        "possible infinite loop detected, "
-                                        "forcing break. name=(%s)", name);
-                                ret = -ENOENT;
-                                goto unlock;
-                        }
-                }
+    return dentry_present;
+}
 
-                if ((inode->ino != 1) &&
-                    (i == 0)) {
-                        gf_log (table->name, GF_LOG_DEBUG,
-                                "no dentry for non-root inode %"PRId64,
-                                inode->ino);
-                        ret = -ENOENT;
-                        goto unlock;
-                }
+int
+__inode_path(inode_t *inode, const char *name, char **bufp)
+{
+    inode_table_t *table = NULL;
+    inode_t *itrav = NULL;
+    dentry_t *trav = NULL;
+    size_t i = 0, size = 0;
+    int64_t ret = 0;
+    int len = 0;
+    char *buf = NULL;
+
+    if (!inode || gf_uuid_is_null(inode->gfid)) {
+        GF_ASSERT(0);
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, EINVAL, LG_MSG_INVALID_ARG,
+                         "invalid inode");
+        return -EINVAL;
+    }
+
+    table = inode->table;
+
+    itrav = inode;
+    for (trav = __dentry_search_arbit(itrav); trav;
+         trav = __dentry_search_arbit(itrav)) {
+        itrav = trav->parent;
+        i++; /* "/" */
+        i += strlen(trav->name);
+        if (i > PATH_MAX) {
+            gf_smsg(table->name, GF_LOG_CRITICAL, 0, LG_MSG_DENTRY_CYCLIC_LOOP,
+                    "name=%s", name, NULL);
+            ret = -ENOENT;
+            goto out;
+        }
+    }
+
+    if (!__is_root_gfid(itrav->gfid)) {
+        /* "<gfid:00000000-0000-0000-0000-000000000000>"/path */
+        i += GFID_STR_PFX_LEN;
+    }
+
+    if (name) {
+        i++;
+        i += strlen(name);
+    }
+
+    ret = i;
+    size = i + 1;
+    buf = GF_CALLOC(size, sizeof(char), gf_common_mt_char);
+    if (buf) {
+        buf[size - 1] = 0;
+
+        if (name) {
+            len = strlen(name);
+            memcpy(buf + (i - len), name, len);
+            buf[i - len - 1] = '/';
+            i -= (len + 1);
+        }
 
-                if (name) {
-                        i++;
-                        i += strlen (name);
-                }
+        itrav = inode;
+        for (trav = __dentry_search_arbit(itrav); trav;
+             trav = __dentry_search_arbit(itrav)) {
+            itrav = trav->parent;
+            len = strlen(trav->name);
+            memcpy(buf + (i - len), trav->name, len);
+            buf[i - len - 1] = '/';
+            i -= (len + 1);
+        }
 
-                ret = i;
-                size = i + 1;
-                buf = GF_CALLOC (size, sizeof (char), gf_common_mt_char);
-                if (buf) {
-
-                        buf[size - 1] = 0;
-
-                        if (name) {
-                                len = strlen (name);
-                                strncpy (buf + (i - len), name, len);
-                                buf[i-len-1] = '/';
-                                i -= (len + 1);
-                        }
-
-                        for (trav = __dentry_search_arbit (inode); trav;
-                             trav = __dentry_search_arbit (trav->parent)) {
-                                len = strlen (trav->name);
-                                strncpy (buf + (i - len), trav->name, len);
-                                buf[i-len-1] = '/';
-                                i -= (len + 1);
-                        }
-                        *bufp = buf;
-                } else {
-                        gf_log (table->name, GF_LOG_ERROR,
-                                "out of memory");
-                        ret = -ENOMEM;
-                }
+        if (!__is_root_gfid(itrav->gfid)) {
+            snprintf(&buf[i - GFID_STR_PFX_LEN], GFID_STR_PFX_LEN,
+                     INODE_PATH_FMT, uuid_utoa(itrav->gfid));
+            buf[i - 1] = '>';
         }
-unlock:
-        pthread_mutex_unlock (&table->lock);
 
-        if (inode->ino == 1 && !name) {
-                ret = 1;
-                if (buf) {
-                        GF_FREE (buf);
-                }
-                buf = GF_CALLOC (ret + 1, sizeof (char), gf_common_mt_char);
-                if (buf) {
-                        strcpy (buf, "/");
-                        *bufp = buf;
-                } else {
-                        gf_log (table->name, GF_LOG_ERROR,
-                                "out of memory");
-                        ret = -ENOMEM;
-                }
+        *bufp = buf;
+    } else {
+        ret = -ENOMEM;
+    }
+
+out:
+    if (__is_root_gfid(inode->gfid) && !name) {
+        ret = 1;
+        GF_FREE(buf);
+        buf = GF_CALLOC(ret + 1, sizeof(char), gf_common_mt_char);
+        if (buf) {
+            strcpy(buf, "/");
+            *bufp = buf;
+        } else {
+            ret = -ENOMEM;
         }
+    }
 
-        return ret;
+    if (ret < 0)
+        *bufp = NULL;
+    return ret;
 }
 
-static int
-inode_table_prune (inode_table_t *table)
+int
+inode_path(inode_t *inode, const char *name, char **bufp)
 {
-        int               ret = 0;
-        struct list_head  purge = {0, };
-        inode_t          *del = NULL;
-        inode_t          *tmp = NULL;
-        inode_t          *entry = NULL;
+    inode_table_t *table = NULL;
+    int ret = -1;
 
-        if (!table)
-                return -1;
+    if (!inode)
+        return -EINVAL;
 
-        INIT_LIST_HEAD (&purge);
+    table = inode->table;
 
-        pthread_mutex_lock (&table->lock);
-        {
-                while (table->lru_limit
-                       && table->lru_size > (table->lru_limit)) {
+    pthread_mutex_lock(&table->lock);
+    {
+        ret = __inode_path(inode, name, bufp);
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    return ret;
+}
+
+void
+__inode_table_set_lru_limit(inode_table_t *table, uint32_t lru_limit)
+{
+    table->lru_limit = lru_limit;
+    return;
+}
+
+void
+inode_table_set_lru_limit(inode_table_t *table, uint32_t lru_limit)
+{
+    pthread_mutex_lock(&table->lock);
+    {
+        __inode_table_set_lru_limit(table, lru_limit);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-                        entry = list_entry (table->lru.next, inode_t, list);
+    inode_table_prune(table);
 
-                        table->lru_size--;
-                        __inode_retire (entry);
+    return;
+}
 
-                        ret++;
+static int
+inode_table_prune(inode_table_t *table)
+{
+    int ret = 0;
+    int ret1 = 0;
+    struct list_head purge = {
+        0,
+    };
+    inode_t *del = NULL;
+    inode_t *tmp = NULL;
+    inode_t *entry = NULL;
+    uint64_t nlookup = 0;
+    int64_t lru_size = 0;
+
+    if (!table)
+        return -1;
+
+    INIT_LIST_HEAD(&purge);
+
+    pthread_mutex_lock(&table->lock);
+    {
+        if (!table->lru_limit)
+            goto purge_list;
+
+        lru_size = table->lru_size;
+        while (lru_size > (table->lru_limit)) {
+            if (list_empty(&table->lru)) {
+                gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                                 LG_MSG_INVALID_INODE_LIST,
+                                 "Empty inode lru list found"
+                                 " but with (%d) lru_size",
+                                 table->lru_size);
+                break;
+            }
+
+            lru_size--;
+            entry = list_entry(table->lru.next, inode_t, list);
+            /* The logic of invalidation is required only if invalidator_fn
+               is present */
+            if (table->invalidator_fn) {
+                /* check for valid inode with 'nlookup' */
+                nlookup = GF_ATOMIC_GET(entry->nlookup);
+                if (nlookup) {
+                    if (entry->invalidate_sent) {
+                        list_move_tail(&entry->list, &table->lru);
+                        continue;
+                    }
+                    __inode_ref(entry, true);
+                    tmp = entry;
+                    break;
                 }
+            }
 
-                list_splice_init (&table->purge, &purge);
-                table->purge_size = 0;
+            table->lru_size--;
+            __inode_retire(entry);
+            ret++;
         }
-        pthread_mutex_unlock (&table->lock);
 
+    purge_list:
+        list_splice_init(&table->purge, &purge);
+        table->purge_size = 0;
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    /* Pick 1 inode for invalidation */
+    if (tmp) {
+        xlator_t *old_THIS = THIS;
+        THIS = table->invalidator_xl;
+        ret1 = table->invalidator_fn(table->invalidator_xl, tmp);
+        THIS = old_THIS;
+        pthread_mutex_lock(&table->lock);
         {
-                list_for_each_entry_safe (del, tmp, &purge, list) {
-                        list_del_init (&del->list);
-                        __inode_forget (del, 0);
-                        __inode_destroy (del);
-                }
+            if (!ret1) {
+                tmp->invalidate_sent = true;
+                __inode_unref(tmp, false);
+            } else {
+                /* Move this back to the lru list*/
+                __inode_unref(tmp, true);
+            }
         }
-
-        return ret;
+        pthread_mutex_unlock(&table->lock);
+    }
+
+    /* Just so that if purge list is handled too, then clear it off */
+    list_for_each_entry_safe(del, tmp, &purge, list)
+    {
+        list_del_init(&del->list);
+        inode_forget_atomic(del, 0);
+        __inode_destroy(del);
+    }
+
+    return ret;
 }
 
-
 static void
-__inode_table_init_root (inode_table_t *table)
+__inode_table_init_root(inode_table_t *table)
 {
-        inode_t     *root = NULL;
-        struct iatt  iatt = {0, };
+    inode_t *root = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    if (!table)
+        return;
 
-        if (!table)
-                return;
+    root = inode_create(table);
 
-        root = __inode_create (table);
+    list_add(&root->list, &table->lru);
+    table->lru_size++;
 
-        iatt.ia_gfid[15] = 1;
-        iatt.ia_ino = 1;
-        iatt.ia_type = IA_IFDIR;
+    iatt.ia_gfid[15] = 1;
+    iatt.ia_ino = 1;
+    iatt.ia_type = IA_IFDIR;
 
-        table->root = root;
-        __inode_link (root, NULL, NULL, &iatt);
+    __inode_link(root, NULL, NULL, &iatt, 0);
+    table->root = root;
 }
 
-
 inode_table_t *
-inode_table_new (size_t lru_limit, xlator_t *xl)
+inode_table_with_invalidator(uint32_t lru_limit, xlator_t *xl,
+                             int32_t (*invalidator_fn)(xlator_t *, inode_t *),
+                             xlator_t *invalidator_xl)
 {
-        inode_table_t *new = NULL;
-        int            ret = 0;
-        int            i = 0;
+    inode_table_t *new = NULL;
+    uint32_t mem_pool_size = lru_limit;
+    int ret = -1;
+    int i = 0;
 
-        new = (void *)GF_CALLOC(1, sizeof (*new), gf_common_mt_inode_table_t);
-        if (!new)
-                return NULL;
+    new = (void *)GF_CALLOC(1, sizeof(*new), gf_common_mt_inode_table_t);
+    if (!new)
+        return NULL;
 
-        new->xl = xl;
+    new->xl = xl;
+    new->ctxcount = xl->graph->xl_count + 1;
 
-        new->lru_limit = lru_limit;
+    new->lru_limit = lru_limit;
+    new->invalidator_fn = invalidator_fn;
+    new->invalidator_xl = invalidator_xl;
 
-        new->hashsize = 14057; /* TODO: Random Number?? */
+    new->hashsize = 14057; /* TODO: Random Number?? */
 
-        /* In case FUSE is initing the inode table. */
-        if (lru_limit == 0)
-                lru_limit = DEFAULT_INODE_MEMPOOL_ENTRIES;
+    /* In case FUSE is initing the inode table. */
+    if (!mem_pool_size || (mem_pool_size > DEFAULT_INODE_MEMPOOL_ENTRIES))
+        mem_pool_size = DEFAULT_INODE_MEMPOOL_ENTRIES;
 
-        new->inode_pool = mem_pool_new (inode_t, lru_limit);
+    new->inode_pool = mem_pool_new(inode_t, mem_pool_size);
+    if (!new->inode_pool)
+        goto out;
 
-        if (!new->inode_pool) {
-                GF_FREE (new);
-                return NULL;
+    new->dentry_pool = mem_pool_new(dentry_t, mem_pool_size);
+    if (!new->dentry_pool)
+        goto out;
+
+    new->inode_hash = (void *)GF_CALLOC(65536, sizeof(struct list_head),
+                                        gf_common_mt_list_head);
+    if (!new->inode_hash)
+        goto out;
+
+    new->name_hash = (void *)GF_CALLOC(new->hashsize, sizeof(struct list_head),
+                                       gf_common_mt_list_head);
+    if (!new->name_hash)
+        goto out;
+
+    /* if number of fd open in one process is more than this,
+       we may hit perf issues */
+    new->fd_mem_pool = mem_pool_new(fd_t, 1024);
+
+    if (!new->fd_mem_pool)
+        goto out;
+
+    for (i = 0; i < 65536; i++) {
+        INIT_LIST_HEAD(&new->inode_hash[i]);
+    }
+
+    for (i = 0; i < new->hashsize; i++) {
+        INIT_LIST_HEAD(&new->name_hash[i]);
+    }
+
+    INIT_LIST_HEAD(&new->active);
+    INIT_LIST_HEAD(&new->lru);
+    INIT_LIST_HEAD(&new->purge);
+    INIT_LIST_HEAD(&new->invalidate);
+
+    ret = gf_asprintf(&new->name, "%s/inode", xl->name);
+    if (-1 == ret) {
+        /* TODO: This should be ok to continue, check with avati */
+        ;
+    }
+
+    new->cleanup_started = _gf_false;
+
+    __inode_table_init_root(new);
+
+    pthread_mutex_init(&new->lock, NULL);
+
+    ret = 0;
+out:
+    if (ret) {
+        if (new) {
+            GF_FREE(new->inode_hash);
+            GF_FREE(new->name_hash);
+            if (new->dentry_pool)
+                mem_pool_destroy(new->dentry_pool);
+            if (new->inode_pool)
+                mem_pool_destroy(new->inode_pool);
+            GF_FREE(new);
+            new = NULL;
         }
+    }
 
-        new->dentry_pool = mem_pool_new (dentry_t, lru_limit);
+    return new;
+}
 
-        if (!new->dentry_pool) {
-                GF_FREE (new);
-                return NULL;
+inode_table_t *
+inode_table_new(uint32_t lru_limit, xlator_t *xl)
+{
+    /* Only fuse for now requires the inode table with invalidator */
+    return inode_table_with_invalidator(lru_limit, xl, NULL, NULL);
+}
+
+int
+inode_table_ctx_free(inode_table_t *table)
+{
+    int ret = 0;
+    inode_t *del = NULL;
+    inode_t *tmp = NULL;
+    int purge_count = 0;
+    int lru_count = 0;
+    int active_count = 0;
+    xlator_t *this = NULL;
+    int itable_size = 0;
+
+    if (!table)
+        return -1;
+
+    this = THIS;
+
+    pthread_mutex_lock(&table->lock);
+    {
+        list_for_each_entry_safe(del, tmp, &table->purge, list)
+        {
+            if (del->_ctx) {
+                __inode_ctx_free(del);
+                purge_count++;
+            }
         }
 
-        new->inode_hash = (void *)GF_CALLOC (65536,
-                                             sizeof (struct list_head),
-                                             gf_common_mt_list_head);
-        if (!new->inode_hash) {
-                GF_FREE (new);
-                return NULL;
+        list_for_each_entry_safe(del, tmp, &table->lru, list)
+        {
+            if (del->_ctx) {
+                __inode_ctx_free(del);
+                lru_count++;
+            }
         }
 
-        new->name_hash = (void *)GF_CALLOC (new->hashsize,
-                                            sizeof (struct list_head),
-                                            gf_common_mt_list_head);
-        if (!new->name_hash) {
-                GF_FREE (new->inode_hash);
-                GF_FREE (new);
-                return NULL;
+        /* should the contexts of active inodes be freed?
+         * Since before this function being called fds would have
+         * been migrated and would have held the ref on the new
+         * inode from the new inode table, the older inode would not
+         * be used.
+         */
+        list_for_each_entry_safe(del, tmp, &table->active, list)
+        {
+            if (del->_ctx) {
+                __inode_ctx_free(del);
+                active_count++;
+            }
         }
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    ret = purge_count + lru_count + active_count;
+    itable_size = table->active_size + table->lru_size + table->purge_size;
+    gf_msg_callingfn(this->name, GF_LOG_INFO, 0, LG_MSG_INODE_CONTEXT_FREED,
+                     "total %d (itable size: "
+                     "%d) inode contexts have been freed (active: %d, ("
+                     "active size: %d), lru: %d, (lru size: %d),  purge: "
+                     "%d, (purge size: %d))",
+                     ret, itable_size, active_count, table->active_size,
+                     lru_count, table->lru_size, purge_count,
+                     table->purge_size);
+    return ret;
+}
+
+void
+inode_table_destroy_all(glusterfs_ctx_t *ctx)
+{
+    glusterfs_graph_t *trav_graph = NULL, *tmp = NULL;
+    xlator_t *tree = NULL;
+    inode_table_t *inode_table = NULL;
+
+    if (ctx == NULL)
+        goto out;
+
+    /* TODO: Traverse ctx->graphs with in ctx->lock and also the other
+     * graph additions and traversals in ctx->lock.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        tree = trav_graph->first;
+        inode_table = tree->itable;
+        tree->itable = NULL;
+        if (inode_table)
+            inode_table_destroy(inode_table);
+    }
+out:
+    return;
+}
 
-	 new->fd_mem_pool = mem_pool_new (fd_t, 16384);
+void
+inode_table_destroy(inode_table_t *inode_table)
+{
+    inode_t *trav = NULL;
 
-	 if (!new->fd_mem_pool) {
-		  GF_FREE (new->inode_hash);
-		  GF_FREE (new);
-	 }
+    if (inode_table == NULL)
+        return;
 
-        for (i = 0; i < 65536; i++) {
-                INIT_LIST_HEAD (&new->inode_hash[i]);
+    /* Ideally at this point in time, there should be no inodes with
+     * refs remaining. But there are quite a few chances where the inodes
+     * leak. So we can take three approaches for cleaning up the inode table:
+     * 1. Assume there are no leaks and then send a forget on all the inodes
+     *    in lru list.(If no leaks there should be no inodes in active list)
+     * 2. Knowing there could be leaks and not freeing those inodes will
+     *    also not free its inode context and this could leak a lot of
+     *    memory, force free the inodes by changing the ref to 0.
+     *    The problem with this is that any reference to inode after this
+     *    calling this function will lead to a crash.
+     * 3. Knowing there could be leakes, just free the inode contexts of
+     *    all the inodes. and let the inodes be alive. This way the major
+     *    memory consumed by the inode contexts are freed, but there can
+     *    be errors when any inode contexts are accessed after destroying
+     *    this table.
+     *
+     * Not sure which is the approach to be taken, going by approach 2.
+     */
+
+    /* Approach 3:
+     * ret = inode_table_ctx_free (inode_table);
+     */
+    pthread_mutex_lock(&inode_table->lock);
+    {
+        inode_table->cleanup_started = _gf_true;
+        /* Process lru list first as we need to unset their dentry
+         * entries (the ones which may not be unset during
+         * '__inode_passivate' as they were hashed) which in turn
+         * shall unref their parent
+         *
+         * These parent inodes when unref'ed may well again fall
+         * into lru list and if we are at the end of traversing
+         * the list, we may miss to delete/retire that entry. Hence
+         * traverse the lru list till it gets empty.
+         */
+        while (!list_empty(&inode_table->lru)) {
+            trav = list_first_entry(&inode_table->lru, inode_t, list);
+            inode_forget_atomic(trav, 0);
+            __inode_retire(trav);
+            inode_table->lru_size--;
         }
 
+        /* Same logic for invalidate list */
+        while (!list_empty(&inode_table->invalidate)) {
+            trav = list_first_entry(&inode_table->invalidate, inode_t, list);
+            inode_forget_atomic(trav, 0);
+            __inode_retire(trav);
+            inode_table->invalidate_size--;
+        }
 
-        for (i = 0; i < new->hashsize; i++) {
-                INIT_LIST_HEAD (&new->name_hash[i]);
+        while (!list_empty(&inode_table->active)) {
+            trav = list_first_entry(&inode_table->active, inode_t, list);
+            /* forget and unref the inode to retire and add it to
+             * purge list. By this time there should not be any
+             * inodes present in the active list except for root
+             * inode. Its a ref_leak otherwise. */
+            if (trav && (trav != inode_table->root))
+                gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0,
+                                 LG_MSG_REF_COUNT,
+                                 "Active inode(%p) with refcount"
+                                 "(%d) found during cleanup",
+                                 trav, trav->ref);
+            inode_forget_atomic(trav, 0);
+            __inode_ref_reduce_by_n(trav, 0);
         }
+    }
+    pthread_mutex_unlock(&inode_table->lock);
 
-        INIT_LIST_HEAD (&new->active);
-        INIT_LIST_HEAD (&new->lru);
-        INIT_LIST_HEAD (&new->purge);
+    inode_table_prune(inode_table);
 
-        ret = gf_asprintf (&new->name, "%s/inode", xl->name);
-        if (-1 == ret) {
-                /* TODO: This should be ok to continue, check with avati */
-                ;
-        }
+    GF_FREE(inode_table->inode_hash);
+    GF_FREE(inode_table->name_hash);
+    if (inode_table->dentry_pool)
+        mem_pool_destroy(inode_table->dentry_pool);
+    if (inode_table->inode_pool)
+        mem_pool_destroy(inode_table->inode_pool);
+    if (inode_table->fd_mem_pool)
+        mem_pool_destroy(inode_table->fd_mem_pool);
 
-        __inode_table_init_root (new);
+    pthread_mutex_destroy(&inode_table->lock);
 
-        pthread_mutex_init (&new->lock, NULL);
+    GF_FREE(inode_table->name);
+    GF_FREE(inode_table);
 
-        return new;
+    return;
 }
 
-
 inode_t *
-inode_from_path (inode_table_t *itable, const char *path)
+inode_from_path(inode_table_t *itable, const char *path)
 {
-        inode_t  *inode = NULL;
-        inode_t  *parent = NULL;
-        inode_t  *root = NULL;
-        inode_t  *curr = NULL;
-        char     *pathname = NULL;
-        char     *component = NULL, *next_component = NULL;
-        char     *strtokptr = NULL;
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    inode_t *root = NULL;
+    inode_t *curr = NULL;
+    char *pathname = NULL;
+    char *component = NULL, *next_component = NULL;
+    char *strtokptr = NULL;
+
+    if (!itable || !path)
+        return NULL;
 
-        if (!itable || !path)
-                return NULL;
+    /* top-down approach */
+    pathname = gf_strdup(path);
+    if (pathname == NULL) {
+        goto out;
+    }
+
+    root = itable->root;
+    parent = inode_ref(root);
+    component = strtok_r(pathname, "/", &strtokptr);
 
-        /* top-down approach */
-        pathname = gf_strdup (path);
-        if (pathname == NULL) {
-                gf_log ("inode", GF_LOG_ERROR, "out of memory");
-                goto out;
+    if (component == NULL)
+        /* root inode */
+        inode = inode_ref(parent);
+
+    while (component) {
+        curr = inode_grep(itable, parent, component);
+
+        if (curr == NULL) {
+            strtok_r(NULL, "/", &strtokptr);
+            break;
         }
 
-        root = itable->root;
-        parent = inode_ref (root);
-        component = strtok_r (pathname, "/", &strtokptr);
+        next_component = strtok_r(NULL, "/", &strtokptr);
 
-        if (component == NULL)
-                /* root inode */
-                inode = inode_ref (parent);
+        if (next_component) {
+            inode_unref(parent);
+            parent = curr;
+            curr = NULL;
+        } else {
+            inode = curr;
+        }
 
-        while (component) {
-                curr = inode_grep (itable, parent, component);
+        component = next_component;
+    }
 
-                if (curr == NULL) {
-                        strtok_r (NULL, "/", &strtokptr);
-                        break;
-                }
+    if (parent)
+        inode_unref(parent);
 
-                next_component = strtok_r (NULL, "/", &strtokptr);
+    GF_FREE(pathname);
 
-                if (next_component) {
-                        inode_unref (parent);
-                        parent = curr;
-                        curr = NULL;
-                } else {
-                        inode = curr;
-                }
+out:
+    return inode;
+}
 
-                component = next_component;
-        }
+void
+inode_set_need_lookup(inode_t *inode, xlator_t *this)
+{
+    uint64_t need_lookup = LOOKUP_NEEDED;
 
-        if (parent)
-                inode_unref (parent);
+    if (!inode || !this)
+        return;
 
-        if (pathname)
-                GF_FREE (pathname);
+    inode_ctx_set(inode, this, &need_lookup);
 
-out:
-        return inode;
+    return;
 }
 
+/* Function behaviour:
+ * Function return true if inode_ctx is not present,
+ * or value stored in inode_ctx is LOOKUP_NEEDED.
+ * If inode_ctx value is LOOKUP_NOT_NEEDED, which means
+ * inode_ctx is present for xlator this, but no lookup
+ * needed.
+ */
+gf_boolean_t
+inode_needs_lookup(inode_t *inode, xlator_t *this)
+{
+    uint64_t need_lookup = 0;
+    gf_boolean_t ret = _gf_false;
+    int op_ret = -1;
 
-int
-__inode_ctx_put2 (inode_t *inode, xlator_t *xlator, uint64_t value1,
-                  uint64_t value2)
-{
-        int ret = 0;
-        int index = 0;
-        int put_idx = -1;
-
-        if (!inode || !xlator)
-                return -1;
-
-        for (index = 0; index < xlator->graph->xl_count; index++) {
-                if (!inode->_ctx[index].xl_key) {
-                        if (put_idx == -1)
-                                put_idx = index;
-                        /* dont break, to check if key already exists
-                           further on */
-                }
-                if (inode->_ctx[index].xl_key == xlator) {
-                        put_idx = index;
-                        break;
-                }
-        }
+    if (!inode || !this)
+        return ret;
 
-        if (put_idx == -1) {
-                ret = -1;
-                goto out;;
-        }
+    op_ret = inode_ctx_get(inode, this, &need_lookup);
+    if (op_ret == -1) {
+        ret = _gf_true;
+    } else if (need_lookup == LOOKUP_NEEDED) {
+        ret = _gf_true;
+        need_lookup = LOOKUP_NOT_NEEDED;
+        inode_ctx_set(inode, this, &need_lookup);
+    }
+
+    return ret;
+}
 
-        inode->_ctx[put_idx].xl_key = xlator;
-        inode->_ctx[put_idx].value1 = value1;
-        inode->_ctx[put_idx].value2 = value2;
+int
+__inode_ctx_set2(inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
+                 uint64_t *value2_p)
+{
+    int ret = 0;
+    int set_idx = -1;
+
+    if (!inode || !xlator || !inode->_ctx)
+        return -1;
+
+    set_idx = __inode_get_xl_index(inode, xlator);
+    if (set_idx == -1) {
+        ret = -1;
+        goto out;
+        ;
+    }
+
+    inode->_ctx[set_idx].xl_key = xlator;
+    if (value1_p)
+        inode->_ctx[set_idx].value1 = *value1_p;
+    if (value2_p)
+        inode->_ctx[set_idx].value2 = *value2_p;
 out:
-        return ret;
+    return ret;
 }
 
+int
+__inode_ctx_set0(inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
+{
+    return __inode_ctx_set2(inode, xlator, value1_p, NULL);
+}
+
+int
+__inode_ctx_set1(inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
+{
+    return __inode_ctx_set2(inode, xlator, NULL, value2_p);
+}
 
 int
-inode_ctx_put2 (inode_t *inode, xlator_t *xlator, uint64_t value1,
-                uint64_t value2)
+inode_ctx_set2(inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
+               uint64_t *value2_p)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!inode || !xlator)
-                return -1;
+    if (!inode || !xlator)
+        return -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_put2 (inode, xlator, value1, value2);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_set2(inode, xlator, value1_p, value2_p);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
+int
+inode_ctx_set1(inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
+{
+    int ret = 0;
+
+    if (!inode || !xlator)
+        return -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_set1(inode, xlator, value2_p);
+    }
+    UNLOCK(&inode->lock);
 
+    return ret;
+}
 int
-__inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
-                  uint64_t *value2)
+inode_ctx_set0(inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
 {
-        int index = 0;
-        int ret = 0;
+    int ret = 0;
 
-        if (!inode || !xlator)
-                return -1;
+    if (!inode || !xlator)
+        return -1;
 
-        for (index = 0; index < xlator->graph->xl_count; index++) {
-                if (inode->_ctx[index].xl_key == xlator)
-                        break;
-        }
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_set0(inode, xlator, value1_p);
+    }
+    UNLOCK(&inode->lock);
 
-        if (index == xlator->graph->xl_count) {
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        if (value1)
-                *value1 = inode->_ctx[index].value1;
+int
+__inode_ctx_get2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                 uint64_t *value2)
+{
+    int index = 0;
+    int ret = -1;
 
-        if (value2)
-                *value2 = inode->_ctx[index].value2;
+    if (!inode || !xlator || !inode->_ctx)
+        goto out;
 
+    index = xlator->xl_id;
+    if (inode->_ctx[index].xl_key != xlator)
+        goto out;
+
+    if (inode->_ctx[index].value1) {
+        if (value1) {
+            *value1 = inode->_ctx[index].value1;
+            ret = 0;
+        }
+    }
+    if (inode->_ctx[index].value2) {
+        if (value2) {
+            *value2 = inode->_ctx[index].value2;
+            ret = 0;
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
+int
+__inode_ctx_get0(inode_t *inode, xlator_t *xlator, uint64_t *value1)
+{
+    uint64_t tmp_value = 0;
+    int ret = 0;
+
+    ret = __inode_ctx_get2(inode, xlator, &tmp_value, NULL);
+    if (!ret && value1)
+        *value1 = tmp_value;
+
+    return ret;
+}
 
 int
-inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
-                uint64_t *value2)
+__inode_ctx_get1(inode_t *inode, xlator_t *xlator, uint64_t *value2)
 {
-        int ret = 0;
+    uint64_t tmp_value = 0;
+    int ret = 0;
 
-        if (!inode || !xlator)
-                return -1;
+    ret = __inode_ctx_get2(inode, xlator, NULL, &tmp_value);
+    if (!ret && value2)
+        *value2 = tmp_value;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get2 (inode, xlator, value1, value2);
-        }
-        UNLOCK (&inode->lock);
+    return ret;
+}
 
-        return ret;
+int
+inode_ctx_get2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+               uint64_t *value2)
+{
+    int ret = 0;
+
+    if (!inode || !xlator)
+        return -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get2(inode, xlator, value1, value2);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
+int
+inode_ctx_get1(inode_t *inode, xlator_t *xlator, uint64_t *value2)
+{
+    int ret = 0;
+
+    if (!inode || !xlator)
+        return -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get1(inode, xlator, value2);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
 int
-inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
-                uint64_t *value2)
+inode_ctx_get0(inode_t *inode, xlator_t *xlator, uint64_t *value1)
 {
-        int index = 0;
-        int ret = 0;
+    int ret = 0;
 
-        if (!inode || !xlator)
-                return -1;
+    if (!inode || !xlator)
+        return -1;
 
-        LOCK (&inode->lock);
-        {
-                for (index = 0; index < xlator->graph->xl_count; index++) {
-                        if (inode->_ctx[index].xl_key == xlator)
-                                break;
-                }
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get0(inode, xlator, value1);
+    }
+    UNLOCK(&inode->lock);
 
-                if (index == xlator->graph->xl_count) {
-                        ret = -1;
-                        goto unlock;
-                }
+    return ret;
+}
+
+int
+inode_ctx_del2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+               uint64_t *value2)
+{
+    int index = 0;
+    int ret = 0;
+
+    if (!inode || !xlator)
+        return -1;
 
-                if (value1)
-                        *value1 = inode->_ctx[index].value1;
-                if (value2)
-                        *value2 = inode->_ctx[index].value2;
+    LOCK(&inode->lock);
+    {
+        if (!inode->_ctx)
+            goto unlock;
 
-                inode->_ctx[index].key    = 0;
-                inode->_ctx[index].value1 = 0;
-                inode->_ctx[index].value2 = 0;
+        index = xlator->xl_id;
+        if (inode->_ctx[index].xl_key != xlator) {
+            ret = -1;
+            goto unlock;
         }
+
+        if (inode->_ctx[index].value1 && value1)
+            *value1 = inode->_ctx[index].value1;
+        if (inode->_ctx[index].value2 && value2)
+            *value2 = inode->_ctx[index].value2;
+
+        inode->_ctx[index].key = 0;
+        inode->_ctx[index].xl_key = NULL;
+        inode->_ctx[index].value1 = 0;
+        inode->_ctx[index].value2 = 0;
+    }
 unlock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
-
-int
-__inode_ctx_put (inode_t *inode, xlator_t *key, uint64_t value)
+/* function behavior:
+ - if value1 is set, value1 in ctx is reset to 0 with current value passed
+ back in value1 address.
+ - if value2 is set, value2 in ctx is reset to 0 with current value passed
+ back in value2 address.
+ - if both are set, both fields are reset.
+*/
+static int
+__inode_ctx_reset2(inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                   uint64_t *value2)
 {
-        return __inode_ctx_put2 (inode, key, value, 0);
-}
+    int index = 0;
+    int ret = 0;
+
+    if (!inode || !xlator)
+        return -1;
+
+    LOCK(&inode->lock);
+    {
+        index = xlator->xl_id;
+        if (inode->_ctx[index].xl_key != xlator) {
+            ret = -1;
+            goto unlock;
+        }
 
+        if (inode->_ctx[index].value1 && value1) {
+            *value1 = inode->_ctx[index].value1;
+            inode->_ctx[index].value1 = 0;
+        }
+        if (inode->_ctx[index].value2 && value2) {
+            *value2 = inode->_ctx[index].value2;
+            inode->_ctx[index].value2 = 0;
+        }
+    }
+unlock:
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
 int
-inode_ctx_put (inode_t *inode, xlator_t *key, uint64_t value)
+inode_ctx_reset2(inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
+                 uint64_t *value2_p)
 {
-        return inode_ctx_put2 (inode, key, value, 0);
+    uint64_t tmp_value1 = 0;
+    uint64_t tmp_value2 = 0;
+    int ret = 0;
+
+    ret = __inode_ctx_reset2(inode, xlator, &tmp_value1, &tmp_value2);
+    if (!ret) {
+        if (value1_p)
+            *value1_p = tmp_value1;
+        if (value2_p)
+            *value2_p = tmp_value2;
+    }
+    return ret;
 }
 
-
 int
-__inode_ctx_get (inode_t *inode, xlator_t *key, uint64_t *value)
+inode_ctx_reset1(inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
 {
-        return __inode_ctx_get2 (inode, key, value, 0);
-}
+    uint64_t tmp_value2 = 0;
+    int ret = 0;
+
+    ret = __inode_ctx_reset2(inode, xlator, NULL, &tmp_value2);
 
+    if (!ret && value2_p)
+        *value2_p = tmp_value2;
 
+    return ret;
+}
 int
-inode_ctx_get (inode_t *inode, xlator_t *key, uint64_t *value)
+inode_ctx_reset0(inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
 {
-        return inode_ctx_get2 (inode, key, value, 0);
-}
+    uint64_t tmp_value1 = 0;
+    int ret = 0;
+
+    ret = __inode_ctx_reset2(inode, xlator, &tmp_value1, NULL);
+
+    if (!ret && value1_p)
+        *value1_p = tmp_value1;
 
+    return ret;
+}
 
 int
-inode_ctx_del (inode_t *inode, xlator_t *key, uint64_t *value)
+inode_is_linked(inode_t *inode)
 {
-        return inode_ctx_del2 (inode, key, value, 0);
-}
+    int ret = 0;
+    inode_table_t *table = NULL;
 
+    if (!inode) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, LG_MSG_INODE_NOT_FOUND,
+                         "inode not found");
+        return 0;
+    }
 
-void
-inode_dump (inode_t *inode, char *prefix)
-{
-        char               key[GF_DUMP_MAX_BUF_LEN];
-        int                ret       = -1;
-        xlator_t          *xl        = NULL;
-        int                i         = 0;
-        char               uuidbuf[256];
-        fd_t              *fd        = NULL;
-        struct _inode_ctx *inode_ctx = NULL;
-        struct  fd_wrapper {
-                fd_t *fd;
-                struct list_head next;
-        } *fd_wrapper, *tmp;
-        struct list_head fd_list;
-
-        if (!inode)
-                return;
-
-        INIT_LIST_HEAD (&fd_list);
-
-        ret = TRY_LOCK(&inode->lock);
-
-        if (ret != 0) {
-                gf_log ("", GF_LOG_WARNING, "Unable to dump inode"
-                        " errno: %d", errno);
-                return;
-        }
+    table = inode->table;
 
-        {
-                uuid_unparse (inode->gfid, uuidbuf);
-                gf_proc_dump_build_key(key, prefix, "gfid");
-                gf_proc_dump_write(key, "%s", uuidbuf);
-                gf_proc_dump_build_key(key, prefix, "nlookup");
-                gf_proc_dump_write(key, "%ld", inode->nlookup);
-                gf_proc_dump_build_key(key, prefix, "ref");
-                gf_proc_dump_write(key, "%u", inode->ref);
-                gf_proc_dump_build_key(key, prefix, "ino");
-                gf_proc_dump_write(key, "%ld", inode->ino);
-                gf_proc_dump_build_key(key, prefix, "ia_type");
-                gf_proc_dump_write(key, "%d", inode->ia_type);
-                if (inode->_ctx) {
-                        inode_ctx = GF_CALLOC (inode->table->xl->graph->xl_count,
-                                               sizeof (*inode_ctx),
-                                               gf_common_mt_inode_ctx);
-                        if (inode_ctx == NULL) {
-                                gf_log ("", GF_LOG_ERROR, "out of memory");
-                                goto unlock;
-                        }
-
-                        for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
-                                inode_ctx[i] = inode->_ctx[i];
-                        }
-                }
+    pthread_mutex_lock(&table->lock);
+    {
+        ret = __is_inode_hashed(inode);
+    }
+    pthread_mutex_unlock(&table->lock);
 
-                if (list_empty (&inode->fd_list)) {
-                        goto unlock;
-                }
+    return ret;
+}
 
-                list_for_each_entry (fd, &inode->fd_list, inode_list) {
-                        fd_wrapper = GF_CALLOC (1, sizeof (*fd_wrapper),
-                                                gf_common_mt_char);
-                        if (fd_wrapper == NULL) {
-                                gf_log ("", GF_LOG_ERROR, "out of memory");
-                                goto unlock;
-                        }
+void
+inode_dump(inode_t *inode, char *prefix)
+{
+    int ret = -1;
+    xlator_t *xl = NULL;
+    int i = 0;
+    fd_t *fd = NULL;
+    struct _inode_ctx *inode_ctx = NULL;
+    struct list_head fd_list;
+    int ref = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    uint64_t nlookup = 0;
+
+    if (!inode)
+        return;
 
-                        INIT_LIST_HEAD (&fd_wrapper->next);
-                        list_add_tail (&fd_wrapper->next, &fd_list);
+    INIT_LIST_HEAD(&fd_list);
 
-                        fd_wrapper->fd = _fd_ref (fd);
+    ret = TRY_LOCK(&inode->lock);
+    if (ret != 0) {
+        return;
+    }
+
+    {
+        nlookup = GF_ATOMIC_GET(inode->nlookup);
+        gf_proc_dump_write("gfid", "%s", uuid_utoa(inode->gfid));
+        gf_proc_dump_write("nlookup", "%" PRIu64, nlookup);
+        gf_proc_dump_write("fd-count", "%u", inode->fd_count);
+        gf_proc_dump_write("active-fd-count", "%u", inode->active_fd_count);
+        gf_proc_dump_write("ref", "%u", inode->ref);
+        gf_proc_dump_write("invalidate-sent", "%d", inode->invalidate_sent);
+        gf_proc_dump_write("ia_type", "%d", inode->ia_type);
+        if (inode->_ctx) {
+            inode_ctx = GF_CALLOC(inode->table->ctxcount, sizeof(*inode_ctx),
+                                  gf_common_mt_inode_ctx);
+            if (inode_ctx == NULL) {
+                goto unlock;
+            }
+
+            for (i = 0; i < inode->table->ctxcount; i++) {
+                inode_ctx[i] = inode->_ctx[i];
+                xl = inode_ctx[i].xl_key;
+                ref = inode_ctx[i].ref;
+                if (ref != 0 && xl) {
+                    gf_proc_dump_build_key(key, "ref_by_xl:", "%s", xl->name);
+                    gf_proc_dump_write(key, "%d", ref);
                 }
+            }
+        }
+
+        if (dump_options.xl_options.dump_fdctx != _gf_true)
+            goto unlock;
+
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            fd_ctx_dump(fd, prefix);
         }
+    }
 unlock:
-        UNLOCK(&inode->lock);
-
-        if (inode_ctx && (dump_options.xl_options.dump_inodectx == _gf_true)) {
-                for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
-                        if (inode_ctx[i].xl_key) {
-                                xl = (xlator_t *)(long)inode_ctx[i].xl_key;
-                                if (xl->dumpops && xl->dumpops->inodectx)
-                                        xl->dumpops->inodectx (xl, inode);
-                        }
-                }
+    UNLOCK(&inode->lock);
+
+    if (inode_ctx && (dump_options.xl_options.dump_inodectx == _gf_true)) {
+        for (i = 0; i < inode->table->ctxcount; i++) {
+            if (inode_ctx[i].xl_key) {
+                xl = (xlator_t *)(long)inode_ctx[i].xl_key;
+                if (xl->dumpops && xl->dumpops->inodectx)
+                    xl->dumpops->inodectx(xl, inode);
+            }
         }
+    }
 
-        if (!list_empty (&fd_list)
-            && (dump_options.xl_options.dump_fdctx == _gf_true)) {
-                list_for_each_entry_safe (fd_wrapper, tmp, &fd_list,
-                                          next) {
-                        list_del (&fd_wrapper->next);
-                        fd_ctx_dump (fd_wrapper->fd, prefix);
+    GF_FREE(inode_ctx);
 
-                        fd_unref (fd_wrapper->fd);
-                        GF_FREE (fd_wrapper);
-                }
-        }
+    return;
+}
 
-        if (inode_ctx != NULL) {
-                GF_FREE (inode_ctx);
-        }
+void
+inode_table_dump(inode_table_t *itable, char *prefix)
+{
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int ret = 0;
 
+    if (!itable)
         return;
+
+    ret = pthread_mutex_trylock(&itable->lock);
+
+    if (ret != 0) {
+        return;
+    }
+
+    gf_proc_dump_build_key(key, prefix, "hashsize");
+    gf_proc_dump_write(key, "%" GF_PRI_SIZET, itable->hashsize);
+    gf_proc_dump_build_key(key, prefix, "name");
+    gf_proc_dump_write(key, "%s", itable->name);
+
+    gf_proc_dump_build_key(key, prefix, "lru_limit");
+    gf_proc_dump_write(key, "%d", itable->lru_limit);
+    gf_proc_dump_build_key(key, prefix, "active_size");
+    gf_proc_dump_write(key, "%d", itable->active_size);
+    gf_proc_dump_build_key(key, prefix, "lru_size");
+    gf_proc_dump_write(key, "%d", itable->lru_size);
+    gf_proc_dump_build_key(key, prefix, "purge_size");
+    gf_proc_dump_write(key, "%d", itable->purge_size);
+    gf_proc_dump_build_key(key, prefix, "invalidate_size");
+    gf_proc_dump_write(key, "%d", itable->invalidate_size);
+
+    INODE_DUMP_LIST(&itable->active, key, prefix, "active");
+    INODE_DUMP_LIST(&itable->lru, key, prefix, "lru");
+    INODE_DUMP_LIST(&itable->purge, key, prefix, "purge");
+    INODE_DUMP_LIST(&itable->invalidate, key, prefix, "invalidate");
+
+    pthread_mutex_unlock(&itable->lock);
 }
 
 void
-inode_table_dump (inode_table_t *itable, char *prefix)
+inode_dump_to_dict(inode_t *inode, char *prefix, dict_t *dict)
 {
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    uint64_t nlookup = 0;
+
+    ret = TRY_LOCK(&inode->lock);
+    if (ret)
+        return;
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_set_dynstr(dict, key, gf_strdup(uuid_utoa(inode->gfid)));
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    nlookup = GF_ATOMIC_GET(inode->nlookup);
+    ret = dict_set_uint64(dict, key, nlookup);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_set_uint32(dict, key, inode->ref);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_set_int32(dict, key, inode->ia_type);
+    if (ret)
+        goto out;
 
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     ret = 0;
+out:
+    UNLOCK(&inode->lock);
+    return;
+}
 
-        if (!itable)
-                return;
+void
+inode_table_dump_to_dict(inode_table_t *itable, char *prefix, dict_t *dict)
+{
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int ret = 0;
+#ifdef DEBUG
+    inode_t *inode = NULL;
+    int count = 0;
+#endif
+    ret = pthread_mutex_trylock(&itable->lock);
+    if (ret)
+        return;
 
-        memset(key, 0, sizeof(key));
-        ret = pthread_mutex_trylock(&itable->lock);
+    snprintf(key, sizeof(key), "%s.itable.lru_limit", prefix);
+    ret = dict_set_uint32(dict, key, itable->lru_limit);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.itable.active_size", prefix);
+    ret = dict_set_uint32(dict, key, itable->active_size);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.itable.lru_size", prefix);
+    ret = dict_set_uint32(dict, key, itable->lru_size);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.itable.purge_size", prefix);
+    ret = dict_set_uint32(dict, key, itable->purge_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    /* Dumping inode details in dictionary and sending it to CLI is not
+       required as when a developer (or support team) asks for this command
+       output, they just want to get top level detail of inode table.
+       If one wants to debug, let them take statedump and debug, this
+       wouldn't be available in CLI during production setup.
+    */
+    list_for_each_entry(inode, &itable->active, list)
+    {
+        snprintf(key, sizeof(key), "%s.itable.active%d", prefix, count++);
+        inode_dump_to_dict(inode, key, dict);
+    }
+    count = 0;
+
+    list_for_each_entry(inode, &itable->lru, list)
+    {
+        snprintf(key, sizeof(key), "%s.itable.lru%d", prefix, count++);
+        inode_dump_to_dict(inode, key, dict);
+    }
+    count = 0;
+
+    list_for_each_entry(inode, &itable->purge, list)
+    {
+        snprintf(key, sizeof(key), "%s.itable.purge%d", prefix, count++);
+        inode_dump_to_dict(inode, key, dict);
+    }
+#endif
 
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump inode table"
-                       " errno: %d", errno);
-                return;
+out:
+    pthread_mutex_unlock(&itable->lock);
+
+    return;
+}
+
+size_t
+inode_ctx_size(inode_t *inode)
+{
+    int i = 0;
+    size_t size = 0;
+    xlator_t *xl = NULL, *old_THIS = NULL;
+
+    if (!inode)
+        goto out;
+
+    LOCK(&inode->lock);
+    {
+        for (i = 0; i < inode->table->ctxcount; i++) {
+            if (!inode->_ctx[i].xl_key)
+                continue;
+
+            xl = (xlator_t *)(long)inode->_ctx[i].xl_key;
+            old_THIS = THIS;
+            THIS = xl;
+
+            /* If inode ref is taken when THIS is global xlator,
+             * the ctx xl_key is set, but the value is NULL.
+             * For global xlator the cbks can be NULL, hence check
+             * for the same */
+            if (!xl->cbks) {
+                THIS = old_THIS;
+                continue;
+            }
+
+            if (xl->cbks->ictxsize)
+                size += xl->cbks->ictxsize(xl, inode);
+
+            THIS = old_THIS;
         }
+    }
+    UNLOCK(&inode->lock);
 
-        gf_proc_dump_build_key(key, prefix, "hashsize");
-        gf_proc_dump_write(key, "%d", itable->hashsize);
-        gf_proc_dump_build_key(key, prefix, "name");
-        gf_proc_dump_write(key, "%s", itable->name);
-
-        gf_proc_dump_build_key(key, prefix, "lru_limit");
-        gf_proc_dump_write(key, "%d", itable->lru_limit);
-        gf_proc_dump_build_key(key, prefix, "active_size");
-        gf_proc_dump_write(key, "%d", itable->active_size);
-        gf_proc_dump_build_key(key, prefix, "lru_size");
-        gf_proc_dump_write(key, "%d", itable->lru_size);
-        gf_proc_dump_build_key(key, prefix, "purge_size");
-        gf_proc_dump_write(key, "%d", itable->purge_size);
-
-        INODE_DUMP_LIST(&itable->active, key, prefix, "active");
-        INODE_DUMP_LIST(&itable->lru, key, prefix, "lru");
-        INODE_DUMP_LIST(&itable->purge, key, prefix, "purge");
-
-        pthread_mutex_unlock(&itable->lock);
+out:
+    return size;
+}
+
+/* *
+ * This function finds name of the inode, if it has dentry. The dentry will be
+ * created only if inode_link happens with valid parent and name. And this
+ * function is only applicable for directories because multiple dentries are
+ * not possible(no hardlinks)
+ * */
+void
+inode_find_directory_name(inode_t *inode, const char **name)
+{
+    dentry_t *dentry = NULL;
+
+    GF_VALIDATE_OR_GOTO("inode", inode, out);
+    GF_VALIDATE_OR_GOTO("inode", name, out);
+
+    if (!IA_ISDIR(inode->ia_type))
+        return;
+
+    pthread_mutex_lock(&inode->table->lock);
+    {
+        dentry = __dentry_search_arbit(inode);
+        if (dentry) {
+            *name = dentry->name;
+        }
+    }
+    pthread_mutex_unlock(&inode->table->lock);
+out:
+    return;
 }
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
deleted file mode 100644
index 15a4f98f7bd..00000000000
--- a/libglusterfs/src/inode.h
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _INODE_H
-#define _INODE_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <sys/types.h>
-
-#define DEFAULT_INODE_MEMPOOL_ENTRIES   16384
-struct _inode_table;
-typedef struct _inode_table inode_table_t;
-
-struct _inode;
-typedef struct _inode inode_t;
-
-struct _dentry;
-typedef struct _dentry dentry_t;
-
-#include "list.h"
-#include "xlator.h"
-#include "iatt.h"
-#include "uuid.h"
-
-
-struct _inode_table {
-        pthread_mutex_t    lock;
-        size_t             hashsize;    /* bucket size of inode hash and dentry hash */
-        char              *name;        /* name of the inode table, just for gf_log() */
-        inode_t           *root;        /* root directory inode, with number 1 */
-        xlator_t          *xl;          /* xlator to be called to do purge */
-        uint32_t           lru_limit;   /* maximum LRU cache size */
-        struct list_head  *inode_hash;  /* buckets for inode hash table */
-        struct list_head  *name_hash;   /* buckets for dentry hash table */
-        struct list_head   active;      /* list of inodes currently active (in an fop) */
-        uint32_t           active_size; /* count of inodes in active list */
-        struct list_head   lru;         /* list of inodes recently used.
-                                           lru.next most recent */
-        uint32_t           lru_size;    /* count of inodes in lru list  */
-        struct list_head   purge;       /* list of inodes to be purged soon */
-        uint32_t           purge_size;  /* count of inodes in purge list */
-
-        struct mem_pool   *inode_pool;  /* memory pool for inodes */
-        struct mem_pool   *dentry_pool; /* memory pool for dentrys */
-        struct mem_pool   *fd_mem_pool; /* memory pool for fd_t */
-};
-
-
-struct _dentry {
-        struct list_head   inode_list;   /* list of dentries of inode */
-        struct list_head   hash;         /* hash table pointers */
-        inode_t           *inode;        /* inode of this directory entry */
-        char              *name;         /* name of the directory entry */
-        inode_t           *parent;       /* directory of the entry */
-};
-
-struct _inode_ctx {
-        union {
-                uint64_t    key;
-                xlator_t   *xl_key;
-        };
-        union {
-                uint64_t    value1;
-                void       *ptr1;
-        };
-        union {
-                uint64_t    value2;
-                void       *ptr2;
-        };
-};
-
-struct _inode {
-        inode_table_t       *table;         /* the table this inode belongs to */
-        uuid_t               gfid;
-        gf_lock_t            lock;
-        uint64_t             nlookup;
-        uint32_t             ref;           /* reference count on this inode */
-        ino_t                ino;           /* inode number in the storage (persistent) */
-        ia_type_t            ia_type;       /* what kind of file */
-        struct list_head     fd_list;       /* list of open files on this inode */
-        struct list_head     dentry_list;   /* list of directory entries for this inode */
-        struct list_head     hash;          /* hash table pointers */
-        struct list_head     list;          /* active/lru/purge */
-
-	struct _inode_ctx   *_ctx;    /* replacement for dict_t *(inode->ctx) */
-};
-
-
-inode_table_t *
-inode_table_new (size_t lru_limit, xlator_t *xl);
-
-inode_t *
-inode_new (inode_table_t *table);
-
-inode_t *
-inode_link (inode_t *inode, inode_t *parent,
-            const char *name, struct iatt *stbuf);
-
-void
-inode_unlink (inode_t *inode, inode_t *parent, const char *name);
-
-inode_t *
-inode_parent (inode_t *inode, ino_t par, const char *name);
-
-inode_t *
-inode_ref (inode_t *inode);
-
-inode_t *
-inode_unref (inode_t *inode);
-
-int
-inode_lookup (inode_t *inode);
-
-int
-inode_forget (inode_t *inode, uint64_t nlookup);
-
-int
-inode_rename (inode_table_t *table, inode_t *olddir, const char *oldname,
-	      inode_t *newdir, const char *newname,
-	      inode_t *inode, struct iatt *stbuf);
-
-inode_t *
-inode_grep (inode_table_t *table, inode_t *parent, const char *name);
-
-inode_t *
-inode_get (inode_table_t *table, ino_t ino, uint64_t gen);
-
-inode_t *
-inode_find (inode_table_t *table, uuid_t gfid);
-
-int
-inode_path (inode_t *inode, const char *name, char **bufp);
-
-inode_t *
-inode_from_path (inode_table_t *table, const char *path);
-
-int
-__inode_ctx_put (inode_t *inode, xlator_t *xlator, uint64_t value);
-
-int
-inode_ctx_put (inode_t *inode, xlator_t *xlator, uint64_t value);
-
-int
-__inode_ctx_get (inode_t *inode, xlator_t *xlator, uint64_t *value);
-
-int
-inode_ctx_get (inode_t *inode, xlator_t *xlator, uint64_t *value);
-
-int
-inode_ctx_del (inode_t *inode, xlator_t *xlator, uint64_t *value);
-
-int
-inode_ctx_put2 (inode_t *inode, xlator_t *xlator, uint64_t value1,
-                uint64_t value2);
-
-int
-inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
-                uint64_t *value2);
-
-int
-inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
-                uint64_t *value2);
-
-#endif /* _INODE_H */
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index a8bd192cf88..4e7d2958764 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -1,747 +1,1146 @@
 /*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-#include "iobuf.h"
-#include "statedump.h"
+#include "glusterfs/iobuf.h"
+#include "glusterfs/statedump.h"
 #include <stdio.h>
-
+#include "glusterfs/libglusterfs-messages.h"
 
 /*
   TODO: implement destroy margins and prefetching of arenas
 */
 
-void
-__iobuf_arena_init_iobufs (struct iobuf_arena *iobuf_arena)
-{
-        size_t              arena_size = 0;
-        size_t              page_size = 0;
-        int                 iobuf_cnt = 0;
-        struct iobuf       *iobuf = NULL;
-        int                 offset = 0;
-        int                 i = 0;
-
-        arena_size = iobuf_arena->iobuf_pool->arena_size;
-        page_size  = iobuf_arena->iobuf_pool->page_size;
-        iobuf_cnt  = arena_size / page_size;
-
-        iobuf_arena->iobufs = GF_CALLOC (sizeof (*iobuf), iobuf_cnt,
-                                        gf_common_mt_iobuf);
-        if (!iobuf_arena->iobufs)
-                return;
+#define IOBUF_ARENA_MAX_INDEX                                                  \
+    (sizeof(gf_iobuf_init_config) / (sizeof(struct iobuf_init_config)))
 
-        iobuf = iobuf_arena->iobufs;
-        for (i = 0; i < iobuf_cnt; i++) {
-                INIT_LIST_HEAD (&iobuf->list);
-                LOCK_INIT (&iobuf->lock);
+/* Make sure this array is sorted based on pagesize */
+static const struct iobuf_init_config gf_iobuf_init_config[] = {
+    /* { pagesize, num_pages }, */
+    {128, 1024},     {512, 512},       {2 * 1024, 512}, {8 * 1024, 128},
+    {32 * 1024, 64}, {128 * 1024, 32}, {256 * 1024, 8}, {1 * 1024 * 1024, 2},
+};
 
-                iobuf->iobuf_arena = iobuf_arena;
+static int
+gf_iobuf_get_arena_index(const size_t page_size)
+{
+    int i;
 
-                iobuf->ptr = iobuf_arena->mem_base + offset;
+    for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+        if (page_size <= gf_iobuf_init_config[i].pagesize)
+            return i;
+    }
 
-                list_add (&iobuf->list, &iobuf_arena->passive.list);
-                iobuf_arena->passive_cnt++;
+    return -1;
+}
 
-                offset += page_size;
-                iobuf++;
+static size_t
+gf_iobuf_get_pagesize(const size_t page_size, int *index)
+{
+    int i;
+    size_t size = 0;
+
+    for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+        size = gf_iobuf_init_config[i].pagesize;
+        if (page_size <= size) {
+            if (index != NULL)
+                *index = i;
+            return size;
         }
-}
+    }
 
+    return -1;
+}
 
-void
-__iobuf_arena_destroy_iobufs (struct iobuf_arena *iobuf_arena)
+static void
+__iobuf_arena_init_iobufs(struct iobuf_arena *iobuf_arena)
 {
-        size_t              arena_size = 0;
-        size_t              page_size = 0;
-        int                 iobuf_cnt = 0;
-        struct iobuf       *iobuf = NULL;
-        int                 i = 0;
+    const int iobuf_cnt = iobuf_arena->page_count;
+    struct iobuf *iobuf = NULL;
+    int offset = 0;
+    int i = 0;
+
+    iobuf_arena->iobufs = GF_CALLOC(sizeof(*iobuf), iobuf_cnt,
+                                    gf_common_mt_iobuf);
+    if (!iobuf_arena->iobufs)
+        return;
 
-        arena_size = iobuf_arena->iobuf_pool->arena_size;
-        page_size  = iobuf_arena->iobuf_pool->page_size;
-        iobuf_cnt  = arena_size / page_size;
+    iobuf = iobuf_arena->iobufs;
+    for (i = 0; i < iobuf_cnt; i++) {
+        INIT_LIST_HEAD(&iobuf->list);
+        LOCK_INIT(&iobuf->lock);
 
-        if (!iobuf_arena->iobufs)
-                return;
+        iobuf->iobuf_arena = iobuf_arena;
 
-        iobuf = iobuf_arena->iobufs;
-        for (i = 0; i < iobuf_cnt; i++) {
-                GF_ASSERT (iobuf->ref == 0);
+        iobuf->ptr = iobuf_arena->mem_base + offset;
 
-                list_del_init (&iobuf->list);
-                iobuf++;
-        }
+        list_add(&iobuf->list, &iobuf_arena->passive.list);
+        iobuf_arena->passive_cnt++;
 
-        GF_FREE (iobuf_arena->iobufs);
-}
+        offset += iobuf_arena->page_size;
+        iobuf++;
+    }
 
+    return;
+}
 
-void
-__iobuf_arena_destroy (struct iobuf_arena *iobuf_arena)
+static void
+__iobuf_arena_destroy_iobufs(struct iobuf_arena *iobuf_arena)
 {
-        struct iobuf_pool *iobuf_pool = NULL;
+    int iobuf_cnt = 0;
+    struct iobuf *iobuf = NULL;
+    int i = 0;
 
-        if (!iobuf_arena)
-                return;
+    if (!iobuf_arena->iobufs) {
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0, LG_MSG_IOBUFS_NOT_FOUND,
+                         "iobufs not found");
+        return;
+    }
 
-        iobuf_pool = iobuf_arena->iobuf_pool;
+    iobuf_cnt = iobuf_arena->page_count;
+    iobuf = iobuf_arena->iobufs;
+    for (i = 0; i < iobuf_cnt; i++) {
+        GF_ASSERT(GF_ATOMIC_GET(iobuf->ref) == 0);
 
-        __iobuf_arena_destroy_iobufs (iobuf_arena);
+        LOCK_DESTROY(&iobuf->lock);
+        list_del_init(&iobuf->list);
+        iobuf++;
+    }
 
-        if (iobuf_arena->mem_base
-            && iobuf_arena->mem_base != MAP_FAILED)
-                munmap (iobuf_arena->mem_base, iobuf_pool->arena_size);
+    GF_FREE(iobuf_arena->iobufs);
 
-        GF_FREE (iobuf_arena);
+    return;
 }
 
-
-struct iobuf_arena *
-__iobuf_arena_alloc (struct iobuf_pool *iobuf_pool)
+static void
+__iobuf_arena_destroy(struct iobuf_pool *iobuf_pool,
+                      struct iobuf_arena *iobuf_arena)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        size_t              arena_size = 0;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
 
-        iobuf_arena = GF_CALLOC (sizeof (*iobuf_arena), 1,
-                             gf_common_mt_iobuf_arena);
-        if (!iobuf_arena)
-                goto err;
+    if (iobuf_pool->rdma_deregistration)
+        iobuf_pool->rdma_deregistration(iobuf_pool->mr_list, iobuf_arena);
 
-        INIT_LIST_HEAD (&iobuf_arena->list);
-        INIT_LIST_HEAD (&iobuf_arena->active.list);
-        INIT_LIST_HEAD (&iobuf_arena->passive.list);
-        iobuf_arena->iobuf_pool = iobuf_pool;
+    __iobuf_arena_destroy_iobufs(iobuf_arena);
 
-        arena_size = iobuf_pool->arena_size;
-        iobuf_arena->mem_base = mmap (NULL, arena_size, PROT_READ|PROT_WRITE,
-                                      MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-        if (iobuf_arena->mem_base == MAP_FAILED)
-                goto err;
+    if (iobuf_arena->mem_base && iobuf_arena->mem_base != MAP_FAILED)
+        munmap(iobuf_arena->mem_base, iobuf_arena->arena_size);
 
-        __iobuf_arena_init_iobufs (iobuf_arena);
-        if (!iobuf_arena->iobufs)
-                goto err;
+    GF_FREE(iobuf_arena);
+out:
+    return;
+}
 
-        iobuf_pool->arena_cnt++;
+static struct iobuf_arena *
+__iobuf_arena_alloc(struct iobuf_pool *iobuf_pool, size_t page_size,
+                    int32_t num_iobufs)
+{
+    struct iobuf_arena *iobuf_arena = NULL;
+    size_t rounded_size = 0;
+    int index = 0; /* unused */
 
-        return iobuf_arena;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
 
-err:
-        __iobuf_arena_destroy (iobuf_arena);
-        return NULL;
-}
+    iobuf_arena = GF_CALLOC(sizeof(*iobuf_arena), 1, gf_common_mt_iobuf_arena);
+    if (!iobuf_arena)
+        goto err;
 
+    INIT_LIST_HEAD(&iobuf_arena->list);
+    INIT_LIST_HEAD(&iobuf_arena->all_list);
+    INIT_LIST_HEAD(&iobuf_arena->active.list);
+    INIT_LIST_HEAD(&iobuf_arena->passive.list);
+    iobuf_arena->iobuf_pool = iobuf_pool;
 
-struct iobuf_arena *
-__iobuf_arena_unprune (struct iobuf_pool *iobuf_pool)
-{
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *tmp = NULL;
+    rounded_size = gf_iobuf_get_pagesize(page_size, &index);
 
-        list_for_each_entry (tmp, &iobuf_pool->purge.list, list) {
-                list_del_init (&tmp->list);
-                iobuf_arena = tmp;
-                break;
-        }
+    iobuf_arena->page_size = rounded_size;
+    iobuf_arena->page_count = num_iobufs;
 
-        return iobuf_arena;
-}
+    iobuf_arena->arena_size = rounded_size * num_iobufs;
 
+    iobuf_arena->mem_base = mmap(NULL, iobuf_arena->arena_size,
+                                 PROT_READ | PROT_WRITE,
+                                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (iobuf_arena->mem_base == MAP_FAILED) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_MAPPING_FAILED, NULL);
+        goto err;
+    }
 
-struct iobuf_arena *
-__iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool)
-{
-        struct iobuf_arena *iobuf_arena = NULL;
+    if (iobuf_pool->rdma_registration) {
+        iobuf_pool->rdma_registration(iobuf_pool->device, iobuf_arena);
+    }
 
-        iobuf_arena = __iobuf_arena_unprune (iobuf_pool);
+    list_add_tail(&iobuf_arena->all_list, &iobuf_pool->all_arenas);
 
-        if (!iobuf_arena)
-                iobuf_arena = __iobuf_arena_alloc (iobuf_pool);
+    __iobuf_arena_init_iobufs(iobuf_arena);
+    if (!iobuf_arena->iobufs) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, LG_MSG_INIT_IOBUF_FAILED, NULL);
+        goto err;
+    }
 
-        if (!iobuf_arena)
-                return NULL;
+    iobuf_pool->arena_cnt++;
 
-        list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas.list);
+    return iobuf_arena;
 
-        return iobuf_arena;
+err:
+    __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
+
+out:
+    return NULL;
 }
 
+static struct iobuf_arena *
+__iobuf_arena_unprune(struct iobuf_pool *iobuf_pool, const size_t page_size,
+                      const int index)
+{
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
 
-struct iobuf_arena *
-iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool)
+    list_for_each_entry(tmp, &iobuf_pool->purge[index], list)
+    {
+        list_del_init(&tmp->list);
+        iobuf_arena = tmp;
+        break;
+    }
+out:
+    return iobuf_arena;
+}
+
+static struct iobuf_arena *
+__iobuf_pool_add_arena(struct iobuf_pool *iobuf_pool, const size_t page_size,
+                       const int32_t num_pages, const int index)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
 
-        pthread_mutex_lock (&iobuf_pool->mutex);
-        {
-                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool);
+    iobuf_arena = __iobuf_arena_unprune(iobuf_pool, page_size, index);
+
+    if (!iobuf_arena) {
+        iobuf_arena = __iobuf_arena_alloc(iobuf_pool, page_size, num_pages);
+        if (!iobuf_arena) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND,
+                    NULL);
+            return NULL;
         }
-        pthread_mutex_unlock (&iobuf_pool->mutex);
+    }
+    list_add(&iobuf_arena->list, &iobuf_pool->arenas[index]);
 
-        return iobuf_arena;
+    return iobuf_arena;
 }
 
-
+/* This function destroys all the iobufs and the iobuf_pool */
 void
-iobuf_pool_destroy (struct iobuf_pool *iobuf_pool)
+iobuf_pool_destroy(struct iobuf_pool *iobuf_pool)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *tmp = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *tmp = NULL;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
+
+    pthread_mutex_lock(&iobuf_pool->mutex);
+    {
+        for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+            list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->arenas[i],
+                                     list)
+            {
+                list_del_init(&iobuf_arena->list);
+                iobuf_pool->arena_cnt--;
 
-        if (!iobuf_pool)
-                return;
+                __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
+            }
+            list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->purge[i],
+                                     list)
+            {
+                list_del_init(&iobuf_arena->list);
+                iobuf_pool->arena_cnt--;
+                __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
+            }
+            /* If there are no iobuf leaks, there should be no
+             * arenas in the filled list. If at all there are any
+             * arenas in the filled list, the below function will
+             * assert.
+             */
+            list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->filled[i],
+                                     list)
+            {
+                list_del_init(&iobuf_arena->list);
+                iobuf_pool->arena_cnt--;
+                __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
+            }
+            /* If there are no iobuf leaks, there shoould be
+             * no standard allocated arenas, iobuf_put will free
+             * such arenas.
+             * TODO: Free the stdalloc arenas forcefully if present?
+             */
+        }
+    }
+    pthread_mutex_unlock(&iobuf_pool->mutex);
 
-        list_for_each_entry_safe (iobuf_arena, tmp, &iobuf_pool->arenas.list,
-                                  list) {
+    pthread_mutex_destroy(&iobuf_pool->mutex);
 
-                list_del_init (&iobuf_arena->list);
-                iobuf_pool->arena_cnt--;
+    GF_FREE(iobuf_pool);
 
-                __iobuf_arena_destroy (iobuf_arena);
-        }
+out:
+    return;
 }
 
-
-struct iobuf_pool *
-iobuf_pool_new (size_t arena_size, size_t page_size)
+static void
+iobuf_create_stdalloc_arena(struct iobuf_pool *iobuf_pool)
 {
-        struct iobuf_pool  *iobuf_pool = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
 
-        if (arena_size < page_size)
-                return NULL;
+    /* No locking required here as its called only once during init */
+    iobuf_arena = GF_CALLOC(sizeof(*iobuf_arena), 1, gf_common_mt_iobuf_arena);
+    if (!iobuf_arena)
+        goto err;
 
-        iobuf_pool = GF_CALLOC (sizeof (*iobuf_pool), 1,
-                                gf_common_mt_iobuf_pool);
-        if (!iobuf_pool)
-                return NULL;
+    INIT_LIST_HEAD(&iobuf_arena->list);
+    INIT_LIST_HEAD(&iobuf_arena->active.list);
+    INIT_LIST_HEAD(&iobuf_arena->passive.list);
 
-        pthread_mutex_init (&iobuf_pool->mutex, NULL);
-        INIT_LIST_HEAD (&iobuf_pool->arenas.list);
-        INIT_LIST_HEAD (&iobuf_pool->filled.list);
-        INIT_LIST_HEAD (&iobuf_pool->purge.list);
+    iobuf_arena->iobuf_pool = iobuf_pool;
 
-        iobuf_pool->arena_size = arena_size;
-        iobuf_pool->page_size  = page_size;
+    iobuf_arena->page_size = 0x7fffffff;
 
-        iobuf_pool_add_arena (iobuf_pool);
+    list_add_tail(&iobuf_arena->list,
+                  &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX]);
 
-        return iobuf_pool;
+err:
+    return;
 }
 
-
-void
-__iobuf_pool_prune (struct iobuf_pool *iobuf_pool)
+struct iobuf_pool *
+iobuf_pool_new(void)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *tmp = NULL;
+    struct iobuf_pool *iobuf_pool = NULL;
+    int i = 0;
+    size_t page_size = 0;
+    size_t arena_size = 0;
+    int32_t num_pages = 0;
+
+    iobuf_pool = GF_CALLOC(sizeof(*iobuf_pool), 1, gf_common_mt_iobuf_pool);
+    if (!iobuf_pool)
+        goto out;
+    INIT_LIST_HEAD(&iobuf_pool->all_arenas);
+    pthread_mutex_init(&iobuf_pool->mutex, NULL);
+    for (i = 0; i <= IOBUF_ARENA_MAX_INDEX; i++) {
+        INIT_LIST_HEAD(&iobuf_pool->arenas[i]);
+        INIT_LIST_HEAD(&iobuf_pool->filled[i]);
+        INIT_LIST_HEAD(&iobuf_pool->purge[i]);
+    }
+
+    iobuf_pool->default_page_size = 128 * GF_UNIT_KB;
+
+    iobuf_pool->rdma_registration = NULL;
+    iobuf_pool->rdma_deregistration = NULL;
+
+    for (i = 0; i < GF_RDMA_DEVICE_COUNT; i++) {
+        iobuf_pool->device[i] = NULL;
+        iobuf_pool->mr_list[i] = NULL;
+    }
+
+    /* No locking required here
+     * as no one else can use this pool yet
+     */
+    for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+        page_size = gf_iobuf_init_config[i].pagesize;
+        num_pages = gf_iobuf_init_config[i].num_pages;
+
+        if (__iobuf_pool_add_arena(iobuf_pool, page_size, num_pages, i) != NULL)
+            arena_size += page_size * num_pages;
+    }
+
+    /* Need an arena to handle all the bigger iobuf requests */
+    iobuf_create_stdalloc_arena(iobuf_pool);
+
+    iobuf_pool->arena_size = arena_size;
+out:
 
-        if (list_empty (&iobuf_pool->arenas.list))
-                /* buffering - preserve this one arena (if at all)
-                   for __iobuf_arena_unprune */
-                return;
+    return iobuf_pool;
+}
 
-        list_for_each_entry_safe (iobuf_arena, tmp, &iobuf_pool->purge.list,
-                                  list) {
-                if (iobuf_arena->active_cnt)
-                        continue;
+static void
+__iobuf_arena_prune(struct iobuf_pool *iobuf_pool,
+                    struct iobuf_arena *iobuf_arena, const int index)
+{
+    /* code flow comes here only if the arena is in purge list and we can
+     * free the arena only if we have at least one arena in 'arenas' list
+     * (ie, at least few iobufs free in arena), that way, there won't
+     * be spurious mmap/unmap of buffers
+     */
+    if (list_empty(&iobuf_pool->arenas[index]))
+        goto out;
 
-                list_del_init (&iobuf_arena->list);
-                iobuf_pool->arena_cnt--;
+    /* All cases matched, destroy */
+    list_del_init(&iobuf_arena->list);
+    list_del_init(&iobuf_arena->all_list);
+    iobuf_pool->arena_cnt--;
 
-                __iobuf_arena_destroy (iobuf_arena);
-        }
-}
+    __iobuf_arena_destroy(iobuf_pool, iobuf_arena);
 
+out:
+    return;
+}
 
 void
-iobuf_pool_prune (struct iobuf_pool *iobuf_pool)
+iobuf_pool_prune(struct iobuf_pool *iobuf_pool)
 {
-        pthread_mutex_lock (&iobuf_pool->mutex);
-        {
-                __iobuf_pool_prune (iobuf_pool);
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *tmp = NULL;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
+
+    pthread_mutex_lock(&iobuf_pool->mutex);
+    {
+        for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+            if (list_empty(&iobuf_pool->arenas[i])) {
+                continue;
+            }
+
+            list_for_each_entry_safe(iobuf_arena, tmp, &iobuf_pool->purge[i],
+                                     list)
+            {
+                __iobuf_arena_prune(iobuf_pool, iobuf_arena, i);
+            }
         }
-        pthread_mutex_unlock (&iobuf_pool->mutex);
-}
+    }
+    pthread_mutex_unlock(&iobuf_pool->mutex);
 
+out:
+    return;
+}
 
-struct iobuf_arena *
-__iobuf_select_arena (struct iobuf_pool *iobuf_pool)
+/* Always called under the iobuf_pool mutex lock */
+static struct iobuf_arena *
+__iobuf_select_arena(struct iobuf_pool *iobuf_pool, const size_t page_size,
+                     const int index)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_arena *trav = NULL;
-
-        /* look for unused iobuf from the head-most arena */
-        list_for_each_entry (trav, &iobuf_pool->arenas.list, list) {
-                if (trav->passive_cnt) {
-                        iobuf_arena = trav;
-                        break;
-                }
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *trav = NULL;
+
+    /* look for unused iobuf from the head-most arena */
+    list_for_each_entry(trav, &iobuf_pool->arenas[index], list)
+    {
+        if (trav->passive_cnt) {
+            iobuf_arena = trav;
+            break;
         }
+    }
 
-        if (!iobuf_arena) {
-                /* all arenas were full */
-                iobuf_arena = __iobuf_pool_add_arena (iobuf_pool);
-        }
+    if (!iobuf_arena) {
+        /* all arenas were full, find the right count to add */
+        iobuf_arena = __iobuf_pool_add_arena(
+            iobuf_pool, page_size, gf_iobuf_init_config[index].num_pages,
+            index);
+    }
 
-        return iobuf_arena;
+    return iobuf_arena;
 }
 
-
-struct iobuf *
-__iobuf_ref (struct iobuf *iobuf)
+/* Always called under the iobuf_pool mutex lock */
+static struct iobuf *
+__iobuf_get(struct iobuf_pool *iobuf_pool, const size_t page_size,
+            const int index)
 {
-        iobuf->ref++;
+    struct iobuf *iobuf = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
 
-        return iobuf;
-}
+    /* most eligible arena for picking an iobuf */
+    iobuf_arena = __iobuf_select_arena(iobuf_pool, page_size, index);
+    if (!iobuf_arena)
+        return NULL;
 
+    list_for_each_entry(iobuf, &iobuf_arena->passive.list, list) break;
 
-struct iobuf *
-__iobuf_unref (struct iobuf *iobuf)
-{
-        iobuf->ref--;
+    list_del(&iobuf->list);
+    iobuf_arena->passive_cnt--;
 
-        return iobuf;
-}
+    list_add(&iobuf->list, &iobuf_arena->active.list);
+    iobuf_arena->active_cnt++;
 
+    /* no resetting requied for this element */
+    iobuf_arena->alloc_cnt++;
 
-struct iobuf *
-__iobuf_get (struct iobuf_arena *iobuf_arena)
-{
-        struct iobuf      *iobuf = NULL;
-        struct iobuf_pool *iobuf_pool = NULL;
+    if (iobuf_arena->max_active < iobuf_arena->active_cnt)
+        iobuf_arena->max_active = iobuf_arena->active_cnt;
 
-        iobuf_pool = iobuf_arena->iobuf_pool;
+    if (iobuf_arena->passive_cnt == 0) {
+        list_del(&iobuf_arena->list);
+        list_add(&iobuf_arena->list, &iobuf_pool->filled[index]);
+    }
 
-        list_for_each_entry (iobuf, &iobuf_arena->passive.list, list)
-                break;
+    return iobuf;
+}
 
-        list_del (&iobuf->list);
-        iobuf_arena->passive_cnt--;
+static struct iobuf *
+iobuf_get_from_stdalloc(struct iobuf_pool *iobuf_pool, const size_t page_size)
+{
+    struct iobuf *iobuf = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_arena *trav = NULL;
+    int ret = -1;
+
+    /* The first arena in the 'MAX-INDEX' will always be used for misc */
+    list_for_each_entry(trav, &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX], list)
+    {
+        iobuf_arena = trav;
+        break;
+    }
+
+    iobuf = GF_CALLOC(1, sizeof(*iobuf), gf_common_mt_iobuf);
+    if (!iobuf)
+        goto out;
+
+    /* 4096 is the alignment */
+    iobuf->free_ptr = GF_CALLOC(1, ((page_size + GF_IOBUF_ALIGN_SIZE) - 1),
+                                gf_common_mt_char);
+    if (!iobuf->free_ptr)
+        goto out;
+
+    iobuf->ptr = GF_ALIGN_BUF(iobuf->free_ptr, GF_IOBUF_ALIGN_SIZE);
+    iobuf->iobuf_arena = iobuf_arena;
+    LOCK_INIT(&iobuf->lock);
+
+    /* Hold a ref because you are allocating and using it */
+    GF_ATOMIC_INIT(iobuf->ref, 1);
+
+    ret = 0;
+out:
+    if (ret && iobuf) {
+        GF_FREE(iobuf->free_ptr);
+        GF_FREE(iobuf);
+        iobuf = NULL;
+    }
 
-        list_add (&iobuf->list, &iobuf_arena->active.list);
-        iobuf_arena->active_cnt++;
+    return iobuf;
+}
 
-        if (iobuf_arena->passive_cnt == 0) {
-                list_del (&iobuf_arena->list);
-                list_add (&iobuf_arena->list, &iobuf_pool->filled.list);
+struct iobuf *
+iobuf_get2(struct iobuf_pool *iobuf_pool, size_t page_size)
+{
+    struct iobuf *iobuf = NULL;
+    size_t rounded_size = 0;
+    int index = 0;
+
+    if (page_size == 0) {
+        page_size = iobuf_pool->default_page_size;
+    }
+
+    rounded_size = gf_iobuf_get_pagesize(page_size, &index);
+    if (rounded_size == -1) {
+        /* make sure to provide the requested buffer with standard
+           memory allocations */
+        iobuf = iobuf_get_from_stdalloc(iobuf_pool, page_size);
+
+        gf_msg_debug("iobuf", 0,
+                     "request for iobuf of size %zu "
+                     "is serviced using standard calloc() (%p) as it "
+                     "exceeds the maximum available buffer size",
+                     page_size, iobuf);
+
+        iobuf_pool->request_misses++;
+        return iobuf;
+    } else if (index == -1) {
+        gf_smsg("iobuf", GF_LOG_ERROR, 0, LG_MSG_PAGE_SIZE_EXCEEDED,
+                "page_size=%zu", page_size, NULL);
+        return NULL;
+    }
+
+    pthread_mutex_lock(&iobuf_pool->mutex);
+    {
+        iobuf = __iobuf_get(iobuf_pool, rounded_size, index);
+        if (!iobuf) {
+            pthread_mutex_unlock(&iobuf_pool->mutex);
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_IOBUF_NOT_FOUND,
+                    NULL);
+            goto post_unlock;
         }
 
-        return iobuf;
+        iobuf_ref(iobuf);
+    }
+    pthread_mutex_unlock(&iobuf_pool->mutex);
+post_unlock:
+    return iobuf;
 }
 
-
 struct iobuf *
-iobuf_get (struct iobuf_pool *iobuf_pool)
+iobuf_get_page_aligned(struct iobuf_pool *iobuf_pool, size_t page_size,
+                       size_t align_size)
 {
-        struct iobuf       *iobuf = NULL;
-        struct iobuf_arena *iobuf_arena = NULL;
-
-        pthread_mutex_lock (&iobuf_pool->mutex);
-        {
-                /* most eligible arena for picking an iobuf */
-                iobuf_arena = __iobuf_select_arena (iobuf_pool);
-                if (!iobuf_arena)
-                        goto unlock;
+    size_t req_size = 0;
+    struct iobuf *iobuf = NULL;
 
-                iobuf = __iobuf_get (iobuf_arena);
-                if (!iobuf)
-                        goto unlock;
+    req_size = page_size;
 
-                __iobuf_ref (iobuf);
-        }
-unlock:
-        pthread_mutex_unlock (&iobuf_pool->mutex);
+    if (req_size == 0) {
+        req_size = iobuf_pool->default_page_size;
+    }
 
-        return iobuf;
+    iobuf = iobuf_get2(iobuf_pool, req_size + align_size);
+    if (!iobuf)
+        return NULL;
+    /* If std allocation was used, then free_ptr will be non-NULL. In this
+     * case, we do not want to modify the original free_ptr.
+     * On the other hand, if the buf was gotten through the available
+     * arenas, then we use iobuf->free_ptr to store the original
+     * pointer to the offset into the mmap'd block of memory and in turn
+     * reuse iobuf->ptr to hold the page-aligned address. And finally, in
+     * iobuf_put(), we copy iobuf->free_ptr into iobuf->ptr - back to where
+     * it was originally when __iobuf_get() returned this iobuf.
+     */
+    if (!iobuf->free_ptr)
+        iobuf->free_ptr = iobuf->ptr;
+    iobuf->ptr = GF_ALIGN_BUF(iobuf->ptr, align_size);
+
+    return iobuf;
 }
 
-
-void
-__iobuf_put (struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
+struct iobuf *
+iobuf_get(struct iobuf_pool *iobuf_pool)
 {
-        struct iobuf_pool *iobuf_pool = NULL;
+    struct iobuf *iobuf = NULL;
+    int index = 0;
 
-        iobuf_pool = iobuf_arena->iobuf_pool;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
 
-        if (iobuf_arena->passive_cnt == 0) {
-                list_del (&iobuf_arena->list);
-                list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas.list);
+    index = gf_iobuf_get_arena_index(iobuf_pool->default_page_size);
+    if (index == -1) {
+        gf_smsg("iobuf", GF_LOG_ERROR, 0, LG_MSG_PAGE_SIZE_EXCEEDED,
+                "page_size=%zu", iobuf_pool->default_page_size, NULL);
+        return NULL;
+    }
+
+    pthread_mutex_lock(&iobuf_pool->mutex);
+    {
+        iobuf = __iobuf_get(iobuf_pool, iobuf_pool->default_page_size, index);
+        if (!iobuf) {
+            pthread_mutex_unlock(&iobuf_pool->mutex);
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_IOBUF_NOT_FOUND,
+                    NULL);
+            goto out;
         }
 
-        list_del_init (&iobuf->list);
-        iobuf_arena->active_cnt--;
+        iobuf_ref(iobuf);
+    }
+    pthread_mutex_unlock(&iobuf_pool->mutex);
 
-        list_add (&iobuf->list, &iobuf_arena->passive.list);
-        iobuf_arena->passive_cnt++;
-
-        if (iobuf_arena->active_cnt == 0) {
-                list_del (&iobuf_arena->list);
-                list_add_tail (&iobuf_arena->list, &iobuf_pool->purge.list);
-        }
+out:
+    return iobuf;
 }
 
+static void
+__iobuf_put(struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
+{
+    struct iobuf_pool *iobuf_pool = NULL;
+    int index = 0;
+
+    iobuf_pool = iobuf_arena->iobuf_pool;
+
+    index = gf_iobuf_get_arena_index(iobuf_arena->page_size);
+    if (index == -1) {
+        gf_msg_debug("iobuf", 0,
+                     "freeing the iobuf (%p) "
+                     "allocated with standard calloc()",
+                     iobuf);
+
+        /* free up properly without bothering about lists and all */
+        LOCK_DESTROY(&iobuf->lock);
+        GF_FREE(iobuf->free_ptr);
+        GF_FREE(iobuf);
+        return;
+    }
+
+    if (iobuf_arena->passive_cnt == 0) {
+        list_del(&iobuf_arena->list);
+        list_add_tail(&iobuf_arena->list, &iobuf_pool->arenas[index]);
+    }
+
+    list_del_init(&iobuf->list);
+    iobuf_arena->active_cnt--;
+
+    if (iobuf->free_ptr) {
+        iobuf->ptr = iobuf->free_ptr;
+        iobuf->free_ptr = NULL;
+    }
+
+    list_add(&iobuf->list, &iobuf_arena->passive.list);
+    iobuf_arena->passive_cnt++;
+
+    if (iobuf_arena->active_cnt == 0) {
+        list_del(&iobuf_arena->list);
+        list_add_tail(&iobuf_arena->list, &iobuf_pool->purge[index]);
+        GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
+        __iobuf_arena_prune(iobuf_pool, iobuf_arena, index);
+    }
+out:
+    return;
+}
 
 void
-iobuf_put (struct iobuf *iobuf)
+iobuf_put(struct iobuf *iobuf)
 {
-        struct iobuf_arena *iobuf_arena = NULL;
-        struct iobuf_pool  *iobuf_pool = NULL;
+    struct iobuf_arena *iobuf_arena = NULL;
+    struct iobuf_pool *iobuf_pool = NULL;
 
-        if (!iobuf)
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
 
-        iobuf_arena = iobuf->iobuf_arena;
-        if (!iobuf_arena)
-                return;
+    iobuf_arena = iobuf->iobuf_arena;
+    if (!iobuf_arena) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND, NULL);
+        return;
+    }
 
-        iobuf_pool = iobuf_arena->iobuf_pool;
-        if (!iobuf_pool)
-                return;
+    iobuf_pool = iobuf_arena->iobuf_pool;
+    if (!iobuf_pool) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_POOL_NOT_FOUND, "iobuf",
+                NULL);
+        return;
+    }
 
-        pthread_mutex_lock (&iobuf_pool->mutex);
-        {
-                __iobuf_put (iobuf, iobuf_arena);
-        }
-        pthread_mutex_unlock (&iobuf_pool->mutex);
+    pthread_mutex_lock(&iobuf_pool->mutex);
+    {
+        __iobuf_put(iobuf, iobuf_arena);
+    }
+    pthread_mutex_unlock(&iobuf_pool->mutex);
 
-        iobuf_pool_prune (iobuf_pool);
+out:
+    return;
 }
 
-
 void
-iobuf_unref (struct iobuf *iobuf)
+iobuf_unref(struct iobuf *iobuf)
 {
-        int  ref = 0;
+    int ref = 0;
 
-        if (!iobuf)
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
 
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_unref (iobuf);
-                ref = iobuf->ref;
-        }
-        UNLOCK (&iobuf->lock);
+    ref = GF_ATOMIC_DEC(iobuf->ref);
 
-        if (!ref)
-                iobuf_put (iobuf);
-}
+    if (!ref)
+        iobuf_put(iobuf);
 
+out:
+    return;
+}
 
 struct iobuf *
-iobuf_ref (struct iobuf *iobuf)
+iobuf_ref(struct iobuf *iobuf)
 {
-        if (!iobuf)
-                return NULL;
-
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_ref (iobuf);
-        }
-        UNLOCK (&iobuf->lock);
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
+    GF_ATOMIC_INC(iobuf->ref);
 
-        return iobuf;
+out:
+    return iobuf;
 }
 
-
 struct iobref *
-iobref_new ()
+iobref_new()
 {
-        struct iobref *iobref = NULL;
+    struct iobref *iobref = NULL;
 
-        iobref = GF_CALLOC (sizeof (*iobref), 1,
-                            gf_common_mt_iobref);
-        if (!iobref)
-                return NULL;
+    iobref = GF_MALLOC(sizeof(*iobref), gf_common_mt_iobref);
+    if (!iobref)
+        return NULL;
 
-        LOCK_INIT (&iobref->lock);
+    iobref->iobrefs = GF_CALLOC(sizeof(*iobref->iobrefs), 16,
+                                gf_common_mt_iobrefs);
+    if (!iobref->iobrefs) {
+        GF_FREE(iobref);
+        return NULL;
+    }
 
-        iobref->ref++;
+    iobref->allocated = 16;
+    iobref->used = 0;
 
-        return iobref;
-}
+    LOCK_INIT(&iobref->lock);
 
+    GF_ATOMIC_INIT(iobref->ref, 1);
+    return iobref;
+}
 
 struct iobref *
-iobref_ref (struct iobref *iobref)
+iobref_ref(struct iobref *iobref)
 {
-        if (!iobref)
-                return NULL;
-
-        LOCK (&iobref->lock);
-        {
-                iobref->ref++;
-        }
-        UNLOCK (&iobref->lock);
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
+    GF_ATOMIC_INC(iobref->ref);
 
-        return iobref;
+out:
+    return iobref;
 }
 
-
 void
-iobref_destroy (struct iobref *iobref)
+iobref_destroy(struct iobref *iobref)
 {
-        int            i = 0;
-        struct iobuf  *iobuf = NULL;
+    int i = 0;
+    struct iobuf *iobuf = NULL;
 
-        if (!iobref)
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
 
-        for (i = 0; i < 8; i++) {
-                iobuf = iobref->iobrefs[i];
+    for (i = 0; i < iobref->allocated; i++) {
+        iobuf = iobref->iobrefs[i];
 
-                iobref->iobrefs[i] = NULL;
-                if (iobuf)
-                        iobuf_unref (iobuf);
-        }
+        iobref->iobrefs[i] = NULL;
+        if (iobuf)
+            iobuf_unref(iobuf);
+    }
 
-        GF_FREE (iobref);
-}
+    GF_FREE(iobref->iobrefs);
+    GF_FREE(iobref);
 
+out:
+    return;
+}
 
 void
-iobref_unref (struct iobref *iobref)
+iobref_unref(struct iobref *iobref)
 {
-        int ref = 0;
+    int ref = 0;
 
-        if (!iobref)
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
+    ref = GF_ATOMIC_DEC(iobref->ref);
 
-        LOCK (&iobref->lock);
-        {
-                ref = (--iobref->ref);
-        }
-        UNLOCK (&iobref->lock);
+    if (!ref)
+        iobref_destroy(iobref);
 
-        if (!ref)
-                iobref_destroy (iobref);
+out:
+    return;
 }
 
-
-int
-__iobref_add (struct iobref *iobref, struct iobuf *iobuf)
+void
+iobref_clear(struct iobref *iobref)
 {
-        int  i = 0;
-        int  ret = -ENOMEM;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
 
-        for (i = 0; i < 8; i++) {
-                if (iobref->iobrefs[i] == NULL) {
-                        iobref->iobrefs[i] = iobuf_ref (iobuf);
-                        ret = 0;
-                        break;
-                }
+    for (; i < iobref->allocated; i++) {
+        if (iobref->iobrefs[i] != NULL) {
+            iobuf_unref(iobref->iobrefs[i]);
+        } else {
+            /** iobuf's are attached serially */
+            break;
         }
+    }
+
+    iobref_unref(iobref);
 
-        return ret;
+out:
+    return;
 }
 
+static void
+__iobref_grow(struct iobref *iobref)
+{
+    void *newptr = NULL;
+    int i = 0;
+
+    newptr = GF_REALLOC(iobref->iobrefs,
+                        iobref->allocated * 2 * (sizeof(*iobref->iobrefs)));
+    if (newptr) {
+        iobref->iobrefs = newptr;
+        iobref->allocated *= 2;
+
+        for (i = iobref->used; i < iobref->allocated; i++)
+            iobref->iobrefs[i] = NULL;
+    }
+}
 
 int
-iobref_add (struct iobref *iobref, struct iobuf *iobuf)
+__iobref_add(struct iobref *iobref, struct iobuf *iobuf)
 {
-        int  ret = 0;
+    int i = 0;
+    int ret = -ENOMEM;
 
-        if (!iobref)
-                return -EINVAL;
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
 
-        if (!iobuf)
-                return -EINVAL;
+    if (iobref->used == iobref->allocated) {
+        __iobref_grow(iobref);
 
-        LOCK (&iobref->lock);
-        {
-                ret = __iobref_add (iobref, iobuf);
+        if (iobref->used == iobref->allocated) {
+            ret = -ENOMEM;
+            goto out;
         }
-        UNLOCK (&iobref->lock);
+    }
+
+    for (i = 0; i < iobref->allocated; i++) {
+        if (iobref->iobrefs[i] == NULL) {
+            iobref->iobrefs[i] = iobuf_ref(iobuf);
+            iobref->used++;
+            ret = 0;
+            break;
+        }
+    }
 
-        return ret;
+out:
+    return ret;
 }
 
+int
+iobref_add(struct iobref *iobref, struct iobuf *iobuf)
+{
+    int ret = -EINVAL;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
+
+    LOCK(&iobref->lock);
+    {
+        ret = __iobref_add(iobref, iobuf);
+    }
+    UNLOCK(&iobref->lock);
+
+out:
+    return ret;
+}
 
 int
-iobref_merge (struct iobref *to, struct iobref *from)
+iobref_merge(struct iobref *to, struct iobref *from)
 {
-        int           i = 0;
-        int           ret = 0;
-        struct iobuf *iobuf = NULL;
+    int i = 0;
+    int ret = 0;
+    struct iobuf *iobuf = NULL;
 
-        LOCK (&from->lock);
-        {
-                for (i = 0; i < 8; i++) {
-                        iobuf = from->iobrefs[i];
+    GF_VALIDATE_OR_GOTO("iobuf", to, out);
+    GF_VALIDATE_OR_GOTO("iobuf", from, out);
+
+    LOCK(&from->lock);
+    {
+        for (i = 0; i < from->allocated; i++) {
+            iobuf = from->iobrefs[i];
 
-                        if (!iobuf)
-                                break;
+            if (!iobuf)
+                break;
 
-                        ret = iobref_add (to, iobuf);
+            ret = iobref_add(to, iobuf);
 
-                        if (ret < 0)
-                                break;
-                }
+            if (ret < 0)
+                break;
         }
-        UNLOCK (&from->lock);
+    }
+    UNLOCK(&from->lock);
 
-        return ret;
+out:
+    return ret;
 }
 
-
 size_t
-iobuf_size (struct iobuf *iobuf)
+iobuf_size(struct iobuf *iobuf)
 {
-        size_t size = 0;
+    size_t size = 0;
 
-        if (!iobuf)
-                goto out;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
 
-        if (!iobuf->iobuf_arena)
-                goto out;
+    if (!iobuf->iobuf_arena) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_ARENA_NOT_FOUND, NULL);
+        goto out;
+    }
 
-        if (!iobuf->iobuf_arena->iobuf_pool)
-                goto out;
+    if (!iobuf->iobuf_arena->iobuf_pool) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, LG_MSG_POOL_NOT_FOUND, NULL);
+        goto out;
+    }
 
-        size = iobuf->iobuf_arena->iobuf_pool->page_size;
+    size = iobuf->iobuf_arena->page_size;
 out:
-        return size;
+    return size;
 }
 
-
 size_t
-iobref_size (struct iobref *iobref)
+iobref_size(struct iobref *iobref)
 {
-        size_t size = 0;
-        int    i = 0;
+    size_t size = 0;
+    int i = 0;
 
-        if (!iobref)
-                goto out;
+    GF_VALIDATE_OR_GOTO("iobuf", iobref, out);
 
-        LOCK (&iobref->lock);
-        {
-                for (i = 0; i < 8; i++) {
-                        if (iobref->iobrefs[i])
-                                size += iobuf_size (iobref->iobrefs[i]);
-                }
+    LOCK(&iobref->lock);
+    {
+        for (i = 0; i < iobref->allocated; i++) {
+            if (iobref->iobrefs[i])
+                size += iobuf_size(iobref->iobrefs[i]);
         }
-        UNLOCK (&iobref->lock);
+    }
+    UNLOCK(&iobref->lock);
+
 out:
-        return size;
+    return size;
 }
 
-void 
-iobuf_info_dump (struct iobuf *iobuf, const char *key_prefix)
+void
+iobuf_info_dump(struct iobuf *iobuf, const char *key_prefix)
 {
-        char   key[GF_DUMP_MAX_BUF_LEN];
-        struct iobuf my_iobuf;
-        int    ret = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    struct iobuf my_iobuf;
+    int ret = 0;
 
-        if (!iobuf) 
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf, out);
 
-        memset(&my_iobuf, 0, sizeof(my_iobuf));
-        
-        ret = TRY_LOCK(&iobuf->lock);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump iobuf"
-                " errno: %d", errno);
-                return;
-        }
-        memcpy(&my_iobuf, iobuf, sizeof(my_iobuf));
-        UNLOCK(&iobuf->lock);
+    ret = TRY_LOCK(&iobuf->lock);
+    if (ret) {
+        return;
+    }
+    memcpy(&my_iobuf, iobuf, sizeof(my_iobuf));
+    UNLOCK(&iobuf->lock);
 
-	gf_proc_dump_build_key(key, key_prefix,"ref");
-        gf_proc_dump_write(key, "%d", my_iobuf.ref);
-	gf_proc_dump_build_key(key, key_prefix,"ptr");
-        gf_proc_dump_write(key, "%p", my_iobuf.ptr);
+    gf_proc_dump_build_key(key, key_prefix, "ref");
+    gf_proc_dump_write(key, "%" GF_PRI_ATOMIC, GF_ATOMIC_GET(my_iobuf.ref));
+    gf_proc_dump_build_key(key, key_prefix, "ptr");
+    gf_proc_dump_write(key, "%p", my_iobuf.ptr);
 
+out:
+    return;
 }
 
-void 
-iobuf_arena_info_dump (struct iobuf_arena *iobuf_arena, const char *key_prefix)
+void
+iobuf_arena_info_dump(struct iobuf_arena *iobuf_arena, const char *key_prefix)
 {
-	char key[GF_DUMP_MAX_BUF_LEN];
-	int  i = 1;
-        struct iobuf *trav;
-
-	if (!iobuf_arena)
-                return;
-
-        gf_proc_dump_build_key(key, key_prefix,"mem_base");
-        gf_proc_dump_write(key, "%p", iobuf_arena->mem_base);
-	gf_proc_dump_build_key(key, key_prefix, "active_cnt");
-        gf_proc_dump_write(key, "%d", iobuf_arena->active_cnt);
-	gf_proc_dump_build_key(key, key_prefix, "passive_cnt");
-        gf_proc_dump_write(key, "%d", iobuf_arena->passive_cnt);
-	list_for_each_entry (trav, &iobuf_arena->active.list, list) {
-                gf_proc_dump_build_key(key, key_prefix,"active_iobuf.%d", i++);
-                gf_proc_dump_add_section(key);
-                iobuf_info_dump(trav, key);
-        }
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 1;
+    struct iobuf *trav;
+
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_arena, out);
+
+    gf_proc_dump_build_key(key, key_prefix, "mem_base");
+    gf_proc_dump_write(key, "%p", iobuf_arena->mem_base);
+    gf_proc_dump_build_key(key, key_prefix, "active_cnt");
+    gf_proc_dump_write(key, "%d", iobuf_arena->active_cnt);
+    gf_proc_dump_build_key(key, key_prefix, "passive_cnt");
+    gf_proc_dump_write(key, "%d", iobuf_arena->passive_cnt);
+    gf_proc_dump_build_key(key, key_prefix, "alloc_cnt");
+    gf_proc_dump_write(key, "%" PRIu64, iobuf_arena->alloc_cnt);
+    gf_proc_dump_build_key(key, key_prefix, "max_active");
+    gf_proc_dump_write(key, "%d", iobuf_arena->max_active);
+    gf_proc_dump_build_key(key, key_prefix, "page_size");
+    gf_proc_dump_write(key, "%" GF_PRI_SIZET, iobuf_arena->page_size);
+    list_for_each_entry(trav, &iobuf_arena->active.list, list)
+    {
+        gf_proc_dump_build_key(key, key_prefix, "active_iobuf.%d", i++);
+        gf_proc_dump_add_section("%s", key);
+        iobuf_info_dump(trav, key);
+    }
 
+out:
+    return;
 }
 
 void
-iobuf_stats_dump (struct iobuf_pool *iobuf_pool)
+iobuf_stats_dump(struct iobuf_pool *iobuf_pool)
 {
-    
-        char               msg[1024];
-        struct iobuf_arena *trav = NULL;
-        int                i = 1;
-        int                ret = -1;
+    char msg[1024];
+    struct iobuf_arena *trav = NULL;
+    int i = 1;
+    int j = 0;
+    int ret = -1;
 
-        if (!iobuf_pool)
-                return;
+    GF_VALIDATE_OR_GOTO("iobuf", iobuf_pool, out);
 
-        memset(msg, 0, sizeof(msg));
+    ret = pthread_mutex_trylock(&iobuf_pool->mutex);
 
-        ret = pthread_mutex_trylock(&iobuf_pool->mutex);
-
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump iobuf pool"
-                " errno: %d", errno);
-                return;
+    if (ret) {
+        return;
+    }
+    gf_proc_dump_add_section("iobuf.global");
+    gf_proc_dump_write("iobuf_pool", "%p", iobuf_pool);
+    gf_proc_dump_write("iobuf_pool.default_page_size", "%" GF_PRI_SIZET,
+                       iobuf_pool->default_page_size);
+    gf_proc_dump_write("iobuf_pool.arena_size", "%" GF_PRI_SIZET,
+                       iobuf_pool->arena_size);
+    gf_proc_dump_write("iobuf_pool.arena_cnt", "%d", iobuf_pool->arena_cnt);
+    gf_proc_dump_write("iobuf_pool.request_misses", "%" PRId64,
+                       iobuf_pool->request_misses);
+
+    for (j = 0; j < IOBUF_ARENA_MAX_INDEX; j++) {
+        list_for_each_entry(trav, &iobuf_pool->arenas[j], list)
+        {
+            snprintf(msg, sizeof(msg), "arena.%d", i);
+            gf_proc_dump_add_section("%s", msg);
+            iobuf_arena_info_dump(trav, msg);
+            i++;
         }
-        gf_proc_dump_add_section("iobuf.global");
-        gf_proc_dump_write("iobuf.global.iobuf_pool","%p", iobuf_pool);
-        gf_proc_dump_write("iobuf.global.iobuf_pool.page_size", "%d",
-						 iobuf_pool->page_size);
-        gf_proc_dump_write("iobuf.global.iobuf_pool.arena_size", "%d",
-						 iobuf_pool->arena_size);
-        gf_proc_dump_write("iobuf.global.iobuf_pool.arena_cnt", "%d",
-						 iobuf_pool->arena_cnt);
-
-        list_for_each_entry (trav, &iobuf_pool->arenas.list, list) {
-                snprintf(msg, sizeof(msg), "iobuf.global.iobuf_pool.arena.%d",
-                                                                            i);
-		gf_proc_dump_add_section(msg);
-                iobuf_arena_info_dump(trav,msg);
-                i++;
+        list_for_each_entry(trav, &iobuf_pool->purge[j], list)
+        {
+            snprintf(msg, sizeof(msg), "purge.%d", i);
+            gf_proc_dump_add_section("%s", msg);
+            iobuf_arena_info_dump(trav, msg);
+            i++;
+        }
+        list_for_each_entry(trav, &iobuf_pool->filled[j], list)
+        {
+            snprintf(msg, sizeof(msg), "filled.%d", i);
+            gf_proc_dump_add_section("%s", msg);
+            iobuf_arena_info_dump(trav, msg);
+            i++;
         }
-        
-        pthread_mutex_unlock(&iobuf_pool->mutex);
+    }
 
-        return;
-}
+    pthread_mutex_unlock(&iobuf_pool->mutex);
 
+out:
+    return;
+}
 
 void
 iobuf_to_iovec(struct iobuf *iob, struct iovec *iov)
 {
-        iov->iov_base = iobuf_ptr (iob);
-        iov->iov_len =  iobuf_pagesize (iob);
+    GF_VALIDATE_OR_GOTO("iobuf", iob, out);
+    GF_VALIDATE_OR_GOTO("iobuf", iov, out);
+
+    iov->iov_base = iobuf_ptr(iob);
+    iov->iov_len = iobuf_pagesize(iob);
+
+out:
+    return;
 }
 
+int
+iobuf_copy(struct iobuf_pool *iobuf_pool, const struct iovec *iovec_src,
+           int iovcnt, struct iobref **iobref, struct iobuf **iobuf,
+           struct iovec *iov_dst)
+{
+    size_t size = -1;
+    int ret = 0;
+
+    size = iov_length(iovec_src, iovcnt);
+
+    *iobuf = iobuf_get2(iobuf_pool, size);
+    if (!(*iobuf)) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    *iobref = iobref_new();
+    if (!(*iobref)) {
+        iobuf_unref(*iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(*iobref, *iobuf);
+    if (ret) {
+        iobuf_unref(*iobuf);
+        iobref_unref(*iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(*iobuf), iovec_src, iovcnt);
+
+    iov_dst->iov_base = iobuf_ptr(*iobuf);
+    iov_dst->iov_len = size;
+
+out:
+    return ret;
+}
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
deleted file mode 100644
index fc8ff623ad4..00000000000
--- a/libglusterfs/src/iobuf.h
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _IOBUF_H_
-#define _IOBUF_H_
-
-#include "list.h"
-#include "common-utils.h"
-#include <pthread.h>
-#include <sys/mman.h>
-#include <sys/uio.h>
-
-/* Lets try to define the new anonymous mapping
- * flag, in case the system is still using the
- * now deprecated MAP_ANON flag.
- *
- * Also, this should ideally be in a centralized/common
- * header which can be used by other source files also.
- */
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-/* one allocatable unit for the consumers of the IOBUF API */
-/* each unit hosts @page_size bytes of memory */
-struct iobuf;
-
-/* one region of memory MMAPed from the operating system */
-/* each region MMAPs @arena_size bytes of memory */
-/* each arena hosts @arena_size / @page_size IOBUFs */
-struct iobuf_arena;
-
-/* expandable and contractable pool of memory, internally broken into arenas */
-struct iobuf_pool;
-
-
-struct iobuf {
-        union {
-                struct list_head      list;
-                struct {
-                        struct iobuf *next;
-                        struct iobuf *prev;
-                };
-        };
-        struct iobuf_arena  *iobuf_arena;
-
-        gf_lock_t            lock; /* for ->ptr and ->ref */
-        int                  ref;  /* 0 == passive, >0 == active */
-
-        void                *ptr;  /* usable memory region by the consumer */
-};
-
-
-struct iobuf_arena {
-        union {
-                struct list_head            list;
-                struct {
-                        struct iobuf_arena *next;
-                        struct iobuf_arena *prev;
-                };
-        };
-        struct iobuf_pool  *iobuf_pool;
-
-        void               *mem_base;
-        struct iobuf       *iobufs;     /* allocated iobufs list */
-
-        int                 active_cnt;
-        struct iobuf        active;     /* head node iobuf
-                                           (unused by itself) */
-        int                 passive_cnt;
-        struct iobuf        passive;    /* head node iobuf
-                                           (unused by itself) */
-};
-
-
-struct iobuf_pool {
-        pthread_mutex_t     mutex;
-        size_t              page_size;  /* size of all iobufs in this pool */
-        size_t              arena_size; /* size of memory region in arena */
-
-        int                 arena_cnt;
-        struct iobuf_arena  arenas;     /* head node arena
-                                           (unused by itself) */
-        struct iobuf_arena  filled;     /* arenas without  free iobufs */
-        struct iobuf_arena  purge;      /* arenas which can be purged */
-};
-
-
-
-
-struct iobuf_pool *iobuf_pool_new (size_t arena_size, size_t page_size);
-void iobuf_pool_destroy (struct iobuf_pool *iobuf_pool);
-struct iobuf *iobuf_get (struct iobuf_pool *iobuf_pool);
-void iobuf_unref (struct iobuf *iobuf);
-struct iobuf *iobuf_ref (struct iobuf *iobuf);
-void iobuf_pool_destroy (struct iobuf_pool *iobuf_pool);
-void iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
-
-#define iobuf_ptr(iob) ((iob)->ptr)
-#define iobpool_pagesize(iobpool) ((iobpool)->page_size)
-#define iobuf_pagesize(iob) (iobpool_pagesize((iob)->iobuf_arena->iobuf_pool))
-
-
-struct iobref {
-        gf_lock_t          lock;
-        int                ref;
-        struct iobuf      *iobrefs[8];
-};
-
-struct iobref *iobref_new ();
-struct iobref *iobref_ref (struct iobref *iobref);
-void iobref_unref (struct iobref *iobref);
-int iobref_add (struct iobref *iobref, struct iobuf *iobuf);
-int iobref_merge (struct iobref *to, struct iobref *from);
-
-
-size_t iobuf_size (struct iobuf *iobuf);
-size_t iobref_size (struct iobref *iobref);
-void   iobuf_stats_dump (struct iobuf_pool *iobuf_pool);
-
-#endif /* !_IOBUF_H_ */
diff --git a/libglusterfs/src/latency.c b/libglusterfs/src/latency.c
index bc2409138c4..ce4b0e8255d 100644
--- a/libglusterfs/src/latency.c
+++ b/libglusterfs/src/latency.c
@@ -1,183 +1,84 @@
 /*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 /*
  * This file contains functions to support dumping of
  * latencies of FOPs broken down by subvolumes.
  */
 
-#include "glusterfs.h"
-#include "stack.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "statedump.h"
-
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/statedump.h"
 
-void
-gf_set_fop_from_fn_pointer (call_frame_t *frame, struct xlator_fops *fops, void *fn)
+gf_latency_t *
+gf_latency_new(size_t n)
 {
-        glusterfs_fop_t fop = -1;
+    int i = 0;
+    gf_latency_t *lat = NULL;
 
-        if (fops->stat == fn)
-                fop = GF_FOP_STAT;
-        else if (fops->readlink == fn)
-                fop = GF_FOP_READLINK;
-        else if (fops->mknod == fn)
-                fop = GF_FOP_MKNOD;
-        else if (fops->mkdir == fn)
-                fop = GF_FOP_MKDIR;
-        else if (fops->unlink == fn)
-                fop = GF_FOP_UNLINK;
-        else if (fops->rmdir == fn)
-                fop = GF_FOP_RMDIR;
-        else if (fops->symlink == fn)
-                fop = GF_FOP_SYMLINK;
-        else if (fops->rename == fn)
-                fop = GF_FOP_RENAME;
-        else if (fops->link == fn)
-                fop = GF_FOP_LINK;
-        else if (fops->truncate == fn)
-                fop = GF_FOP_TRUNCATE;
-        else if (fops->open == fn)
-                fop = GF_FOP_OPEN;
-        else if (fops->readv == fn)
-                fop = GF_FOP_READ;
-        else if (fops->writev == fn)
-                fop = GF_FOP_WRITE;
-        else if (fops->statfs == fn)
-                fop = GF_FOP_STATFS;
-        else if (fops->flush == fn)
-                fop = GF_FOP_FLUSH;
-        else if (fops->fsync == fn)
-                fop = GF_FOP_FSYNC;
-        else if (fops->setxattr == fn)
-                fop = GF_FOP_SETXATTR;
-        else if (fops->getxattr == fn)
-                fop = GF_FOP_GETXATTR;
-        else if (fops->removexattr == fn)
-                fop = GF_FOP_REMOVEXATTR;
-        else if (fops->opendir == fn)
-                fop = GF_FOP_OPENDIR;
-        else if (fops->fsyncdir == fn)
-                fop = GF_FOP_FSYNCDIR;
-        else if (fops->access == fn)
-                fop = GF_FOP_ACCESS;
-        else if (fops->create == fn)
-                fop = GF_FOP_CREATE;
-        else if (fops->ftruncate == fn)
-                fop = GF_FOP_FTRUNCATE;
-        else if (fops->fstat == fn)
-                fop = GF_FOP_FSTAT;
-        else if (fops->lk == fn)
-                fop = GF_FOP_LK;
-        else if (fops->lookup == fn)
-                fop = GF_FOP_LOOKUP;
-        else if (fops->readdir == fn)
-                fop = GF_FOP_READDIR;
-        else if (fops->inodelk == fn)
-                fop = GF_FOP_INODELK;
-        else if (fops->finodelk == fn)
-                fop = GF_FOP_FINODELK;
-	else if (fops->entrylk == fn)
-                fop = GF_FOP_ENTRYLK;
-	else if (fops->fentrylk == fn)
-                fop = GF_FOP_FENTRYLK;
-        else if (fops->xattrop == fn)
-                fop = GF_FOP_XATTROP;
-        else if (fops->fxattrop == fn)
-                fop = GF_FOP_FXATTROP;
-        else if (fops->fgetxattr == fn)
-                fop = GF_FOP_FGETXATTR;
-        else if (fops->fsetxattr == fn)
-                fop = GF_FOP_FSETXATTR;
-        else if (fops->rchecksum == fn)
-                fop = GF_FOP_RCHECKSUM;
-        else if (fops->setattr == fn)
-                fop = GF_FOP_SETATTR;
-        else if (fops->fsetattr == fn)
-                fop = GF_FOP_FSETATTR;
-        else if (fops->readdirp == fn)
-                fop = GF_FOP_READDIRP;
-        else if (fops->getspec == fn)
-                fop = GF_FOP_GETSPEC;
-        else
-                fop = -1;
+    lat = GF_MALLOC(n * sizeof(*lat), gf_common_mt_latency_t);
+    if (!lat)
+        return NULL;
 
-        frame->op   = fop;
+    for (i = 0; i < n; i++) {
+        gf_latency_reset(lat + i);
+    }
+    return lat;
 }
 
-
 void
-gf_update_latency (call_frame_t *frame)
+gf_latency_update(gf_latency_t *lat, struct timespec *begin,
+                  struct timespec *end)
 {
-        double elapsed;
-        struct timeval *begin, *end;
-
-        fop_latency_t *lat;
+    if (!(begin->tv_sec && end->tv_sec)) {
+        /*Measure latency might have been enabled/disabled during the op*/
+        return;
+    }
 
-        begin = &frame->begin;
-        end   = &frame->end;
+    double elapsed = gf_tsdiff(begin, end);
 
-        elapsed = (end->tv_sec - begin->tv_sec) * 1e6
-                + (end->tv_usec - begin->tv_usec);
+    if (lat->max < elapsed)
+        lat->max = elapsed;
 
-        lat = &frame->this->latencies[frame->op];
+    if (lat->min > elapsed)
+        lat->min = elapsed;
 
-	lat->total += elapsed;
-        lat->count++;
-        lat->mean = lat->mean + (elapsed - lat->mean) / lat->count;
+    lat->total += elapsed;
+    lat->count++;
 }
 
-
 void
-gf_proc_dump_latency_info (xlator_t *xl)
+gf_latency_reset(gf_latency_t *lat)
 {
-        char key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char key[GF_DUMP_MAX_BUF_LEN];
-        int i;
-
-        snprintf (key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.latency", xl->name);
-        gf_proc_dump_add_section (key_prefix);
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                gf_proc_dump_build_key (key, key_prefix, gf_fop_list[i]);
-
-                gf_proc_dump_write (key, "%.03f,%"PRId64",%.03f",
-                                    xl->latencies[i].mean,
-                                    xl->latencies[i].count,
-				    xl->latencies[i].total);
-        }
+    if (!lat)
+        return;
+    memset(lat, 0, sizeof(*lat));
+    lat->min = ULLONG_MAX;
+    /* make sure 'min' is set to high value, so it would be
+       properly set later */
 }
 
-
 void
-gf_latency_toggle (int signum)
+gf_frame_latency_update(call_frame_t *frame)
 {
-        glusterfs_ctx_t *ctx = NULL;
-
-        ctx = glusterfs_ctx_get ();
-
-        if (ctx) {
-                ctx->measure_latency = !ctx->measure_latency;
-                gf_log ("[core]", GF_LOG_NORMAL,
-                        "Latency measurement turned %s",
-                        ctx->measure_latency ? "on" : "off");
-        }
+    gf_latency_t *lat;
+    /* Can happen mostly at initiator xlator, as STACK_WIND/UNWIND macros
+       set it right anyways for those frames */
+    if (!frame->op)
+        frame->op = frame->root->op;
+
+    if (frame->op < 0 || frame->op >= GF_FOP_MAXVALUE) {
+        gf_log("[core]", GF_LOG_WARNING, "Invalid frame op value: %d",
+               frame->op);
+        return;
+    }
+
+    lat = &frame->this->stats.interval.latencies[frame->op];
+    gf_latency_update(lat, &frame->begin, &frame->end);
 }
diff --git a/libglusterfs/src/latency.h b/libglusterfs/src/latency.h
deleted file mode 100644
index 610f125508d..00000000000
--- a/libglusterfs/src/latency.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __LATENCY_H__
-#define __LATENCY_H__
-
-
-typedef struct fop_latency {
-        uint64_t min;           /* min time for the call (microseconds) */
-        uint64_t max;           /* max time for the call (microseconds) */
-	double total;           /* total time (microseconds) */
-        double std;             /* standard deviation */
-        double mean;            /* mean (microseconds) */
-        uint64_t count;
-} fop_latency_t;
-
-void
-gf_latency_toggle (int signum);
-
-#endif /* __LATENCY_H__ */
diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym
new file mode 100644
index 00000000000..5f18cd56cbe
--- /dev/null
+++ b/libglusterfs/src/libglusterfs.sym
@@ -0,0 +1,1193 @@
+are_dicts_equal
+args_access_cbk_store
+args_access_store
+args_create_cbk_store
+args_create_store
+args_discard_cbk_store
+args_discard_store
+args_entrylk_cbk_store
+args_entrylk_store
+args_fallocate_cbk_store
+args_fallocate_store
+args_fentrylk_cbk_store
+args_fentrylk_store
+args_fgetxattr_cbk_store
+args_fgetxattr_store
+args_finodelk_cbk_store
+args_finodelk_store
+args_flush_cbk_store
+args_flush_store
+args_fremovexattr_cbk_store
+args_fremovexattr_store
+args_fsetattr_cbk_store
+args_fsetattr_store
+args_fsetxattr_cbk_store
+args_fsetxattr_store
+args_fstat_cbk_store
+args_fstat_store
+args_fsync_cbk_store
+args_fsyncdir_cbk_store
+args_fsyncdir_store
+args_fsync_store
+args_ftruncate_cbk_store
+args_ftruncate_store
+args_fxattrop_cbk_store
+args_fxattrop_store
+args_getxattr_cbk_store
+args_getxattr_store
+args_inodelk_cbk_store
+args_inodelk_store
+args_ipc_cbk_store
+args_lease_cbk_store
+args_lease_store
+args_link_cbk_store
+args_link_store
+args_lk_cbk_store
+args_lk_store
+args_lookup_cbk_store
+args_lookup_store
+args_mkdir_cbk_store
+args_mkdir_store
+args_mknod_cbk_store
+args_mknod_store
+args_open_cbk_store
+args_opendir_cbk_store
+args_opendir_store
+args_open_store
+args_rchecksum_cbk_store
+args_rchecksum_store
+args_readdir_cbk_store
+args_readdirp_cbk_store
+args_readdirp_store
+args_readdir_store
+args_readlink_cbk_store
+args_readlink_store
+args_readv_cbk_store
+args_readv_store
+args_removexattr_cbk_store
+args_removexattr_store
+args_rename_cbk_store
+args_rename_store
+args_rmdir_cbk_store
+args_rmdir_store
+args_seek_cbk_store
+args_seek_store
+args_setattr_cbk_store
+args_setattr_store
+args_setxattr_cbk_store
+args_setxattr_store
+args_stat_cbk_store
+args_statfs_cbk_store
+args_statfs_store
+args_stat_store
+args_symlink_cbk_store
+args_symlink_store
+args_truncate_cbk_store
+args_truncate_store
+args_unlink_cbk_store
+args_unlink_store
+args_writev_cbk_store
+args_writev_store
+args_xattrop_cbk_store
+args_xattrop_store
+args_zerofill_cbk_store
+args_zerofill_store
+args_copy_file_range_cbk_store
+args_copy_file_range_store
+bin_to_data
+call_resume
+call_resume_keep_stub
+call_resume_wind
+call_stack_set_groups
+call_stub_destroy
+call_unwind_error
+call_unwind_error_keep_stub
+client_ctx_del
+client_ctx_get
+client_ctx_set
+client_dump
+close_fds_except
+cluster_fop_success_fill
+cluster_fstat
+cluster_ftruncate
+cluster_fxattrop
+cluster_getxattr
+cluster_inodelk
+cluster_link
+cluster_lookup
+cluster_mkdir
+cluster_mknod
+cluster_open
+cluster_readlink
+cluster_replies_wipe
+cluster_rmdir
+cluster_setattr
+cluster_setxattr
+cluster_symlink
+cluster_tiebreaker_inodelk
+cluster_uninodelk
+cluster_unlink
+cluster_xattrop
+cluster_xattrop_cbk
+copy_opts_to_child
+create_frame
+data_copy
+data_destroy
+data_from_dynptr
+data_from_uint64
+data_ref
+data_to_bin
+data_to_int32
+data_to_int64
+data_to_ptr
+data_to_str
+data_to_uint16
+data_to_uint32
+data_to_uint64
+data_to_uint8
+data_to_iatt
+data_unref
+default_access
+default_access_cbk
+default_access_failure_cbk
+default_access_resume
+default_create
+default_create_cbk
+default_create_failure_cbk
+default_create_resume
+default_discard
+default_discard_cbk
+default_discard_failure_cbk
+default_discard_resume
+default_entrylk
+default_entrylk_cbk
+default_entrylk_failure_cbk
+default_entrylk_resume
+default_fallocate
+default_fallocate_cbk
+default_fallocate_failure_cbk
+default_fallocate_resume
+default_fentrylk
+default_fentrylk_cbk
+default_fentrylk_failure_cbk
+default_fentrylk_resume
+default_fgetxattr
+default_fgetxattr_cbk
+default_fgetxattr_failure_cbk
+default_fgetxattr_resume
+default_finodelk
+default_finodelk_cbk
+default_finodelk_failure_cbk
+default_finodelk_resume
+default_flush
+default_flush_cbk
+default_flush_failure_cbk
+default_flush_resume
+default_forget
+default_fremovexattr
+default_fremovexattr_cbk
+default_fremovexattr_failure_cbk
+default_fremovexattr_resume
+default_fsetattr
+default_fsetattr_cbk
+default_fsetattr_failure_cbk
+default_fsetattr_resume
+default_fsetxattr
+default_fsetxattr_cbk
+default_fsetxattr_failure_cbk
+default_fsetxattr_resume
+default_fstat
+default_fstat_cbk
+default_fstat_failure_cbk
+default_fstat_resume
+default_fsync
+default_fsync_cbk
+default_fsyncdir
+default_fsyncdir_cbk
+default_fsyncdir_failure_cbk
+default_fsyncdir_resume
+default_fsync_failure_cbk
+default_fsync_resume
+default_ftruncate
+default_ftruncate_cbk
+default_ftruncate_failure_cbk
+default_ftruncate_resume
+default_fxattrop
+default_fxattrop_cbk
+default_fxattrop_failure_cbk
+default_fxattrop_resume
+default_getactivelk
+default_getactivelk_failure_cbk
+default_getactivelk_resume
+default_getspec
+default_getspec_cbk
+default_getxattr
+default_getxattr_cbk
+default_getxattr_failure_cbk
+default_getxattr_resume
+default_inodelk
+default_inodelk_cbk
+default_inodelk_failure_cbk
+default_inodelk_resume
+default_ipc
+default_ipc_cbk
+default_lease
+default_lease_failure_cbk
+default_lease_resume
+default_link
+default_link_cbk
+default_link_failure_cbk
+default_link_resume
+default_lk
+default_lk_cbk
+default_lk_failure_cbk
+default_lk_resume
+default_lookup
+default_lookup_cbk
+default_lookup_failure_cbk
+default_lookup_resume
+default_mem_acct_init
+default_mkdir
+default_mkdir_cbk
+default_mkdir_failure_cbk
+default_mkdir_resume
+default_mknod
+default_mknod_cbk
+default_mknod_failure_cbk
+default_mknod_resume
+default_notify
+default_open
+default_open_cbk
+default_opendir
+default_opendir_cbk
+default_opendir_failure_cbk
+default_opendir_resume
+default_open_failure_cbk
+default_open_resume
+default_rchecksum
+default_rchecksum_cbk
+default_rchecksum_failure_cbk
+default_rchecksum_resume
+default_readdir
+default_readdir_cbk
+default_readdir_failure_cbk
+default_readdirp
+default_readdirp_cbk
+default_readdirp_failure_cbk
+default_readdirp_resume
+default_readdir_resume
+default_readlink
+default_readlink_cbk
+default_readlink_failure_cbk
+default_readlink_resume
+default_readv
+default_readv_cbk
+default_readv_failure_cbk
+default_readv_resume
+default_release
+default_releasedir
+default_removexattr
+default_removexattr_cbk
+default_removexattr_failure_cbk
+default_removexattr_resume
+default_rename
+default_rename_cbk
+default_rename_failure_cbk
+default_rename_resume
+default_rmdir
+default_rmdir_cbk
+default_rmdir_failure_cbk
+default_rmdir_resume
+default_seek
+default_seek_cbk
+default_seek_failure_cbk
+default_seek_resume
+default_setactivelk
+default_setactivelk_failure_cbk
+default_setactivelk_resume
+default_setattr
+default_setattr_cbk
+default_setattr_failure_cbk
+default_setattr_resume
+default_setxattr
+default_setxattr_cbk
+default_setxattr_failure_cbk
+default_setxattr_resume
+default_stat
+default_stat_cbk
+default_stat_failure_cbk
+default_statfs
+default_statfs_cbk
+default_statfs_failure_cbk
+default_statfs_resume
+default_stat_resume
+default_symlink
+default_symlink_cbk
+default_symlink_failure_cbk
+default_symlink_resume
+default_truncate
+default_truncate_cbk
+default_truncate_failure_cbk
+default_truncate_resume
+default_unlink
+default_unlink_cbk
+default_unlink_failure_cbk
+default_unlink_resume
+default_writev
+default_writev_cbk
+default_writev_failure_cbk
+default_writev_resume
+default_xattrop
+default_xattrop_cbk
+default_xattrop_failure_cbk
+default_xattrop_resume
+default_zerofill
+default_zerofill_cbk
+default_zerofill_failure_cbk
+default_zerofill_resume
+default_put
+default_put_cbk
+default_put_failure_cbk
+default_put_resume
+default_copy_file_range
+default_copy_file_range_cbk
+default_copy_file_range_failure_cbk
+default_copy_file_range_resume
+dht_is_linkfile
+dict_add
+dict_addn
+dict_add_dynstr_with_alloc
+dict_allocate_and_serialize
+dict_copy
+dict_copy_with_ref
+dict_del
+dict_deln
+dict_dump_to_statedump
+dict_dump_to_str
+dict_dump_to_log
+dict_foreach
+dict_foreach_fnmatch
+dict_foreach_match
+dict_for_key_value
+dict_get
+dict_getn
+dict_get_bin
+dict_get_double
+dict_get_gfuuid
+dict_get_iatt
+dict_get_mdata
+dict_get_int16
+dict_get_int32
+dict_get_int32n
+dict_get_int64
+dict_get_int8
+dict_get_ptr
+dict_get_ptr_and_len
+dict_get_str
+dict_get_strn
+dict_get_str_boolean
+dict_get_uint32
+dict_get_uint64
+dict_get_with_ref
+dict_has_key_from_array
+dict_key_count
+dict_keys_join
+dict_lookup
+dict_new
+dict_null_foreach_fn
+dict_ref
+dict_remove_foreach_fn
+dict_rename_key
+dict_reset
+dict_serialize
+dict_serialized_length
+dict_serialized_length_lk
+dict_serialize_value_with_delim
+dict_set
+dict_setn
+dict_set_bin
+dict_set_double
+dict_set_dynptr
+dict_set_dynstr
+dict_set_dynstrn
+dict_set_dynstr_with_alloc
+dict_set_gfuuid
+dict_set_iatt
+dict_set_mdata
+dict_set_int16
+dict_set_int32
+dict_set_int32n
+dict_set_int64
+dict_set_int8
+dict_set_static_bin
+dict_set_static_ptr
+dict_set_str
+dict_set_strn
+dict_setn_nstrn
+dict_set_nstrn
+dict_set_uint32
+dict_set_uint64
+dict_set_flag
+dict_clear_flag
+dict_check_flag
+dict_unref
+dict_unserialize
+drop_token
+eh_destroy
+eh_dump
+eh_new
+eh_save_history
+entry_copy
+gf_event_dispatch
+gf_event_dispatch_destroy
+gf_event_handled
+gf_event_pool_destroy
+gf_event_pool_new
+gf_event_reconfigure_threads
+gf_event_register
+gf_event_select_on
+gf_event_unregister
+gf_event_unregister_close
+fd_anonymous
+fd_anonymous_with_flags
+fd_bind
+fd_close
+fd_create
+fd_create_uint64
+__fd_ctx_del
+fd_ctx_del
+fd_ctx_dump
+__fd_ctx_get
+fd_ctx_get
+__fd_ctx_set
+fd_ctx_set
+fd_is_anonymous
+fd_list_empty
+fd_lk_ctx_empty
+fd_lk_ctx_ref
+fd_lk_ctx_unref
+fd_lk_insert_and_merge
+fd_lookup
+fd_lookup_anonymous
+fd_lookup_uint64
+__fd_ref
+fd_ref
+fd_unref
+_fini
+fop_access_stub
+fop_create_stub
+fop_copy_file_range_stub
+fop_copy_file_range_cbk_stub
+fop_discard_stub
+fop_entrylk_stub
+fop_enum_to_pri_string
+fop_fallocate_stub
+fop_fentrylk_stub
+fop_fgetxattr_stub
+fop_finodelk_stub
+fop_flush_stub
+fop_fremovexattr_cbk_stub
+fop_fremovexattr_stub
+fop_fsetattr_stub
+fop_fsetxattr_cbk_stub
+fop_fsetxattr_stub
+fop_fstat_stub
+fop_fsync_cbk_stub
+fop_fsyncdir_stub
+fop_fsync_stub
+fop_ftruncate_cbk_stub
+fop_ftruncate_stub
+fop_fxattrop_stub
+fop_getactivelk_stub
+fop_getxattr_stub
+fop_icreate_stub
+fop_inodelk_stub
+fop_ipc_stub
+fop_lease_stub
+fop_link_stub
+fop_lk_stub
+fop_log_level
+fop_lookup_cbk_stub
+fop_lookup_stub
+fop_mkdir_stub
+fop_mknod_stub
+fop_namelink_stub
+fop_opendir_stub
+fop_open_stub
+fop_put_stub
+fop_rchecksum_stub
+fop_readdirp_stub
+fop_readdir_stub
+fop_readlink_stub
+fop_readv_stub
+fop_removexattr_cbk_stub
+fop_removexattr_stub
+fop_rename_cbk_stub
+fop_rename_stub
+fop_rmdir_cbk_stub
+fop_rmdir_stub
+fop_seek_stub
+fop_setactivelk_stub
+fop_setattr_stub
+fop_setxattr_stub
+fop_statfs_stub
+fop_stat_stub
+fop_symlink_stub
+fop_truncate_cbk_stub
+fop_truncate_stub
+fop_unlink_cbk_stub
+fop_unlink_stub
+fop_writev_cbk_stub
+fop_writev_stub
+fop_xattrop_stub
+fop_zerofill_stub
+generate_glusterfs_ctx_id
+get_checksum_for_file
+get_checksum_for_path
+get_file_mtime
+get_host_name
+get_mem_size
+get_path_name
+get_xlator_by_name
+get_xlator_by_type
+gf_array_insertionsort
+gf_asprintf
+gf_async
+gf_async_adjust_threads
+gf_async_ctrl
+gf_async_init
+gf_async_fini
+gf_backtrace_save
+gf_bits_count
+gf_bits_index
+gf_build_absolute_path
+__gf_calloc
+gf_canonicalize_path
+gf_check_log_format
+gf_check_logger
+gf_client_disconnect
+gf_client_dump_fdtables
+gf_client_dump_fdtables_to_dict
+gf_client_dump_inodes
+gf_client_dump_inodes_to_dict
+gf_client_get
+gf_client_put
+gf_client_ref
+gf_clienttable_alloc
+gf_client_unref
+gf_cmd_log
+gf_cmd_log_init
+gf_compare_sockaddr
+gf_deitransform
+gf_dirent_entry_free
+gf_dirent_for_name
+gf_dirent_free
+gf_dirent_orig_offset
+gf_dm_hashfn
+gf_dnscache_init
+gf_dnscache_deinit
+gf_errno_to_error
+gf_error_to_errno
+_gf_event
+gf_fd_fdptr_get
+gf_fd_fdtable_alloc
+gf_fd_fdtable_copy_all_fds
+gf_fd_fdtable_destroy
+gf_fd_fdtable_get_all_fds
+gf_fdptr_put
+gf_fd_put
+gf_fd_unused_get
+gf_fill_iatt_for_dirent
+gf_fop_int
+gf_fop_string
+__gf_free
+gf_free_mig_locks
+gf_getgrouplist
+gf_get_hostname_from_ip
+gf_get_index_by_elem
+gf_global_mem_acct_enable_set
+gfid_to_ino
+gf_inode_type_to_str
+gf_is_ip_in_net
+gf_is_local_addr
+gf_is_same_address
+gf_is_service_running
+gf_is_str_int
+gf_is_valid_xattr_namespace
+gf_is_zero_filled_stat
+gf_itransform
+gf_link_inodes_from_dirent
+_gf_log
+_gf_log_callingfn
+gf_log_disable_suppression_before_exit
+gf_log_dump_graph
+_gf_log_eh
+gf_log_fini
+gf_log_get_localtime
+gf_log_get_loglevel
+gf_log_globals_init
+gf_log_init
+gf_log_inject_timer_event
+gf_log_logrotate
+gf_log_set_localtime
+gf_log_set_log_buf_size
+gf_log_set_log_flush_timeout
+gf_log_set_logformat
+gf_log_set_logger
+gf_log_set_loglevel
+gf_lstat_dir
+__gf_malloc
+gf_mem_acct_enable_set
+gf_monitor_metrics
+_gf_msg
+_gf_msg_nomem
+gf_nwrite
+gf_path_strip_trailing_slashes
+gf_print_trace
+gf_proc_dump_add_section
+gf_proc_dump_info
+gf_proc_dump_init
+gf_proc_dump_mallinfo
+gf_proc_dump_mem_info
+gf_proc_dump_mem_info_to_dict
+gf_proc_dump_mempool_info
+gf_proc_dump_mempool_info_to_dict
+gf_proc_dump_pending_frames
+gf_proc_dump_pending_frames_to_dict
+gf_proc_dump_write
+gf_proc_dump_xlator_history
+gf_proc_dump_xlator_meminfo
+gf_proc_dump_xlator_private
+gf_proc_dump_xlator_profile
+gf_process_getspec_servers_list
+gf_process_reserved_ports
+__gf_realloc
+_gf_ref_get
+_gf_ref_init
+_gf_ref_put
+gf_resolve_ip6
+gf_resolve_path_parent
+gf_rev_dns_lookup
+gf_rev_dns_lookup_cached
+gf_rsync_strong_checksum
+gf_rsync_md5_checksum
+gf_rsync_weak_checksum
+gf_set_log_file_path
+gf_set_log_ident
+gf_set_timestamp
+gf_set_volfile_server_common
+_gf_smsg
+gf_sock_union_equal_addr
+gf_store_handle_create_on_absence
+gf_store_handle_destroy
+gf_store_handle_new
+gf_store_handle_retrieve
+gf_store_iter_destroy
+gf_store_iter_get_matching
+gf_store_iter_get_next
+gf_store_iter_new
+gf_store_lock
+gf_store_locked_local
+gf_store_mkdir
+gf_store_mkstemp
+gf_store_read_and_tokenize
+gf_store_rename_tmppath
+gf_store_retrieve_value
+gf_store_save_value
+gf_store_save_items
+gf_store_unlink_tmppath
+gf_store_unlock
+gf_string2boolean
+gf_string2bytesize_int64
+gf_string2bytesize_uint64
+gf_string2double
+gf_string2int
+gf_string2int32
+gf_string2percent
+gf_string2time
+gf_string2uint
+gf_string2uint32
+gf_string2uint64
+gf_string2uint_base10
+gf_strip_whitespace
+gf_strncpy
+gf_strTrim
+gf_strstr
+gf_thread_cleanup_xint
+gf_thread_create
+gf_thread_vcreate
+gf_thread_create_detached
+gf_thread_set_name
+gf_thread_set_vname
+gf_timer_call_after
+gf_timer_call_cancel
+gf_timer_registry_destroy
+_gf_timestuff
+gf_trim
+gf_tw_add_timer
+gf_tw_del_timer
+gf_tw_mod_timer
+gf_tw_mod_timer_pending
+gf_uint64_2human_readable
+gf_umount_lazy
+gf_update_latency
+gf_uuid_clear
+gf_uuid_compare
+gf_uuid_copy
+gf_uuid_is_null
+gf_uuid_generate
+gf_uuid_parse
+gf_uuid_unparse
+gf_valid_pid
+gf_vasprintf
+gf_volfile_reconfigure
+gf_xxh64_wrapper
+gf_zero_fill_stat
+gid_cache_add
+gid_cache_init
+gid_cache_lookup
+gid_cache_reconf
+gid_cache_release
+glusterd_check_log_level
+glusterfs_compute_sha256
+glusterfs_ctx_new
+glusterfs_ctx_tw_get
+glusterfs_ctx_tw_put
+glusterfs_delete_volfile_checksum
+glusterfs_globals_init
+glusterfs_graph_activate
+glusterfs_graph_attach
+glusterfs_graph_construct
+glusterfs_graph_deactivate
+glusterfs_graph_destroy
+glusterfs_graph_destroy_residual
+glusterfs_graph_prepare
+glusterfs_read_secure_access_file
+glusterfs_graph_print_file
+glusterfs_graph_set_first
+glusterfs_is_local_pathinfo
+glusterfs_leaf_position
+glusterfs_reachable_leaves
+__glusterfs_this_location
+glusterfs_this_set
+glusterfs_volfile_reconfigure
+glusterfs_xlator_link
+graph_reconf_validateopt
+_init
+inode_ctx_del2
+__inode_ctx_get0
+inode_ctx_get0
+__inode_ctx_get1
+inode_ctx_get1
+__inode_ctx_get2
+inode_ctx_get2
+inode_ctx_merge
+inode_ctx_reset0
+inode_ctx_reset1
+inode_ctx_reset2
+__inode_ctx_set0
+inode_ctx_set0
+__inode_ctx_set1
+inode_ctx_set1
+__inode_ctx_set2
+inode_ctx_set2
+inode_ctx_size
+inode_dump
+inode_dump_to_dict
+__inode_find
+inode_find
+inode_find_directory_name
+inode_forget
+inode_forget_with_unref
+inode_from_path
+inode_grep
+inode_grep_for_gfid
+inode_has_dentry
+inode_invalidate
+inode_is_linked
+inode_link
+inode_lookup
+inode_needs_lookup
+inode_new
+inode_parent
+__inode_path
+inode_path
+inode_ref
+inode_ref_reduce_by_n
+inode_rename
+inode_resolve
+inode_set_need_lookup
+inode_table_ctx_free
+inode_table_destroy
+inode_table_destroy_all
+inode_table_dump
+inode_table_dump_to_dict
+inode_table_new
+inode_table_with_invalidator
+__inode_table_set_lru_limit
+inode_table_set_lru_limit
+inode_unlink
+inode_unref
+int_to_data
+iobref_add
+iobref_clear
+iobref_merge
+iobref_new
+iobref_ref
+iobref_size
+iobref_unref
+iobuf_get
+iobuf_get2
+iobuf_get_page_aligned
+iobuf_pool_destroy
+iobuf_pool_new
+iobuf_size
+iobuf_to_iovec
+iobuf_unref
+iobuf_copy
+is_data_equal
+__is_fuse_call
+is_gf_log_command
+is_graph_topology_equal
+__is_root_gfid
+is_valid_lease_id
+leaseid_utoa
+gf_existing_leaseid
+gf_leaseid_get
+list_node_add
+list_node_add_order
+list_node_del
+lkowner_utoa
+loc_copy
+loc_copy_overload_parent
+loc_gfid
+loc_gfid_utoa
+loc_is_nameless
+loc_is_root
+loc_pargfid
+loc_path
+loc_touchup
+loc_wipe
+log_base2
+_mask_cancellation
+mask_match
+mem_get
+mem_get0
+mem_pool_destroy
+mem_pool_new_fn
+mem_pools_fini
+mem_pools_init
+mem_put
+mkdir_p
+next_token
+nwstrtail
+os_daemon
+os_daemon_return
+parser_deinit
+parser_get_next_match
+parser_init
+parser_set_string
+parser_unset_string
+quota_conf_read_gfid
+quota_conf_read_version
+quota_conf_skip_header
+quota_data_to_meta
+quota_dict_get_inode_meta
+quota_dict_get_meta
+quota_dict_set_meta
+quota_meta_is_null
+rb_create
+rb_delete
+rb_destroy
+rb_find
+rb_probe
+rbthash_get
+rbthash_insert
+rbthash_remove
+rbthash_table_destroy
+rbthash_table_init
+rbuf_dtor
+rbuf_get_buffer
+rbuf_init
+rbuf_reserve_write_area
+rbuf_wait_for_completion
+rbuf_write_complete
+recursive_rmdir
+runcmd
+runinit
+runner
+runner_add_arg
+runner_add_args
+runner_argprintf
+runner_chio
+runner_end
+runner_log
+runner_redir
+runner_run
+runner_run_nowait
+runner_run_reuse
+runner_start
+set_sys_log_level
+skipwhite
+strfd_close
+strfd_open
+strprintf
+strtail
+str_to_data
+SuperFastHash
+syncbarrier_destroy
+syncbarrier_init
+syncbarrier_wait
+syncbarrier_wake
+synccond_init
+synccond_destroy
+synccond_wait
+synccond_timedwait
+synccond_signal
+synccond_broadcast
+syncenv_destroy
+syncenv_new
+synclock_destroy
+synclock_init
+synclock_lock
+synclock_trylock
+synclock_unlock
+syncop_access
+syncop_close
+syncop_create
+syncop_copy_file_range
+syncopctx_getctx
+syncopctx_setfsgid
+syncopctx_setfsgroups
+syncopctx_setfslkowner
+syncopctx_setfspid
+syncopctx_setfsuid
+syncop_dirfd
+syncop_dir_scan
+syncop_discard
+syncop_fallocate
+syncop_flush
+syncop_fgetxattr
+syncop_fremovexattr
+syncop_fsetattr
+syncop_fsetxattr
+syncop_fstat
+syncop_fsync
+syncop_fsyncdir
+syncop_ftruncate
+syncop_ftw
+syncop_ftw_throttle
+syncop_fxattrop
+syncop_getactivelk
+syncop_getxattr
+syncop_gfid_to_path
+syncop_gfid_to_path_hard
+syncop_inode_find
+syncop_inodelk
+syncop_entrylk
+syncop_ipc
+syncop_is_subvol_local
+syncop_link
+syncop_listxattr
+syncop_lk
+syncop_lookup
+syncop_mkdir
+syncop_mknod
+syncop_mt_dir_scan
+syncop_open
+syncop_opendir
+syncop_readdir
+syncop_readdirp
+syncop_readlink
+syncop_readv
+syncop_removexattr
+syncop_rename
+syncop_rmdir
+syncop_seek
+syncop_setactivelk
+syncop_setattr
+syncop_setxattr
+syncop_stat
+syncop_statfs
+syncop_symlink
+syncop_truncate
+syncop_unlink
+syncop_write
+syncop_writev
+syncop_xattrop
+syncop_zerofill
+syncop_lease
+synctask_get
+synctask_new
+synctask_new1
+synctask_set
+synctask_setid
+synctask_sleep
+synctask_wake
+synctask_yield
+sys_access
+sys_chmod
+sys_chown
+sys_close
+sys_closedir
+sys_copy_file_range
+sys_creat
+sys_fallocate
+sys_fchmod
+sys_fchown
+sys_fdatasync
+sys_fgetxattr
+sys_flistxattr
+sys_fremovexattr
+sys_fsetxattr
+sys_fstat
+sys_fstatat
+sys_fsync
+sys_ftruncate
+sys_futimes
+sys_lchown
+sys_lgetxattr
+sys_link
+sys_linkat
+sys_llistxattr
+sys_lremovexattr
+sys_lseek
+sys_lsetxattr
+sys_lstat
+sys_mkdir
+sys_mkdirat
+sys_mknod
+sys_open
+sys_openat
+sys_opendir
+sys_pread
+sys_pwrite
+sys_pwritev
+sys_read
+sys_readdir
+sys_readlink
+sys_readv
+sys_rename
+sys_rmdir
+sys_stat
+sys_statvfs
+sys_symlink
+sys_symlinkat
+sys_truncate
+sys_unlink
+sys_unlinkat
+sys_utimensat
+sys_write
+sys_writev
+sys_socket
+sys_accept
+sys_kill
+sys_sysctl
+tbf_init
+tbf_throttle
+timespec_now
+timespec_now_realtime
+timespec_sub
+timespec_adjust_delta
+timespec_cmp
+token_iter_init
+trap
+trie_add
+trie_destroy
+trie_measure
+trie_measure_vec
+trie_new
+trienode_get_word
+_unmask_cancellation
+uuid_utoa
+uuid_utoa_r
+validate_brick_name
+valid_host_name
+valid_ipv4_address
+valid_internet_address
+xlator_destroy
+xlator_foreach
+xlator_foreach_depth_first
+xlator_init
+xlator_mem_acct_init
+xlator_mem_acct_unref
+xlator_notify
+xlator_option_info_list
+xlator_option_init_bool
+xlator_option_init_double
+xlator_option_init_int32
+xlator_option_init_path
+xlator_option_init_percent
+xlator_option_init_percent_or_size
+xlator_option_init_size
+xlator_option_init_size_uint64
+xlator_option_init_size_int64
+xlator_option_init_str
+xlator_option_init_time
+xlator_option_init_uint32
+xlator_option_init_uint64
+xlator_option_init_int64
+xlator_option_init_xlator
+xlator_option_reconf_bool
+xlator_option_reconf_int32
+xlator_option_reconf_path
+xlator_option_reconf_percent
+xlator_option_reconf_percent_or_size
+xlator_option_reconf_size
+xlator_option_reconf_size_uint64
+xlator_option_reconf_size_int64
+xlator_option_reconf_str
+xlator_option_reconf_time
+xlator_option_reconf_uint32
+xlator_option_reconf_uint64
+xlator_option_reconf_int64
+xlator_option_reconf_xlator
+xlator_options_validate
+xlator_options_validate_list
+xlator_option_validate
+xlator_option_validate_addr_list
+xlator_search_by_name
+xlator_set_inode_lru_limit
+xlator_set_type
+xlator_set_type_virtual
+xlator_subvolume_count
+xlator_tree_free_members
+xlator_volopt_dynload
+xlator_volume_option_get
+xlator_volume_option_get_list
+xlator_memrec_free
+xlator_mem_cleanup
+gluster_graph_take_reference
+default_fops
+gf_fop_list
+gf_upcall_list
+vol_type_str
+global_ctx
+global_ctx_mutex
+global_xlator
+use_spinlocks
+dump_options
+glusterfs_leaseid_buf_get
+glusterfs_leaseid_exist
+gf_replace_old_iatt_in_dict
+gf_replace_new_iatt_in_dict
+gf_changelog_init
+gf_changelog_register_generic
+gf_gfid_generate_from_xxh64
+find_xlator_option_in_cmd_args_t
+gf_d_type_from_ia_type
+glusterfs_graph_fini
+glusterfs_process_svc_attach_volfp
+glusterfs_mux_volfile_reconfigure
+glusterfs_process_svc_detach
+mgmt_is_multiplexed_daemon
+xlator_is_cleanup_starting
+gf_nanosleep
+gf_syncfs
+graph_total_client_xlator
+get_xattrs_to_heal
+gf_latency_statedump_and_reset
+gf_latency_new
+gf_latency_reset
+gf_latency_update
+gf_frame_latency_update
diff --git a/libglusterfs/src/list.h b/libglusterfs/src/list.h
deleted file mode 100644
index d4851dfecd4..00000000000
--- a/libglusterfs/src/list.h
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _LLIST_H
-#define _LLIST_H
-
-
-struct list_head {
-	struct list_head *next;
-	struct list_head *prev;
-};
-
-
-#define INIT_LIST_HEAD(head) do {			\
-		(head)->next = (head)->prev = head;	\
-	} while (0)
-
-
-static inline void
-list_add (struct list_head *new, struct list_head *head)
-{
-	new->prev = head;
-	new->next = head->next;
-
-	new->prev->next = new;
-	new->next->prev = new;
-}
-
-
-static inline void
-list_add_tail (struct list_head *new, struct list_head *head)
-{
-	new->next = head;
-	new->prev = head->prev;
-
-	new->prev->next = new;
-	new->next->prev = new;
-}
-
-
-static inline void
-list_del (struct list_head *old)
-{
-	old->prev->next = old->next;
-	old->next->prev = old->prev;
-
-	old->next = (void *)0xbabebabe;
-	old->prev = (void *)0xcafecafe;
-}
-
-
-static inline void
-list_del_init (struct list_head *old)
-{
-	old->prev->next = old->next;
-	old->next->prev = old->prev;
-
-	old->next = old;
-	old->prev = old;
-}
-
-
-static inline void
-list_move (struct list_head *list, struct list_head *head)
-{
-	list_del (list);
-	list_add (list, head);
-}
-
-
-static inline void
-list_move_tail (struct list_head *list, struct list_head *head)
-{
-	list_del (list);
-	list_add_tail (list, head);
-}
-
-
-static inline int
-list_empty (struct list_head *head)
-{
-	return (head->next == head);
-}
-
-
-static inline void
-__list_splice (struct list_head *list, struct list_head *head)
-{
-	(list->prev)->next = (head->next);
-	(head->next)->prev = (list->prev);
-
-	(head)->next = (list->next);
-	(list->next)->prev = (head);
-}
-
-
-static inline void
-list_splice (struct list_head *list, struct list_head *head)
-{
-	if (list_empty (list))
-		return;
-
-	__list_splice (list, head);
-}
-
-
-static inline void
-list_splice_init (struct list_head *list, struct list_head *head)
-{
-	if (list_empty (list))
-		return;
-
-	__list_splice (list, head);
-	INIT_LIST_HEAD (list);
-}
-
-
-#define list_entry(ptr, type, member)					\
-	((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))
-
-
-#define list_for_each(pos, head)				     \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-
-#define list_for_each_entry(pos, head, member)				\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-
-#define list_for_each_entry_safe(pos, n, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member),	\
-		n = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-#endif /* _LLIST_H */
diff --git a/libglusterfs/src/locking.c b/libglusterfs/src/locking.c
new file mode 100644
index 00000000000..7577054e33a
--- /dev/null
+++ b/libglusterfs/src/locking.c
@@ -0,0 +1,27 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#if defined(HAVE_SPINLOCK)
+/* None of this matters otherwise. */
+
+#include <pthread.h>
+#include <unistd.h>
+
+#define LOCKING_IMPL
+#include "glusterfs/locking.h"
+
+int use_spinlocks = 0;
+
+static void __attribute__((constructor)) gf_lock_setup(void)
+{
+    // use_spinlocks = (sysconf(_SC_NPROCESSORS_ONLN) > 1);
+}
+
+#endif
diff --git a/libglusterfs/src/locking.h b/libglusterfs/src/locking.h
deleted file mode 100644
index c5043aa14e7..00000000000
--- a/libglusterfs/src/locking.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _LOCKING_H
-#define _LOCKING_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <pthread.h>
-
-#if HAVE_SPINLOCK
-#define LOCK_INIT(x)    pthread_spin_init (x, 0)
-#define LOCK(x)         pthread_spin_lock (x)
-#define TRY_LOCK(x)     pthread_spin_trylock (x)
-#define UNLOCK(x)       pthread_spin_unlock (x)
-#define LOCK_DESTROY(x) pthread_spin_destroy (x)
-
-typedef pthread_spinlock_t gf_lock_t;
-#else
-#define LOCK_INIT(x)    pthread_mutex_init (x, 0)
-#define LOCK(x)         pthread_mutex_lock (x)
-#define TRY_LOCK(x)     pthread_mutex_trylock (x)
-#define UNLOCK(x)       pthread_mutex_unlock (x)
-#define LOCK_DESTROY(x) pthread_mutex_destroy (x)
-
-typedef pthread_mutex_t gf_lock_t;
-#endif /* HAVE_SPINLOCK */
-
-
-#endif /* _LOCKING_H */
diff --git a/libglusterfs/src/logging.c b/libglusterfs/src/logging.c
index 5b89aafb490..a930d3e3b63 100644
--- a/libglusterfs/src/logging.c
+++ b/libglusterfs/src/logging.c
@@ -1,27 +1,13 @@
 /*
-  Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <errno.h>
 #include <pthread.h>
 #include <stdio.h>
@@ -30,628 +16,2431 @@
 #include <locale.h>
 #include <string.h>
 #include <stdlib.h>
-
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#ifdef GF_LINUX_HOST_OS
 #include <syslog.h>
-#endif
+#include <sys/resource.h>
 
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
+#else
+#include "execinfo_compat.h"
 #endif
 
+#include <sys/stat.h>
+
+#include "glusterfs/syscall.h"
 
-static pthread_mutex_t  logfile_mutex;
-static char            *filename = NULL;
-static uint8_t          logrotate = 0;
-static FILE            *logfile = NULL;
-static gf_loglevel_t    loglevel = GF_LOG_MAX;
-static int              gf_log_syslog = 0;
+#define GF_JSON_MSG_LENGTH 8192
+#define GF_SYSLOG_CEE_FORMAT                                                   \
+    "@cee: {\"msg\": \"%s\", \"gf_code\": \"%u\", \"gf_message\": \"%s\"}"
+#define GF_LOG_CONTROL_FILE "/etc/glusterfs/logger.conf"
+#define GF_LOG_BACKTRACE_DEPTH 5
+#define GF_LOG_BACKTRACE_SIZE 4096
+#define GF_MAX_SLOG_PAIR_COUNT 100
 
-char                    gf_log_xl_log_set;
-gf_loglevel_t           gf_log_loglevel; /* extern'd */
-FILE                   *gf_log_logfile;
+#include "glusterfs/logging.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/timer.h"
+#include "glusterfs/libglusterfs-messages.h"
 
-static char            *cmd_log_filename = NULL;
-static FILE            *cmdlogfile = NULL;
+/* Do not replace gf_log in TEST_LOG with gf_msg, as there is a slight chance
+ * that it could lead to an infinite recursion.*/
+#define TEST_LOG(__msg, __args...)                                             \
+    gf_log("logging-infra", GF_LOG_DEBUG, __msg, ##__args);
+
+static void
+gf_log_flush_timeout_cbk(void *data);
+
+int
+gf_log_inject_timer_event(glusterfs_ctx_t *ctx);
+
+static void
+gf_log_flush_extra_msgs(glusterfs_ctx_t *ctx, uint32_t new);
+
+static int
+log_buf_init(log_buf_t *buf, const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             int errnum, uint64_t msgid, char **appmsgstr, int graph_id);
+static void
+gf_log_rotate(glusterfs_ctx_t *ctx);
+
+static char gf_level_strings[] = {
+    ' ', /* NONE */
+    'M', /* EMERGENCY */
+    'A', /* ALERT */
+    'C', /* CRITICAL */
+    'E', /* ERROR */
+    'W', /* WARNING */
+    'N', /* NOTICE */
+    'I', /* INFO */
+    'D', /* DEBUG */
+    'T', /* TRACE */
+};
 
 void
-gf_log_logrotate (int signum)
+gf_log_logrotate(int signum)
 {
-	logrotate = 1;
+    if (THIS->ctx) {
+        THIS->ctx->log.logrotate = 1;
+        THIS->ctx->log.cmd_history_logrotate = 1;
+    }
 }
 
 void
-gf_log_enable_syslog (void)
+gf_log_enable_syslog(void)
 {
-        gf_log_syslog = 1;
+    if (THIS->ctx)
+        THIS->ctx->log.gf_log_syslog = 1;
 }
 
 void
-gf_log_disable_syslog (void)
+gf_log_disable_syslog(void)
 {
-        gf_log_syslog = 0;
+    if (THIS->ctx)
+        THIS->ctx->log.gf_log_syslog = 0;
 }
 
 gf_loglevel_t
-gf_log_get_loglevel (void)
+gf_log_get_loglevel(void)
 {
-	return loglevel;
+    if (THIS->ctx)
+        return THIS->ctx->log.loglevel;
+    else
+        /* return global defaults (see gf_log_globals_init) */
+        return GF_LOG_INFO;
 }
 
 void
-gf_log_set_loglevel (gf_loglevel_t level)
+gf_log_set_loglevel(glusterfs_ctx_t *ctx, gf_loglevel_t level)
 {
-        gf_log_loglevel = loglevel = level;
+    if (ctx)
+        ctx->log.loglevel = level;
 }
 
+int
+gf_log_get_localtime(void)
+{
+    if (THIS->ctx)
+        return THIS->ctx->log.localtime;
+    else
+        /* return global defaults (see gf_log_globals_init) */
+        return 0;
+}
 
-gf_loglevel_t
-gf_log_get_xl_loglevel (void *this)
+void
+gf_log_set_localtime(int on_off)
 {
-        xlator_t *xl = this;
-        if (!xl)
-                return 0;
-        return xl->loglevel;
+    if (THIS->ctx)
+        THIS->ctx->log.localtime = on_off;
 }
 
 void
-gf_log_set_xl_loglevel (void *this, gf_loglevel_t level)
+gf_log_flush(void)
 {
-        xlator_t *xl = this;
-        if (!xl)
-                return;
-        gf_log_xl_log_set = 1;
-        xl->loglevel = level;
+    xlator_t *this = THIS;
+    glusterfs_ctx_t *ctx = this->ctx;
+
+    if (ctx && ctx->log.logger == gf_logger_glusterlog) {
+        pthread_mutex_lock(&ctx->log.logfile_mutex);
+        fflush(ctx->log.gf_log_logfile);
+        pthread_mutex_unlock(&ctx->log.logfile_mutex);
+    }
+
+    return;
 }
 
 void
-gf_log_fini (void)
+gf_log_set_xl_loglevel(void *this, gf_loglevel_t level)
 {
-	pthread_mutex_destroy (&logfile_mutex);
+    xlator_t *xl = this;
+    if (!xl)
+        return;
+    xl->loglevel = level;
 }
 
+/* TODO: The following get/set functions are yet not invoked from anywhere
+ * in the code. The _intention_ is to pass CLI arguments to various daemons
+ * that are started, which would then invoke these set APIs as required.
+ *
+ * glusterd would read the defaults from its .vol file configuration shipped
+ * as a part of the packages distributed.
+ *
+ * For any gluster* daemon that is started the shipped configuration becomes the
+ * default, if a volume has to change its logging format or logger, then a
+ * gluster CLI is invoked to set this property for the volume in question.
+ *
+ * The property is maintained by glusterd, and passed to the daemon as a CLI
+ * option, IOW persistence of the option is maintained by glusterd persistent
+ * storage (i.e .vol file) only
+ *
+ * care needs to be taken to configure and start daemons based on the versions
+ * that supports these features */
 
-int
-gf_log_init (const char *file)
-{
-	if (!file){
-		fprintf (stderr, "gf_log_init: no filename specified\n");
-		return -1;
-	}
-
-	pthread_mutex_init (&logfile_mutex, NULL);
-
-        filename = gf_strdup (file);
-	if (!filename) {
-		fprintf (stderr, "gf_log_init: strdup error\n");
-		return -1;
-	}
-
-	logfile = fopen (file, "a");
-	if (!logfile){
-		fprintf (stderr,
-			 "gf_log_init: failed to open logfile \"%s\" (%s)\n",
-			 file,
-			 strerror (errno));
-		return -1;
-	}
+void
+gf_log_set_logformat(gf_log_format_t format)
+{
+    if (THIS->ctx)
+        THIS->ctx->log.logformat = format;
+}
 
-#ifdef GF_LINUX_HOST_OS
-        /* For the 'syslog' output. one can grep 'GlusterFS' in syslog
-           for serious logs */
-        openlog ("GlusterFS", LOG_PID, LOG_DAEMON);
-#endif
+void
+gf_log_set_logger(gf_log_logger_t logger)
+{
+    if (THIS->ctx)
+        THIS->ctx->log.logger = logger;
+}
 
-	gf_log_logfile = logfile;
+gf_loglevel_t
+gf_log_get_xl_loglevel(void *this)
+{
+    xlator_t *xl = this;
+    if (!xl)
+        return 0;
+    return xl->loglevel;
+}
 
-	return 0;
+void
+gf_log_set_log_buf_size(uint32_t buf_size)
+{
+    uint32_t old = 0;
+    glusterfs_ctx_t *ctx = THIS->ctx;
+
+    pthread_mutex_lock(&ctx->log.log_buf_lock);
+    {
+        old = ctx->log.lru_size;
+        ctx->log.lru_size = buf_size;
+    }
+    pthread_mutex_unlock(&ctx->log.log_buf_lock);
+
+    /* If the old size is less than/equal to the new size, then do nothing.
+     *
+     * But if the new size is less than the old size, then
+     *   a. If the cur size of the buf is less than or equal the new size,
+     *      then do nothing.
+     *   b. But if the current size of the buf is greater than the new size,
+     *      then flush the least recently used (cur size - new_size) msgs
+     *      to disk.
+     */
+    if (buf_size < old)
+        gf_log_flush_extra_msgs(ctx, buf_size);
 }
 
+void
+gf_log_set_log_flush_timeout(uint32_t timeout)
+{
+    THIS->ctx->log.timeout = timeout;
+}
 
-/*
- * Initialize logging to a central server.
- * If successful, log messages will be written both to
- * the local file and to the remote server.
+/* If log_buf_init() fails (indicated by a return value of -1),
+ * call log_buf_destroy() to clean up memory allocated in heap and to return
+ * the log_buf_t object back to its memory pool.
  */
+static int
+log_buf_init(log_buf_t *buf, const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             int errnum, uint64_t msgid, char **appmsgstr, int graph_id)
+{
+    int ret = -1;
 
-static int __central_log_enabled = 0;
+    if (!buf || !domain || !file || !function || !appmsgstr || !*appmsgstr)
+        goto out;
 
-struct _msg_queue {
-        struct list_head msgs;
-};
+    buf->msg = gf_strdup(*appmsgstr);
+    if (!buf->msg)
+        goto out;
 
-struct _log_msg {
-        const char *msg;
-        struct list_head queue;
-};
+    buf->msg_id = msgid;
+    buf->errnum = errnum;
+    buf->domain = gf_strdup(domain);
+    if (!buf->domain)
+        goto out;
 
+    buf->file = gf_strdup(file);
+    if (!buf->file)
+        goto out;
 
+    buf->function = gf_strdup(function);
+    if (!buf->function)
+        goto out;
 
+    buf->line = line;
+    buf->level = level;
+    buf->refcount = 0;
+    buf->graph_id = graph_id;
+    INIT_LIST_HEAD(&buf->msg_list);
 
-void 
-gf_log_lock (void)
-{
-	pthread_mutex_lock (&logfile_mutex);
+    ret = 0;
+out:
+    return ret;
 }
 
+static int
+log_buf_destroy(log_buf_t *buf)
+{
+    if (!buf)
+        return -1;
 
-void 
-gf_log_unlock (void)
+    GF_FREE(buf->msg);
+    GF_FREE(buf->domain);
+    GF_FREE(buf->file);
+    GF_FREE(buf->function);
+
+    mem_put(buf);
+    return 0;
+}
+
+static void
+gf_log_rotate(glusterfs_ctx_t *ctx)
 {
-	pthread_mutex_unlock (&logfile_mutex);
+    int fd = -1;
+    FILE *new_logfile = NULL;
+    FILE *old_logfile = NULL;
+
+    /* not involving locks on initial check to speed it up */
+    if (ctx->log.logrotate) {
+        /* let only one winner through on races */
+        pthread_mutex_lock(&ctx->log.logfile_mutex);
+
+        if (!ctx->log.logrotate) {
+            pthread_mutex_unlock(&ctx->log.logfile_mutex);
+            return;
+        } else {
+            ctx->log.logrotate = 0;
+            pthread_mutex_unlock(&ctx->log.logfile_mutex);
+        }
+
+        fd = sys_open(ctx->log.filename, O_CREAT | O_WRONLY | O_APPEND,
+                      S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+            gf_smsg("logrotate", GF_LOG_ERROR, errno,
+                    LG_MSG_OPEN_LOGFILE_FAILED, NULL);
+            return;
+        }
+
+        new_logfile = fdopen(fd, "a");
+        if (!new_logfile) {
+            gf_smsg("logrotate", GF_LOG_CRITICAL, errno,
+                    LG_MSG_OPEN_LOGFILE_FAILED, "filename=%s",
+                    ctx->log.filename, NULL);
+            sys_close(fd);
+            return;
+        }
+
+        pthread_mutex_lock(&ctx->log.logfile_mutex);
+        {
+            if (ctx->log.logfile)
+                old_logfile = ctx->log.logfile;
+
+            ctx->log.gf_log_logfile = ctx->log.logfile = new_logfile;
+        }
+        pthread_mutex_unlock(&ctx->log.logfile_mutex);
+
+        if (old_logfile != NULL)
+            fclose(old_logfile);
+    }
+
+    return;
 }
 
+void
+gf_log_globals_fini(void)
+{
+    /* TODO: Nobody is invoking the fini, but cleanup needs to happen here,
+     * needs cleanup for, log.ident, log.filename, closelog, log file close
+     * rotate state, possibly under a lock */
+    pthread_mutex_destroy(&THIS->ctx->log.logfile_mutex);
+    pthread_mutex_destroy(&THIS->ctx->log.log_buf_lock);
+}
 
 void
-gf_log_cleanup (void)
+gf_log_disable_suppression_before_exit(glusterfs_ctx_t *ctx)
 {
-	pthread_mutex_destroy (&logfile_mutex);
+    /*
+     * First set log buf size to 0. This would ensure two things:
+     * i. that all outstanding log messages are flushed to disk, and
+     * ii. all subsequent calls to gf_msg will result in the logs getting
+     *     directly flushed to disk without being buffered.
+     *
+     * Then, cancel the current log timer event.
+     */
+
+    gf_log_set_log_buf_size(0);
+    pthread_mutex_lock(&ctx->log.log_buf_lock);
+    {
+        if (ctx->log.log_flush_timer) {
+            gf_timer_call_cancel(ctx, ctx->log.log_flush_timer);
+            ctx->log.log_flush_timer = NULL;
+        }
+    }
+    pthread_mutex_unlock(&ctx->log.log_buf_lock);
 }
 
+/** gf_log_fini - function to perform the cleanup of the log information
+ * @data - glusterfs context
+ * @return: success: 0
+ *          failure: -1
+ */
 int
-_gf_log_callingfn (const char *domain, const char *file, const char *function,
-                   int line, gf_loglevel_t level, const char *fmt, ...)
-{
-	const char     *basename        = NULL;
-	struct tm      *tm              = NULL;
-        xlator_t       *this            = NULL;
-        char           *str1            = NULL;
-        char           *str2            = NULL;
-        char           *msg             = NULL;
-	char            timestr[256]    = {0,};
-	char            callstr[4096]   = {0,};
-        struct timeval  tv              = {0,};
-        size_t          len             = 0;
-        int             ret             = 0;
-        gf_loglevel_t   xlator_loglevel = 0;
-	va_list         ap;
-
-	if (!logfile)
-		return -1;
-
-        this = THIS;
-
-        xlator_loglevel = this->loglevel;
-        if (xlator_loglevel == 0)
-                xlator_loglevel = loglevel;
-
-        if (level > xlator_loglevel)
-                goto out;
+gf_log_fini(void *data)
+{
+    glusterfs_ctx_t *ctx = data;
+    int ret = 0;
+    FILE *old_logfile = NULL;
+
+    if (ctx == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_log_disable_suppression_before_exit(ctx);
+
+    pthread_mutex_lock(&ctx->log.logfile_mutex);
+    {
+        if (ctx->log.logfile) {
+            old_logfile = ctx->log.logfile;
+
+            /* Logfile needs to be set to NULL, so that any
+               call to gf_log after calling gf_log_fini, will
+               log the message to stderr.
+            */
+            ctx->log.loglevel = GF_LOG_NONE;
+            ctx->log.logfile = NULL;
+        }
+    }
+    pthread_mutex_unlock(&ctx->log.logfile_mutex);
 
-	static char *level_strings[] = {"",  /* NONE */
-                                        "M", /* EMERGENCY */
-                                        "A", /* ALERT */
-					"C", /* CRITICAL */
-					"E", /* ERROR */
-					"W", /* WARNING */
-					"N", /* NOTICE */
-                                        "I", /* INFO/NORMAL */
-					"D", /* DEBUG */
-                                        "T", /* TRACE */
-					""};
-
-	if (!domain || !file || !function || !fmt) {
-		fprintf (stderr,
-			 "logging: %s:%s():%d: invalid argument\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-		return -1;
-	}
-
-#if HAVE_BACKTRACE
-	/* Print 'calling function' */
-	do {
-		void *array[5];
-                char **callingfn = NULL;
-		size_t size = 0;
-
-		size = backtrace (array, 5);
-                if (size)
-                        callingfn = backtrace_symbols (&array[2], size-2);
-                if (!callingfn)
-                        break;
-
-                if (size == 5)
-                        snprintf (callstr, 4096, "(-->%s (-->%s (-->%s)))",
-                                  callingfn[2], callingfn[1], callingfn[0]);
-                if (size == 4)
-                        snprintf (callstr, 4096, "(-->%s (-->%s))",
-                                  callingfn[1], callingfn[0]);
-                if (size == 3)
-                        snprintf (callstr, 4096, "(-->%s)", callingfn[0]);
-
-                free (callingfn);
-	} while (0);
-#endif /* HAVE_BACKTRACE */
-
-        ret = gettimeofday (&tv, NULL);
-        if (-1 == ret)
-                goto out;
+    if (old_logfile && (fclose(old_logfile) != 0))
+        ret = -1;
 
-	tm    = localtime (&tv.tv_sec);
+    GF_FREE(ctx->log.ident);
+    GF_FREE(ctx->log.filename);
 
-	pthread_mutex_lock (&logfile_mutex);
-	{
-		va_start (ap, fmt);
+out:
+    return ret;
+}
+
+/**
+ * gf_openlog -function to open syslog specific to gluster based on
+ *             existence of file /etc/glusterfs/logger.conf
+ * @ident:    optional identification string similar to openlog()
+ * @option:   optional value to option to openlog().  Passing -1 uses
+ *            'LOG_PID | LOG_NDELAY' as default
+ * @facility: optional facility code similar to openlog().  Passing -1
+ *            uses LOG_DAEMON as default
+ *
+ * @return: void
+ */
+static void
+gf_openlog(const char *ident, int option, int facility)
+{
+    int _option = option;
+    int _facility = facility;
+
+    if (-1 == _option) {
+        _option = LOG_PID | LOG_NDELAY;
+    }
+    if (-1 == _facility) {
+        _facility = LOG_LOCAL1;
+    }
+
+    /* TODO: Should check for errors here and return appropriately */
+    setlocale(LC_ALL, "");
+    setlocale(LC_NUMERIC, "C"); /* C-locale for strtod, ... */
+    /* close the previous syslog if open as we are changing settings */
+    closelog();
+    openlog(ident, _option, _facility);
+}
 
-		strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+/**
+ * _json_escape -function to convert string to json encoded string
+ * @str: input string
+ * @buf: buffer to store encoded string
+ * @len: length of @buf
+ *
+ * @return: success: last unprocessed character position by pointer in @str
+ *          failure: NULL
+ *
+ * Internal function. Heavily inspired by _ul_str_escape() function in
+ * libumberlog
+ *
+ * Sample output:
+ * [1] str = "devel error"
+ *     buf = "devel error"
+ * [2] str = "devel	error"
+ *     buf = "devel\terror"
+ * [3] str = "I/O error on "/tmp/foo" file"
+ *     buf = "I/O error on \"/tmp/foo\" file"
+ * [4] str = "I/O erroron /tmp/bar file"
+ *     buf = "I/O error\u001bon /tmp/bar file"
+ *
+ */
+static char *
+_json_escape(const char *str, char *buf, size_t len)
+{
+    static const unsigned char json_exceptions[UCHAR_MAX + 1] = {
+        [0x01] = 1, [0x02] = 1, [0x03] = 1, [0x04] = 1, [0x05] = 1, [0x06] = 1,
+        [0x07] = 1, [0x08] = 1, [0x09] = 1, [0x0a] = 1, [0x0b] = 1, [0x0c] = 1,
+        [0x0d] = 1, [0x0e] = 1, [0x0f] = 1, [0x10] = 1, [0x11] = 1, [0x12] = 1,
+        [0x13] = 1, [0x14] = 1, [0x15] = 1, [0x16] = 1, [0x17] = 1, [0x18] = 1,
+        [0x19] = 1, [0x1a] = 1, [0x1b] = 1, [0x1c] = 1, [0x1d] = 1, [0x1e] = 1,
+        [0x1f] = 1, ['\\'] = 1, ['"'] = 1};
+    static const char json_hex_chars[16] = "0123456789abcdef";
+    unsigned char *p = NULL;
+    size_t pos = 0;
+
+    if (!str || !buf || len <= 0) {
+        return NULL;
+    }
+
+    for (p = (unsigned char *)str; *p && (pos + 1) < len; p++) {
+        if (json_exceptions[*p] == 0) {
+            buf[pos++] = *p;
+            continue;
+        }
 
-		basename = strrchr (file, '/');
-		if (basename)
-			basename++;
-		else
-			basename = file;
+        if ((pos + 2) >= len) {
+            break;
+        }
 
-                ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %s %s: ",
-                                   timestr, level_strings[level],
-                                   basename, line, function, callstr,
-                                   domain);
-                if (-1 == ret) {
-                        goto unlock;
+        switch (*p) {
+            case '\b':
+                buf[pos++] = '\\';
+                buf[pos++] = 'b';
+                break;
+            case '\n':
+                buf[pos++] = '\\';
+                buf[pos++] = 'n';
+                break;
+            case '\r':
+                buf[pos++] = '\\';
+                buf[pos++] = 'r';
+                break;
+            case '\t':
+                buf[pos++] = '\\';
+                buf[pos++] = 't';
+                break;
+            case '\\':
+                buf[pos++] = '\\';
+                buf[pos++] = '\\';
+                break;
+            case '"':
+                buf[pos++] = '\\';
+                buf[pos++] = '"';
+                break;
+            default:
+                if ((pos + 6) >= len) {
+                    buf[pos] = '\0';
+                    return (char *)p;
                 }
+                buf[pos++] = '\\';
+                buf[pos++] = 'u';
+                buf[pos++] = '0';
+                buf[pos++] = '0';
+                buf[pos++] = json_hex_chars[(*p) >> 4];
+                buf[pos++] = json_hex_chars[(*p) & 0xf];
+                break;
+        }
+    }
 
-                ret = vasprintf (&str2, fmt, ap);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+    buf[pos] = '\0';
+    return (char *)p;
+}
 
-		va_end (ap);
+/**
+ * gf_syslog -function to submit message to syslog specific to gluster
+ * @facility_priority: facility_priority of syslog()
+ * @format:            optional format string to syslog()
+ *
+ * @return: void
+ */
+static void
+gf_syslog(int facility_priority, char *format, ...)
+{
+    char *msg = NULL;
+    char json_msg[GF_JSON_MSG_LENGTH];
+    GF_UNUSED char *p = NULL;
+    va_list ap;
+
+    GF_ASSERT(format);
+
+    va_start(ap, format);
+    if (vasprintf(&msg, format, ap) != -1) {
+        p = _json_escape(msg, json_msg, GF_JSON_MSG_LENGTH);
+        syslog(facility_priority, "%s", msg);
+        free(msg);
+    } else
+        syslog(GF_LOG_CRITICAL, "vasprintf() failed, out of memory?");
+    va_end(ap);
+}
 
-                len = strlen (str1);
-                msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
+void
+gf_log_globals_init(void *data, gf_loglevel_t level)
+{
+    glusterfs_ctx_t *ctx = data;
 
-                strcpy (msg, str1);
-                strcpy (msg + len, str2);
+    pthread_mutex_init(&ctx->log.logfile_mutex, NULL);
 
-		fprintf (logfile, "%s\n", msg);
-		fflush (logfile);
+    ctx->log.loglevel = level;
+    ctx->log.gf_log_syslog = 1;
+    ctx->log.sys_log_level = GF_LOG_CRITICAL;
+    ctx->log.logger = gf_logger_glusterlog;
+    ctx->log.logformat = gf_logformat_withmsgid;
+    ctx->log.lru_size = GF_LOG_LRU_BUFSIZE_DEFAULT;
+    ctx->log.timeout = GF_LOG_FLUSH_TIMEOUT_DEFAULT;
+    ctx->log.localtime = GF_LOG_LOCALTIME_DEFAULT;
+
+    pthread_mutex_init(&ctx->log.log_buf_lock, NULL);
+
+    INIT_LIST_HEAD(&ctx->log.lru_queue);
 
 #ifdef GF_LINUX_HOST_OS
-                /* We want only serious log in 'syslog', not our debug
-                   and trace logs */
-                if (gf_log_syslog && level && (level <= GF_LOG_ERROR))
-                        syslog ((level-1), "%s\n", msg);
+    /* For the 'syslog' output. one can grep 'GlusterFS' in syslog
+       for serious logs */
+    openlog("GlusterFS", LOG_PID, LOG_DAEMON);
 #endif
-	}
+}
 
-unlock:
-	pthread_mutex_unlock (&logfile_mutex);
+int
+gf_log_init(void *data, const char *file, const char *ident)
+{
+    glusterfs_ctx_t *ctx = data;
+    int fd = -1;
+    struct stat buf;
+
+    if (ctx == NULL) {
+        fprintf(stderr, "ERROR: ctx is NULL\n");
+        return -1;
+    }
+    if (ident) {
+        GF_FREE(ctx->log.ident);
+        ctx->log.ident = gf_strdup(ident);
+    }
+
+    /* we keep the files and the syslog open, so that on logger change, we
+     * are ready to log anywhere, that the new value specifies */
+    if (ctx->log.ident) {
+        gf_openlog(ctx->log.ident, -1, LOG_DAEMON);
+    } else {
+        gf_openlog(NULL, -1, LOG_DAEMON);
+    }
+    /* TODO: make FACILITY configurable than LOG_DAEMON */
+    if (sys_stat(GF_LOG_CONTROL_FILE, &buf) == 0) {
+        /* use syslog logging */
+        ctx->log.log_control_file_found = 1;
+    } else {
+        /* use old style logging */
+        ctx->log.log_control_file_found = 0;
+    }
+
+    if (!file) {
+        fprintf(stderr, "ERROR: no filename specified\n");
+        return -1;
+    }
+
+    /* free the (possible) previous filename */
+    GF_FREE(ctx->log.filename);
+    ctx->log.filename = NULL;
+
+    /* close and reopen logfile for log rotate */
+    if (ctx->log.logfile) {
+        fclose(ctx->log.logfile);
+        ctx->log.logfile = NULL;
+        ctx->log.gf_log_logfile = NULL;
+    }
+
+    if (strcmp(file, "-") == 0) {
+        int dupfd = -1;
+
+        ctx->log.filename = gf_strdup("/dev/stderr");
+        if (!ctx->log.filename) {
+            fprintf(stderr, "ERROR: strdup failed\n");
+            return -1;
+        }
+
+        dupfd = dup(fileno(stderr));
+        if (dupfd == -1) {
+            fprintf(stderr, "ERROR: could not dup %d (%s)\n", fileno(stderr),
+                    strerror(errno));
+            return -1;
+        }
 
-        if (msg) {
-                GF_FREE (msg);
+        ctx->log.logfile = fdopen(dupfd, "a");
+        if (!ctx->log.logfile) {
+            fprintf(stderr, "ERROR: could not fdopen on %d (%s)\n", dupfd,
+                    strerror(errno));
+            sys_close(dupfd);
+            return -1;
+        }
+    } else {
+        /* Also create parent dir */
+        char *logdir = gf_strdup(file);
+        if (!logdir) {
+            return -1;
+        }
+        char *tmp_index = rindex(logdir, '/');
+        if (tmp_index) {
+            tmp_index[0] = '\0';
+        }
+        if (mkdir_p(logdir, 0755, true)) {
+            /* EEXIST is handled in mkdir_p() itself */
+            gf_smsg("logging", GF_LOG_ERROR, 0, LG_MSG_STRDUP_ERROR,
+                    "logdir=%s", logdir, "errno=%s", strerror(errno), NULL);
+            GF_FREE(logdir);
+            return -1;
+        }
+        /* no need of this variable */
+        GF_FREE(logdir);
+
+        ctx->log.filename = gf_strdup(file);
+        if (!ctx->log.filename) {
+            fprintf(stderr,
+                    "ERROR: updating log-filename failed: "
+                    "%s\n",
+                    strerror(errno));
+            return -1;
         }
 
-        if (str1)
-                GF_FREE (str1);
+        fd = sys_open(file, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+            fprintf(stderr,
+                    "ERROR: failed to create logfile"
+                    " \"%s\" (%s)\n",
+                    file, strerror(errno));
+            return -1;
+        }
 
-        if (str2)
-                FREE (str2);
+        ctx->log.logfile = fdopen(fd, "a");
+        if (!ctx->log.logfile) {
+            fprintf(stderr,
+                    "ERROR: failed to open logfile \"%s\" "
+                    "(%s)\n",
+                    file, strerror(errno));
+            sys_close(fd);
+            return -1;
+        }
+    }
 
-out:
-        return ret;
+    ctx->log.gf_log_logfile = ctx->log.logfile;
+
+    return 0;
 }
 
-int
-_gf_log (const char *domain, const char *file, const char *function, int line,
-	 gf_loglevel_t level, const char *fmt, ...)
-{
-	const char  *basename = NULL;
-	FILE        *new_logfile = NULL;
-	va_list      ap;
-	struct tm   *tm = NULL;
-	char         timestr[256];
-        struct timeval tv = {0,};
-
-        char        *str1 = NULL;
-        char        *str2 = NULL;
-        char        *msg  = NULL;
-        size_t       len  = 0;
-        int          ret  = 0;
-        xlator_t    *this = NULL;
-        gf_loglevel_t xlator_loglevel = 0;
-
-	if (!logfile)
-		return -1;
-
-        this = THIS;
-
-        xlator_loglevel = this->loglevel;
-        if (xlator_loglevel == 0)
-                xlator_loglevel = loglevel;
-
-        if (level > xlator_loglevel)
-                goto out;
+void
+set_sys_log_level(gf_loglevel_t level)
+{
+    if (THIS->ctx)
+        THIS->ctx->log.sys_log_level = level;
+}
 
-	static char *level_strings[] = {"",  /* NONE */
-                                        "M", /* EMERGENCY */
-                                        "A", /* ALERT */
-					"C", /* CRITICAL */
-					"E", /* ERROR */
-					"W", /* WARNING */
-					"N", /* NOTICE */
-                                        "I", /* INFO/NORMAL */
-					"D", /* DEBUG */
-                                        "T", /* TRACE */
-					""};
-
-	if (!domain || !file || !function || !fmt) {
-		fprintf (stderr,
-			 "logging: %s:%s():%d: invalid argument\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-		return -1;
-	}
-
-
-	if (logrotate) {
-		logrotate = 0;
-
-		new_logfile = fopen (filename, "a");
-		if (!new_logfile) {
-			gf_log ("logrotate", GF_LOG_CRITICAL,
-				"failed to open logfile %s (%s)",
-				filename, strerror (errno));
-			goto log;
-		}
-
-		fclose (logfile);
-		gf_log_logfile = logfile = new_logfile;
-	}
+/* Check if we should be logging
+ * Return value: _gf_false : Print the log
+ *               _gf_true : Do not Print the log
+ */
+static gf_boolean_t
+skip_logging(xlator_t *this, gf_loglevel_t level)
+{
+    gf_loglevel_t existing_level = this->loglevel ? this->loglevel
+                                                  : this->ctx->log.loglevel;
+    if (level > existing_level) {
+        return _gf_true;
+    }
 
-log:
-        ret = gettimeofday (&tv, NULL);
-        if (-1 == ret)
-                goto out;
+    if (level == GF_LOG_NONE) {
+        return _gf_true;
+    }
 
-	tm    = localtime (&tv.tv_sec);
+    return _gf_false;
+}
 
-	pthread_mutex_lock (&logfile_mutex);
-	{
-		va_start (ap, fmt);
+int
+_gf_log_callingfn(const char *domain, const char *file, const char *function,
+                  int line, gf_loglevel_t level, const char *fmt, ...)
+{
+    const char *basename = NULL;
+    xlator_t *this = THIS;
+    char *logline = NULL;
+    char *msg = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *callstr = NULL;
+    struct timeval tv = {
+        0,
+    };
+    int ret = 0;
+    va_list ap;
+    glusterfs_ctx_t *ctx = this->ctx;
+
+    if (!ctx)
+        goto out;
+
+    if (skip_logging(this, level))
+        goto out;
+
+    if (!domain || !file || !function || !fmt) {
+        fprintf(stderr, "logging: %s:%s():%d: invalid argument\n", __FILE__,
+                __PRETTY_FUNCTION__, __LINE__);
+        return -1;
+    }
+
+    basename = strrchr(file, '/');
+    if (basename)
+        basename++;
+    else
+        basename = file;
+
+    /*Saving the backtrace to pre-allocated ctx->btbuf
+     * to avoid allocating memory from the heap*/
+    callstr = gf_backtrace_save(NULL);
+
+    va_start(ap, fmt);
+    ret = vasprintf(&msg, fmt, ap);
+    va_end(ap);
+    if (-1 == ret) {
+        goto out;
+    }
+
+    if (ctx->log.log_control_file_found) {
+        int priority;
+        /* treat GF_LOG_TRACE and GF_LOG_NONE as LOG_DEBUG and
+           other level as is */
+        if (GF_LOG_TRACE == level || GF_LOG_NONE == level) {
+            priority = LOG_DEBUG;
+        } else {
+            priority = level - 1;
+        }
 
-		strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm); 
-                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        gf_syslog(priority, "[%s:%d:%s] %s %d-%s: %s", basename, line, function,
+                  callstr, ((this->graph) ? this->graph->id : 0), domain, msg);
+
+        goto out;
+    }
+
+    ret = gettimeofday(&tv, NULL);
+    if (-1 == ret)
+        goto out;
+
+    gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT);
+
+    ret = gf_asprintf(&logline, "[%s] %c [%s:%d:%s] %s %d-%s: %s\n", timestr,
+                      gf_level_strings[level], basename, line, function,
+                      callstr, ((this->graph) ? this->graph->id : 0), domain,
+                      msg);
+    if (-1 == ret) {
+        goto out;
+    }
+
+    pthread_mutex_lock(&ctx->log.logfile_mutex);
+    {
+        if (ctx->log.logfile) {
+            fputs(logline, ctx->log.logfile);
+            fflush(ctx->log.logfile);
+        } else if (ctx->log.loglevel >= level) {
+            fputs(logline, stderr);
+            fflush(stderr);
+        }
 
-		basename = strrchr (file, '/');
-		if (basename)
-			basename++;
-		else
-			basename = file;
+#ifdef GF_LINUX_HOST_OS
+        /* We want only serious log in 'syslog', not our debug
+           and trace logs */
+        if (ctx->log.gf_log_syslog && level &&
+            (level <= ctx->log.sys_log_level))
+            syslog((level - 1), "%s", logline);
+#endif
+    }
 
-                ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %s: ",
-                                   timestr, level_strings[level],
-                                   basename, line, function,
-                                   domain);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+    pthread_mutex_unlock(&ctx-
author	Krishnan Parthasarathi <kp@gluster.com>	2011-09-23 11:49:29 +0530
committer	Vijay Bellur <vijay@gluster.com>	2011-09-23 01:19:38 -0700
commit	2b21ea3c625fab19e88c21f4c104dca321eef262 (patch)
tree	afc74ccaa9a1c6ed412aace2f9403933aa4bfdf5 /cli
parent	4765dd1a1c51c67ab86687fbd871c89156680c34 (diff)